naveenkrdremio opened a new issue #3849:
URL: https://github.com/apache/iceberg/issues/3849
After running expireSnapshot(snapshotid) . It deleted schema details
corresponding to snapshots from metadata file.
v23 is before expire. v24 is after expire snapshot.
V23 ->
`{
"format-version" : 1,
"table-uuid" : "093d3fd7-ab2b-4ebd-aa2a-2389491254a7",
"location" : "/Users/naveen/Desktop/dremio_work/warehouse/table",
"last-updated-ms" : 1641312141982,
"last-column-id" : 5,
"schema" : {
"type" : "struct",
"schema-id" : 5,
"fields" : [ {
"id" : 1,
"name" : "id",
"required" : false,
"type" : "long"
}, {
"id" : 4,
"name" : "new_column2",
"required" : false,
"type" : "string",
"doc" : "new_column docs"
}, {
"id" : 5,
"name" : "new_column",
"required" : false,
"type" : "string",
"doc" : "new_column docs"
} ]
},
"current-schema-id" : 5,
"schemas" : [ {
"type" : "struct",
"schema-id" : 0,
"fields" : [ {
"id" : 1,
"name" : "id",
"required" : false,
"type" : "long"
}, {
"id" : 2,
"name" : "data",
"required" : false,
"type" : "string"
} ]
}, {
"type" : "struct",
"schema-id" : 1,
"fields" : [ {
"id" : 1,
"name" : "id",
"required" : false,
"type" : "long"
}, {
"id" : 2,
"name" : "data",
"required" : false,
"type" : "string"
}, {
"id" : 3,
"name" : "new_column",
"required" : false,
"type" : "string",
"doc" : "new_column docs"
} ]
}, {
"type" : "struct",
"schema-id" : 2,
"fields" : [ {
"id" : 1,
"name" : "id",
"required" : false,
"type" : "long"
}, {
"id" : 2,
"name" : "data",
"required" : false,
"type" : "string"
}, {
"id" : 3,
"name" : "new_column",
"required" : false,
"type" : "string",
"doc" : "new_column docs"
}, {
"id" : 4,
"name" : "new_column2",
"required" : false,
"type" : "string",
"doc" : "new_column docs"
} ]
}, {
"type" : "struct",
"schema-id" : 3,
"fields" : [ {
"id" : 1,
"name" : "id",
"required" : false,
"type" : "long"
}, {
"id" : 2,
"name" : "data",
"required" : false,
"type" : "string"
}, {
"id" : 4,
"name" : "new_column2",
"required" : false,
"type" : "string",
"doc" : "new_column docs"
} ]
}, {
"type" : "struct",
"schema-id" : 4,
"fields" : [ {
"id" : 1,
"name" : "id",
"required" : false,
"type" : "long"
}, {
"id" : 4,
"name" : "new_column2",
"required" : false,
"type" : "string",
"doc" : "new_column docs"
} ]
}, {
"type" : "struct",
"schema-id" : 5,
"fields" : [ {
"id" : 1,
"name" : "id",
"required" : false,
"type" : "long"
}, {
"id" : 4,
"name" : "new_column2",
"required" : false,
"type" : "string",
"doc" : "new_column docs"
}, {
"id" : 5,
"name" : "new_column",
"required" : false,
"type" : "string",
"doc" : "new_column docs"
} ]
} ],
"partition-spec" : [ ],
"default-spec-id" : 0,
"partition-specs" : [ {
"spec-id" : 0,
"fields" : [ ]
} ],
"last-partition-id" : 999,
"default-sort-order-id" : 0,
"sort-orders" : [ {
"order-id" : 0,
"fields" : [ ]
} ],
"properties" : {
"owner" : "naveen"
},
"current-snapshot-id" : 7991945625433166788,
"snapshots" : [ {
"snapshot-id" : 1317717662965330171,
"parent-snapshot-id" : 3688303849786903557,
"timestamp-ms" : 1640714268405,
"summary" : {
"operation" : "append",
"spark.app.id" : "local-1639995196963",
"added-data-files" : "1",
"added-records" : "1",
"added-files-size" : "949",
"changed-partition-count" : "1",
"total-records" : "22",
"total-files-size" : "14848",
"total-data-files" : "22",
"total-delete-files" : "0",
"total-position-deletes" : "0",
"total-equality-deletes" : "0"
},
"manifest-list" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/snap-1317717662965330171-1-b111d87a-e8b2-4f54-a157-4df43562897b.avro"
}, {
"snapshot-id" : 8631079934421306470,
"parent-snapshot-id" : 1317717662965330171,
"timestamp-ms" : 1641312126753,
"summary" : {
"operation" : "append",
"spark.app.id" : "local-1639995196963",
"added-data-files" : "1",
"added-records" : "1",
"added-files-size" : "949",
"changed-partition-count" : "1",
"total-records" : "23",
"total-files-size" : "15797",
"total-data-files" : "23",
"total-delete-files" : "0",
"total-position-deletes" : "0",
"total-equality-deletes" : "0"
},
"manifest-list" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/snap-8631079934421306470-1-145daba3-7847-4480-906b-e0fbbf21ded0.avro",
"schema-id" : 5
}, {
"snapshot-id" : 5458217040340265637,
"parent-snapshot-id" : 8631079934421306470,
"timestamp-ms" : 1641312133370,
"summary" : {
"operation" : "append",
"spark.app.id" : "local-1639995196963",
"added-data-files" : "1",
"added-records" : "1",
"added-files-size" : "949",
"changed-partition-count" : "1",
"total-records" : "24",
"total-files-size" : "16746",
"total-data-files" : "24",
"total-delete-files" : "0",
"total-position-deletes" : "0",
"total-equality-deletes" : "0"
},
"manifest-list" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/snap-5458217040340265637-1-ef3147fe-77ee-4672-badb-e47fd707ecca.avro",
"schema-id" : 5
}, {
"snapshot-id" : 674098525249392167,
"parent-snapshot-id" : 5458217040340265637,
"timestamp-ms" : 1641312137603,
"summary" : {
"operation" : "append",
"spark.app.id" : "local-1639995196963",
"added-data-files" : "1",
"added-records" : "1",
"added-files-size" : "949",
"changed-partition-count" : "1",
"total-records" : "25",
"total-files-size" : "17695",
"total-data-files" : "25",
"total-delete-files" : "0",
"total-position-deletes" : "0",
"total-equality-deletes" : "0"
},
"manifest-list" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/snap-674098525249392167-1-4f9c680b-0d62-452d-a835-36b93f204eb7.avro",
"schema-id" : 5
}, {
"snapshot-id" : 7991945625433166788,
"parent-snapshot-id" : 674098525249392167,
"timestamp-ms" : 1641312141982,
"summary" : {
"operation" : "append",
"spark.app.id" : "local-1639995196963",
"added-data-files" : "1",
"added-records" : "1",
"added-files-size" : "949",
"changed-partition-count" : "1",
"total-records" : "26",
"total-files-size" : "18644",
"total-data-files" : "26",
"total-delete-files" : "0",
"total-position-deletes" : "0",
"total-equality-deletes" : "0"
},
"manifest-list" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/snap-7991945625433166788-1-5e4ecf13-9edd-4994-805f-903373022ad9.avro",
"schema-id" : 5
} ],
"snapshot-log" : [ {
"timestamp-ms" : 1640714268405,
"snapshot-id" : 1317717662965330171
}, {
"timestamp-ms" : 1641312126753,
"snapshot-id" : 8631079934421306470
}, {
"timestamp-ms" : 1641312133370,
"snapshot-id" : 5458217040340265637
}, {
"timestamp-ms" : 1641312137603,
"snapshot-id" : 674098525249392167
}, {
"timestamp-ms" : 1641312141982,
"snapshot-id" : 7991945625433166788
} ],
"metadata-log" : [ {
"timestamp-ms" : 1639388532582,
"metadata-file" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v1.metadata.json"
}, {
"timestamp-ms" : 1639388640186,
"metadata-file" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v2.metadata.json"
}, {
"timestamp-ms" : 1639391088097,
"metadata-file" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v3.metadata.json"
}, {
"timestamp-ms" : 1639391155838,
"metadata-file" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v4.metadata.json"
}, {
"timestamp-ms" : 1639391191266,
"metadata-file" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v5.metadata.json"
}, {
"timestamp-ms" : 1639463996813,
"metadata-file" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v6.metadata.json"
}, {
"timestamp-ms" : 1639464007053,
"metadata-file" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v7.metadata.json"
}, {
"timestamp-ms" : 1640326100256,
"metadata-file" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v8.metadata.json"
}, {
"timestamp-ms" : 1640327712823,
"metadata-file" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v9.metadata.json"
}, {
"timestamp-ms" : 1640335309825,
"metadata-file" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v10.metadata.json"
}, {
"timestamp-ms" : 1640335348647,
"metadata-file" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v11.metadata.json"
}, {
"timestamp-ms" : 1640597946543,
"metadata-file" :
"file:///Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v12.metadata.json"
}, {
"timestamp-ms" : 1640712816426,
"metadata-file" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v13.metadata.json"
}, {
"timestamp-ms" : 1640712911844,
"metadata-file" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v14.metadata.json"
}, {
"timestamp-ms" : 1640713990642,
"metadata-file" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v15.metadata.json"
}, {
"timestamp-ms" : 1640714003290,
"metadata-file" :
"file:///Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v16.metadata.json"
}, {
"timestamp-ms" : 1640714246449,
"metadata-file" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v17.metadata.json"
}, {
"timestamp-ms" : 1640714268405,
"metadata-file" :
"file:///Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v18.metadata.json"
}, {
"timestamp-ms" : 1641311974881,
"metadata-file" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v19.metadata.json"
}, {
"timestamp-ms" : 1641312126753,
"metadata-file" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v20.metadata.json"
}, {
"timestamp-ms" : 1641312133370,
"metadata-file" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v21.metadata.json"
}, {
"timestamp-ms" : 1641312137603,
"metadata-file" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v22.metadata.json"
} ]
}`
V24 after expire
`{
"format-version" : 1,
"table-uuid" : "093d3fd7-ab2b-4ebd-aa2a-2389491254a7",
"location" : "/Users/naveen/Desktop/dremio_work/warehouse/table",
"last-updated-ms" : 1641313552506,
"last-column-id" : 5,
"schema" : {
"type" : "struct",
"schema-id" : 5,
"fields" : [ {
"id" : 1,
"name" : "id",
"required" : false,
"type" : "long"
}, {
"id" : 4,
"name" : "new_column2",
"required" : false,
"type" : "string",
"doc" : "new_column docs"
}, {
"id" : 5,
"name" : "new_column",
"required" : false,
"type" : "string",
"doc" : "new_column docs"
} ]
},
"current-schema-id" : 5,
"schemas" : [ {
"type" : "struct",
"schema-id" : 0,
"fields" : [ {
"id" : 1,
"name" : "id",
"required" : false,
"type" : "long"
}, {
"id" : 2,
"name" : "data",
"required" : false,
"type" : "string"
} ]
}, {
"type" : "struct",
"schema-id" : 1,
"fields" : [ {
"id" : 1,
"name" : "id",
"required" : false,
"type" : "long"
}, {
"id" : 2,
"name" : "data",
"required" : false,
"type" : "string"
}, {
"id" : 3,
"name" : "new_column",
"required" : false,
"type" : "string",
"doc" : "new_column docs"
} ]
}, {
"type" : "struct",
"schema-id" : 2,
"fields" : [ {
"id" : 1,
"name" : "id",
"required" : false,
"type" : "long"
}, {
"id" : 2,
"name" : "data",
"required" : false,
"type" : "string"
}, {
"id" : 3,
"name" : "new_column",
"required" : false,
"type" : "string",
"doc" : "new_column docs"
}, {
"id" : 4,
"name" : "new_column2",
"required" : false,
"type" : "string",
"doc" : "new_column docs"
} ]
}, {
"type" : "struct",
"schema-id" : 3,
"fields" : [ {
"id" : 1,
"name" : "id",
"required" : false,
"type" : "long"
}, {
"id" : 2,
"name" : "data",
"required" : false,
"type" : "string"
}, {
"id" : 4,
"name" : "new_column2",
"required" : false,
"type" : "string",
"doc" : "new_column docs"
} ]
}, {
"type" : "struct",
"schema-id" : 4,
"fields" : [ {
"id" : 1,
"name" : "id",
"required" : false,
"type" : "long"
}, {
"id" : 4,
"name" : "new_column2",
"required" : false,
"type" : "string",
"doc" : "new_column docs"
} ]
}, {
"type" : "struct",
"schema-id" : 5,
"fields" : [ {
"id" : 1,
"name" : "id",
"required" : false,
"type" : "long"
}, {
"id" : 4,
"name" : "new_column2",
"required" : false,
"type" : "string",
"doc" : "new_column docs"
}, {
"id" : 5,
"name" : "new_column",
"required" : false,
"type" : "string",
"doc" : "new_column docs"
} ]
} ],
"partition-spec" : [ ],
"default-spec-id" : 0,
"partition-specs" : [ {
"spec-id" : 0,
"fields" : [ ]
} ],
"last-partition-id" : 999,
"default-sort-order-id" : 0,
"sort-orders" : [ {
"order-id" : 0,
"fields" : [ ]
} ],
"properties" : {
"owner" : "naveen"
},
"current-snapshot-id" : 7991945625433166788,
"snapshots" : [ {
"snapshot-id" : 8631079934421306470,
"parent-snapshot-id" : 1317717662965330171,
"timestamp-ms" : 1641312126753,
"summary" : {
"operation" : "append",
"spark.app.id" : "local-1639995196963",
"added-data-files" : "1",
"added-records" : "1",
"added-files-size" : "949",
"changed-partition-count" : "1",
"total-records" : "23",
"total-files-size" : "15797",
"total-data-files" : "23",
"total-delete-files" : "0",
"total-position-deletes" : "0",
"total-equality-deletes" : "0"
},
"manifest-list" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/snap-8631079934421306470-1-145daba3-7847-4480-906b-e0fbbf21ded0.avro"
}, {
"snapshot-id" : 674098525249392167,
"parent-snapshot-id" : 5458217040340265637,
"timestamp-ms" : 1641312137603,
"summary" : {
"operation" : "append",
"spark.app.id" : "local-1639995196963",
"added-data-files" : "1",
"added-records" : "1",
"added-files-size" : "949",
"changed-partition-count" : "1",
"total-records" : "25",
"total-files-size" : "17695",
"total-data-files" : "25",
"total-delete-files" : "0",
"total-position-deletes" : "0",
"total-equality-deletes" : "0"
},
"manifest-list" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/snap-674098525249392167-1-4f9c680b-0d62-452d-a835-36b93f204eb7.avro"
}, {
"snapshot-id" : 7991945625433166788,
"parent-snapshot-id" : 674098525249392167,
"timestamp-ms" : 1641312141982,
"summary" : {
"operation" : "append",
"spark.app.id" : "local-1639995196963",
"added-data-files" : "1",
"added-records" : "1",
"added-files-size" : "949",
"changed-partition-count" : "1",
"total-records" : "26",
"total-files-size" : "18644",
"total-data-files" : "26",
"total-delete-files" : "0",
"total-position-deletes" : "0",
"total-equality-deletes" : "0"
},
"manifest-list" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/snap-7991945625433166788-1-5e4ecf13-9edd-4994-805f-903373022ad9.avro"
} ],
"snapshot-log" : [ {
"timestamp-ms" : 1641312137603,
"snapshot-id" : 674098525249392167
}, {
"timestamp-ms" : 1641312141982,
"snapshot-id" : 7991945625433166788
} ],
"metadata-log" : [ {
"timestamp-ms" : 1639388532582,
"metadata-file" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v1.metadata.json"
}, {
"timestamp-ms" : 1639388640186,
"metadata-file" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v2.metadata.json"
}, {
"timestamp-ms" : 1639391088097,
"metadata-file" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v3.metadata.json"
}, {
"timestamp-ms" : 1639391155838,
"metadata-file" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v4.metadata.json"
}, {
"timestamp-ms" : 1639391191266,
"metadata-file" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v5.metadata.json"
}, {
"timestamp-ms" : 1639463996813,
"metadata-file" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v6.metadata.json"
}, {
"timestamp-ms" : 1639464007053,
"metadata-file" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v7.metadata.json"
}, {
"timestamp-ms" : 1640326100256,
"metadata-file" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v8.metadata.json"
}, {
"timestamp-ms" : 1640327712823,
"metadata-file" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v9.metadata.json"
}, {
"timestamp-ms" : 1640335309825,
"metadata-file" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v10.metadata.json"
}, {
"timestamp-ms" : 1640335348647,
"metadata-file" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v11.metadata.json"
}, {
"timestamp-ms" : 1640597946543,
"metadata-file" :
"file:///Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v12.metadata.json"
}, {
"timestamp-ms" : 1640712816426,
"metadata-file" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v13.metadata.json"
}, {
"timestamp-ms" : 1640712911844,
"metadata-file" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v14.metadata.json"
}, {
"timestamp-ms" : 1640713990642,
"metadata-file" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v15.metadata.json"
}, {
"timestamp-ms" : 1640714003290,
"metadata-file" :
"file:///Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v16.metadata.json"
}, {
"timestamp-ms" : 1640714246449,
"metadata-file" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v17.metadata.json"
}, {
"timestamp-ms" : 1640714268405,
"metadata-file" :
"file:///Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v18.metadata.json"
}, {
"timestamp-ms" : 1641311974881,
"metadata-file" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v19.metadata.json"
}, {
"timestamp-ms" : 1641312126753,
"metadata-file" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v20.metadata.json"
}, {
"timestamp-ms" : 1641312133370,
"metadata-file" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v21.metadata.json"
}, {
"timestamp-ms" : 1641312137603,
"metadata-file" :
"/Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v22.metadata.json"
}, {
"timestamp-ms" : 1641312141982,
"metadata-file" :
"file:///Users/naveen/Desktop/dremio_work/warehouse/table/metadata/v23.metadata.json"
} ]
}`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
