This is an automated email from the ASF dual-hosted git repository. kxiao pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 74e5ecd3193eb1faff7bcf150f244930a4d94a6e Author: AlexYue <[email protected]> AuthorDate: Mon Oct 9 11:28:48 2023 +0800 [bugfix](DDL) Fix the bug of incorrect partition policy setting (#25021) --- .../apache/doris/datasource/InternalCatalog.java | 15 ++++++--- .../java/org/apache/doris/alter/AlterTest.java | 37 ++++++++++++++++++++++ 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 308fe182997..65c8f885604 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -2498,10 +2498,13 @@ public class InternalCatalog implements CatalogIf<Database> { "Can not create UNIQUE KEY table that enables Merge-On-write" + " with storage policy(" + partionStoragePolicy + ")"); } - if (!partionStoragePolicy.equals("")) { - storagePolicy = partionStoragePolicy; + // The table's storage policy has higher priority than partition's policy, + // so we'll directly use table's policy when it's set. Otherwise we use the + // partition's policy + if (!storagePolicy.isEmpty()) { + partionStoragePolicy = storagePolicy; } - Env.getCurrentEnv().getPolicyMgr().checkStoragePolicyExist(storagePolicy); + Env.getCurrentEnv().getPolicyMgr().checkStoragePolicyExist(partionStoragePolicy); Partition partition = createPartitionWithIndices(db.getClusterName(), db.getId(), olapTable.getId(), olapTable.getName(), olapTable.getBaseIndexId(), entry.getValue(), @@ -2509,8 +2512,8 @@ public class InternalCatalog implements CatalogIf<Database> { dataProperty.getStorageMedium(), partitionInfo.getReplicaAllocation(entry.getValue()), versionInfo, bfColumns, bfFpp, tabletIdSet, olapTable.getCopiedIndexes(), isInMemory, storageFormat, partitionInfo.getTabletType(entry.getValue()), compressionType, - olapTable.getDataSortInfo(), olapTable.getEnableUniqueKeyMergeOnWrite(), storagePolicy, - idGeneratorBuffer, olapTable.disableAutoCompaction(), + olapTable.getDataSortInfo(), olapTable.getEnableUniqueKeyMergeOnWrite(), + partionStoragePolicy, idGeneratorBuffer, olapTable.disableAutoCompaction(), olapTable.enableSingleReplicaCompaction(), skipWriteIndexOnLoad, olapTable.getCompactionPolicy(), olapTable.getTimeSeriesCompactionGoalSizeMbytes(), olapTable.getTimeSeriesCompactionFileCountThreshold(), @@ -2518,6 +2521,8 @@ public class InternalCatalog implements CatalogIf<Database> { storeRowColumn, isDynamicSchema, binlogConfigForTask, dataProperty.isStorageMediumSpecified()); olapTable.addPartition(partition); + olapTable.getPartitionInfo().getDataProperty(partition.getId()) + .setStoragePolicy(partionStoragePolicy); } } else { throw new DdlException("Unsupported partition method: " + partitionInfo.getType().name()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java index a3ddd9991a8..924fcd53b7d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java @@ -195,6 +195,14 @@ public class AlterTest { "CREATE STORAGE POLICY testPolicyAnotherResource\n" + "PROPERTIES(\n" + " \"storage_resource\" = \"remote_s3_1\",\n" + " \"cooldown_ttl\" = \"1\"\n" + ");"); + createTable("CREATE TABLE test.tbl_remote1\n" + "(\n" + " k1 date,\n" + " k2 int,\n" + " v1 int sum\n" + + ")\n" + "PARTITION BY RANGE(k1)\n" + "(\n" + " PARTITION p1 values less than('2020-02-01'),\n" + + " PARTITION p2 values less than('2020-03-01') ('storage_policy' = 'testPolicy'),\n" + + " PARTITION p3 values less than('2020-04-01'),\n" + + " PARTITION p4 values less than('2020-05-01')\n" + ")\n" + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + + "PROPERTIES" + "(" + " 'replication_num' = '1',\n" + " 'in_memory' = 'false',\n" + + " 'storage_medium' = 'SSD',\n" + " 'storage_cooldown_time' = '2100-05-09 00:00:00'\n" + ");"); + createTable("CREATE TABLE test.tbl_remote\n" + "(\n" + " k1 date,\n" + " k2 int,\n" + " v1 int sum\n" + ")\n" + "PARTITION BY RANGE(k1)\n" + "(\n" + " PARTITION p1 values less than('2020-02-01'),\n" + " PARTITION p2 values less than('2020-03-01'),\n" @@ -685,6 +693,35 @@ public class AlterTest { } + @Test + public void testAlterRemoteStorageTableDataPropertiesPolicy() throws Exception { + Database db = Env.getCurrentInternalCatalog().getDbOrMetaException("default_cluster:test"); + OlapTable tblRemote = (OlapTable) db.getTableOrMetaException("tbl_remote1"); + Partition p1 = tblRemote.getPartition("p1"); + Partition p2 = tblRemote.getPartition("p2"); + Assert.assertEquals(tblRemote.getPartitionInfo().getStoragePolicy(p2.getId()), "testPolicy"); + Partition p3 = tblRemote.getPartition("p3"); + Partition p4 = tblRemote.getPartition("p4"); + + DateLiteral dateLiteral = new DateLiteral("2100-05-09 00:00:00", Type.DATETIME); + long cooldownTimeMs = dateLiteral.unixTimestamp(TimeUtils.getTimeZone()); + DataProperty oldDataPropertyWithPolicy = new DataProperty(TStorageMedium.SSD, cooldownTimeMs, "testPolicy"); + List<Partition> partitionList = Lists.newArrayList(p1, p3, p4); + for (Partition partition : partitionList) { + Assert.assertEquals(tblRemote.getPartitionInfo().getStoragePolicy(partition.getId()), ""); + } + Assert.assertEquals(oldDataPropertyWithPolicy, tblRemote.getPartitionInfo().getDataProperty(p2.getId())); + + // alter recover to old state + String stmt = "alter table test.tbl_remote1 modify partition (p1, p3, p4) set (" + + "'storage_policy' = 'testPolicy')"; + alterTable(stmt, false); + for (Partition partition : partitionList) { + Assert.assertEquals(tblRemote.getPartitionInfo().getStoragePolicy(partition.getId()), "testPolicy"); + } + + } + @Test public void testDynamicPartitionDropAndAdd() throws Exception { // test day range --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
