Murtadha Hubail has submitted this change and it was merged. Change subject: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge ......................................................................
[ASTERIXDB-2316][STO] Fix Merging Components For Full Merge - user model changes: no - storage format changes: no - interface changes: no Details: - Pass correct merging components in case of full merge. - Add full merge test case. Change-Id: I17890a1f85f569e0d3dc5eba046f61c91571c9c1 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2471 Sonar-Qube: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Contrib: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Reviewed-by: Luo Chen <cl...@uci.edu> --- A asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/LsmIndexLifecycleTest.java M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java 2 files changed, 74 insertions(+), 1 deletion(-) Approvals: Luo Chen: Looks good to me, approved Anon. E. Moose #1000171: Jenkins: Verified; No violations found; ; Verified diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/LsmIndexLifecycleTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/LsmIndexLifecycleTest.java new file mode 100644 index 0000000..15f1730 --- /dev/null +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/LsmIndexLifecycleTest.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.test.storage; + +import org.apache.asterix.api.common.AsterixHyracksIntegrationUtil; +import org.apache.asterix.common.TestDataUtil; +import org.apache.asterix.common.api.IDatasetLifecycleManager; +import org.apache.asterix.common.api.INcApplicationContext; +import org.apache.asterix.common.config.GlobalConfig; +import org.apache.asterix.common.context.DatasetInfo; +import org.apache.asterix.metadata.api.IMetadataIndex; +import org.apache.asterix.metadata.bootstrap.MetadataPrimaryIndexes; +import org.apache.asterix.test.common.TestExecutor; +import org.apache.asterix.testframework.context.TestCaseContext; +import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class LsmIndexLifecycleTest { + + protected static final String TEST_CONFIG_FILE_NAME = "src/main/resources/cc.conf"; + private static final TestExecutor testExecutor = new TestExecutor(); + private static final AsterixHyracksIntegrationUtil integrationUtil = new AsterixHyracksIntegrationUtil(); + + @Before + public void setUp() throws Exception { + System.setProperty(GlobalConfig.CONFIG_FILE_PROPERTY, TEST_CONFIG_FILE_NAME); + integrationUtil.init(true, TEST_CONFIG_FILE_NAME); + } + + @After + public void tearDown() throws Exception { + integrationUtil.deinit(true); + } + + @Test + public void fullMergeTest() throws Exception { + String datasetName = "ds"; + TestDataUtil.createIdOnlyDataset(datasetName); + INcApplicationContext appCtx = (INcApplicationContext) (integrationUtil.ncs[0].getApplicationContext()); + IDatasetLifecycleManager dlcm = appCtx.getDatasetLifecycleManager(); + IMetadataIndex dsIdx = MetadataPrimaryIndexes.DATASET_DATASET; + DatasetInfo datasetInfo = dlcm.getDatasetInfo(dsIdx.getDatasetId().getId()); + // flush to ensure multiple disk components + dlcm.flushAllDatasets(); + datasetInfo.waitForIO(); + AbstractLSMIndex index = (AbstractLSMIndex) dlcm.getIndex(dsIdx.getDatasetId().getId(), dsIdx.getResourceId()); + Assert.assertTrue(index.getDiskComponents().size() > 1); + // trigger full merge and ensure we have a single disk component when merge completes + testExecutor.executeSqlppUpdateOrDdl("COMPACT DATASET Metadata.`Dataset`;", + TestCaseContext.OutputFormat.CLEAN_JSON); + datasetInfo.waitForIO(); + Assert.assertTrue(index.getDiskComponents().size() == 1); + } +} diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java index bd91094..af8c702 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java @@ -369,7 +369,7 @@ AbstractLSMIndexOperationContext opCtx = createOpContext(NoOpIndexAccessParameters.INSTANCE); opCtx.setOperation(ctx.getOperation()); opCtx.getComponentHolder().addAll(mergingComponents); - opCtx.getComponentsToBeMerged().addAll(ctx.getComponentsToBeMerged()); + mergingComponents.stream().map(ILSMDiskComponent.class::cast).forEach(opCtx.getComponentsToBeMerged()::add); ILSMDiskComponent firstComponent = (ILSMDiskComponent) mergingComponents.get(0); ILSMDiskComponent lastComponent = (ILSMDiskComponent) mergingComponents.get(mergingComponents.size() - 1); LSMComponentFileReferences mergeFileRefs = getMergeFileReferences(firstComponent, lastComponent); -- To view, visit https://asterix-gerrit.ics.uci.edu/2471 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: I17890a1f85f569e0d3dc5eba046f61c91571c9c1 Gerrit-PatchSet: 2 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Murtadha Hubail <mhub...@apache.org> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Gerrit-Reviewer: Luo Chen <cl...@uci.edu> Gerrit-Reviewer: Murtadha Hubail <mhub...@apache.org> Gerrit-Reviewer: abdullah alamoudi <bamou...@gmail.com>
