abdullah alamoudi has submitted this change and it was merged. Change subject: ASTERIXDB-1917: FLUSH_LSN for disk components is not correctly set ......................................................................
ASTERIXDB-1917: FLUSH_LSN for disk components is not correctly set -Fixed a bug that FLUSH_LSN for flushed disk components is not correctly set (not increasing) when an NC has multiple partitions. -Added LSMIOOperationCallback unit tests to cover this bug Change-Id: If438e34f8f612458d81f618eea04c0c72c49a9fe Reviewed-on: https://asterix-gerrit.ics.uci.edu/1771 Reviewed-by: abdullah alamoudi <[email protected]> Sonar-Qube: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> BAD: Jenkins <[email protected]> --- M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java A asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMBTreeIOOperationCallbackTest.java A asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallbackTest.java A asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMInvertedIndexIOOperationCallbackTest.java A asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMRTreeIOOperationCallbackTest.java 5 files changed, 344 insertions(+), 2 deletions(-) Approvals: abdullah alamoudi: Looks good to me, approved Jenkins: Verified; No violations found; No violations found diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java index f903b65..d04443a 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java @@ -107,8 +107,14 @@ return pointable.getLength() == 0 ? INVALID : pointable.longValue(); } - public void updateLastLSN(long lastLSN) { - mutableLastLSNs[writeIndex] = lastLSN; + public synchronized void updateLastLSN(long lastLSN) { + if (!flushRequested[writeIndex]) { + //if the memory component pointed by writeIndex is being flushed, we should ignore this update call + //since otherwise the original LSN is overwritten. + //Moreover, since the memory component is already being flushed, the next scheduleFlush request must fail. + //See https://issues.apache.org/jira/browse/ASTERIXDB-1917 + mutableLastLSNs[writeIndex] = lastLSN; + } } public void setFirstLSN(long firstLSN) { diff --git a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMBTreeIOOperationCallbackTest.java b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMBTreeIOOperationCallbackTest.java new file mode 100644 index 0000000..bc206e2 --- /dev/null +++ b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMBTreeIOOperationCallbackTest.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.asterix.test.ioopcallbacks; + +import org.apache.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallback; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent; +import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType; +import org.junit.Assert; +import org.mockito.Mockito; + +import junit.framework.TestCase; + +public class LSMBTreeIOOperationCallbackTest extends TestCase { + + public void testNormalSequence() { + try { + LSMBTreeIOOperationCallback callback = new LSMBTreeIOOperationCallback(); + callback.setNumOfMutableComponents(2); + + //request to flush first component + callback.updateLastLSN(1); + callback.beforeOperation(LSMOperationType.FLUSH); + + //request to flush second component + callback.updateLastLSN(2); + callback.beforeOperation(LSMOperationType.FLUSH); + + Assert.assertEquals(1, callback.getComponentLSN(null)); + callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class)); + + Assert.assertEquals(2, callback.getComponentLSN(null)); + callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class)); + } catch (Exception e) { + Assert.fail(); + } + } + + public void testOverWrittenLSN() { + try { + LSMBTreeIOOperationCallback callback = new LSMBTreeIOOperationCallback(); + callback.setNumOfMutableComponents(2); + + //request to flush first component + callback.updateLastLSN(1); + callback.beforeOperation(LSMOperationType.FLUSH); + + //request to flush second component + callback.updateLastLSN(2); + callback.beforeOperation(LSMOperationType.FLUSH); + + //request to flush first component again + //this call should fail + callback.updateLastLSN(3); + //there is no corresponding beforeOperation, since the first component is being flush + //the scheduleFlush request would fail this time + + Assert.assertEquals(1, callback.getComponentLSN(null)); + callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class)); + + Assert.assertEquals(2, callback.getComponentLSN(null)); + callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class)); + } catch (Exception e) { + Assert.fail(); + } + } + +} diff --git a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallbackTest.java b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallbackTest.java new file mode 100644 index 0000000..fe84d4a --- /dev/null +++ b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallbackTest.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.asterix.test.ioopcallbacks; + +import org.apache.asterix.common.ioopcallbacks.LSMBTreeWithBuddyIOOperationCallback; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent; +import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType; +import org.junit.Assert; +import org.mockito.Mockito; + +import junit.framework.TestCase; + +public class LSMBTreeWithBuddyIOOperationCallbackTest extends TestCase { + + public void testNormalSequence() { + try { + LSMBTreeWithBuddyIOOperationCallback callback = new LSMBTreeWithBuddyIOOperationCallback(); + callback.setNumOfMutableComponents(2); + + //request to flush first component + callback.updateLastLSN(1); + callback.beforeOperation(LSMOperationType.FLUSH); + + //request to flush second component + callback.updateLastLSN(2); + callback.beforeOperation(LSMOperationType.FLUSH); + + Assert.assertEquals(1, callback.getComponentLSN(null)); + callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class)); + + Assert.assertEquals(2, callback.getComponentLSN(null)); + callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class)); + } catch (Exception e) { + Assert.fail(); + } + } + + public void testOverWrittenLSN() { + try { + LSMBTreeWithBuddyIOOperationCallback callback = new LSMBTreeWithBuddyIOOperationCallback(); + callback.setNumOfMutableComponents(2); + + //request to flush first component + callback.updateLastLSN(1); + callback.beforeOperation(LSMOperationType.FLUSH); + + //request to flush second component + callback.updateLastLSN(2); + callback.beforeOperation(LSMOperationType.FLUSH); + + //request to flush first component again + //this call should fail + callback.updateLastLSN(3); + //there is no corresponding beforeOperation, since the first component is being flush + //the scheduleFlush request would fail this time + + Assert.assertEquals(1, callback.getComponentLSN(null)); + callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class)); + + Assert.assertEquals(2, callback.getComponentLSN(null)); + callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class)); + } catch (Exception e) { + Assert.fail(); + } + } + +} diff --git a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMInvertedIndexIOOperationCallbackTest.java b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMInvertedIndexIOOperationCallbackTest.java new file mode 100644 index 0000000..1f99db3 --- /dev/null +++ b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMInvertedIndexIOOperationCallbackTest.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.asterix.test.ioopcallbacks; + +import org.apache.asterix.common.ioopcallbacks.LSMInvertedIndexIOOperationCallback; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent; +import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType; +import org.junit.Assert; +import org.mockito.Mockito; + +import junit.framework.TestCase; + +public class LSMInvertedIndexIOOperationCallbackTest extends TestCase { + + public void testNormalSequence() { + try { + LSMInvertedIndexIOOperationCallback callback = new LSMInvertedIndexIOOperationCallback(); + callback.setNumOfMutableComponents(2); + + //request to flush first component + callback.updateLastLSN(1); + callback.beforeOperation(LSMOperationType.FLUSH); + + //request to flush second component + callback.updateLastLSN(2); + callback.beforeOperation(LSMOperationType.FLUSH); + + Assert.assertEquals(1, callback.getComponentLSN(null)); + callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class)); + + Assert.assertEquals(2, callback.getComponentLSN(null)); + callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class)); + } catch (Exception e) { + Assert.fail(); + } + } + + public void testOverWrittenLSN() { + try { + LSMInvertedIndexIOOperationCallback callback = new LSMInvertedIndexIOOperationCallback(); + callback.setNumOfMutableComponents(2); + + //request to flush first component + callback.updateLastLSN(1); + callback.beforeOperation(LSMOperationType.FLUSH); + + //request to flush second component + callback.updateLastLSN(2); + callback.beforeOperation(LSMOperationType.FLUSH); + + //request to flush first component again + //this call should fail + callback.updateLastLSN(3); + //there is no corresponding beforeOperation, since the first component is being flush + //the scheduleFlush request would fail this time + + Assert.assertEquals(1, callback.getComponentLSN(null)); + callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class)); + + Assert.assertEquals(2, callback.getComponentLSN(null)); + callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class)); + } catch (Exception e) { + Assert.fail(); + } + } + +} diff --git a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMRTreeIOOperationCallbackTest.java b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMRTreeIOOperationCallbackTest.java new file mode 100644 index 0000000..38f7ee9 --- /dev/null +++ b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/ioopcallbacks/LSMRTreeIOOperationCallbackTest.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.asterix.test.ioopcallbacks; + +import org.apache.asterix.common.ioopcallbacks.LSMRTreeIOOperationCallback; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent; +import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType; +import org.junit.Assert; +import org.mockito.Mockito; + +import junit.framework.TestCase; + +public class LSMRTreeIOOperationCallbackTest extends TestCase { + + public void testNormalSequence() { + try { + LSMRTreeIOOperationCallback callback = new LSMRTreeIOOperationCallback(); + callback.setNumOfMutableComponents(2); + + //request to flush first component + callback.updateLastLSN(1); + callback.beforeOperation(LSMOperationType.FLUSH); + + //request to flush second component + callback.updateLastLSN(2); + callback.beforeOperation(LSMOperationType.FLUSH); + + Assert.assertEquals(1, callback.getComponentLSN(null)); + callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class)); + + Assert.assertEquals(2, callback.getComponentLSN(null)); + callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class)); + } catch (Exception e) { + Assert.fail(); + } + } + + public void testOverWrittenLSN() { + try { + LSMRTreeIOOperationCallback callback = new LSMRTreeIOOperationCallback(); + callback.setNumOfMutableComponents(2); + + //request to flush first component + callback.updateLastLSN(1); + callback.beforeOperation(LSMOperationType.FLUSH); + + //request to flush second component + callback.updateLastLSN(2); + callback.beforeOperation(LSMOperationType.FLUSH); + + //request to flush first component again + //this call should fail + callback.updateLastLSN(3); + //there is no corresponding beforeOperation, since the first component is being flush + //the scheduleFlush request would fail this time + + Assert.assertEquals(1, callback.getComponentLSN(null)); + callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class)); + + Assert.assertEquals(2, callback.getComponentLSN(null)); + callback.afterFinalize(LSMOperationType.FLUSH, Mockito.mock(ILSMDiskComponent.class)); + } catch (Exception e) { + Assert.fail(); + } + } + +} -- To view, visit https://asterix-gerrit.ics.uci.edu/1771 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: If438e34f8f612458d81f618eea04c0c72c49a9fe Gerrit-PatchSet: 3 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Luo Chen <[email protected]> Gerrit-Reviewer: Ian Maxon <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Luo Chen <[email protected]> Gerrit-Reviewer: Murtadha Hubail <[email protected]> Gerrit-Reviewer: Yingyi Bu <[email protected]> Gerrit-Reviewer: abdullah alamoudi <[email protected]>
