This is an automated email from the ASF dual-hosted git repository. jackie pushed a commit to branch simplify_integration_tests in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit b5d011272d9a495a186559f17f4eb3bc022b65fe Author: Jackie (Xiaotian) Jiang <[email protected]> AuthorDate: Fri Aug 30 15:51:28 2019 -0700 Merge LLC related integration tests Clean up LLCRealtimeClusterIntegrationTest Merge LLCRealtimeClusterSplitCommitIntegrationTest and LLCRealtimeClusterWithEnabledLeadControllerResourceIntegrationTest into LLCRealtimeClusterIntegrationTest to reduse the runtime of integration tests --- .../pinot/controller/helix/ControllerTest.java | 2 +- .../tests/LLCRealtimeClusterIntegrationTest.java | 93 ++++++++++------------ ...CRealtimeClusterSplitCommitIntegrationTest.java | 44 ---------- ...abledLeadControllerResourceIntegrationTest.java | 31 -------- 4 files changed, 44 insertions(+), 126 deletions(-) diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/ControllerTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/ControllerTest.java index 5f51010..069c7bd 100644 --- a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/ControllerTest.java +++ b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/ControllerTest.java @@ -448,7 +448,7 @@ public abstract class ControllerTest { ConfigAccessor configAccessor = _helixManager.getConfigAccessor(); ResourceConfig resourceConfig = configAccessor.getResourceConfig(getHelixClusterName(), LEAD_CONTROLLER_RESOURCE_NAME); - if (!Boolean.toString(enable).equals(resourceConfig.getSimpleConfig(LEAD_CONTROLLER_RESOURCE_ENABLED_KEY))) { + if (Boolean.parseBoolean(resourceConfig.getSimpleConfig(LEAD_CONTROLLER_RESOURCE_ENABLED_KEY)) != enable) { resourceConfig.putSimpleConfig(LEAD_CONTROLLER_RESOURCE_ENABLED_KEY, Boolean.toString(enable)); configAccessor.setResourceConfig(getHelixClusterName(), LEAD_CONTROLLER_RESOURCE_NAME, resourceConfig); } diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/LLCRealtimeClusterIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/LLCRealtimeClusterIntegrationTest.java index 881b4dd..fc8b496 100644 --- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/LLCRealtimeClusterIntegrationTest.java +++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/LLCRealtimeClusterIntegrationTest.java @@ -19,58 +19,56 @@ package org.apache.pinot.integration.tests; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.base.Function; import java.io.File; import java.io.IOException; import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Random; import org.apache.avro.reflect.Nullable; import org.apache.commons.configuration.Configuration; import org.apache.commons.io.FileUtils; import org.apache.helix.ZNRecord; -import org.apache.pinot.common.config.IndexingConfig; -import org.apache.pinot.common.config.SegmentsValidationAndRetentionConfig; import org.apache.pinot.common.config.TableConfig; -import org.apache.pinot.common.config.TableCustomConfig; import org.apache.pinot.common.config.TableNameBuilder; -import org.apache.pinot.common.config.TenantConfig; import org.apache.pinot.common.utils.CommonConstants; import org.apache.pinot.common.utils.CommonConstants.Helix.TableType; import org.apache.pinot.controller.ControllerConf; import org.apache.pinot.util.TestUtils; -import org.testng.Assert; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + /** * Integration test that extends RealtimeClusterIntegrationTest but uses low-level Kafka consumer. */ public class LLCRealtimeClusterIntegrationTest extends RealtimeClusterIntegrationTest { - - public static final String CONSUMER_DIRECTORY = "/tmp/consumer-test"; - public static final long RANDOM_SEED = System.currentTimeMillis(); - public static final Random RANDOM = new Random(RANDOM_SEED); - - public final boolean _isDirectAlloc = RANDOM.nextBoolean(); - public final boolean _isConsumerDirConfigured = RANDOM.nextBoolean(); - private final long _startTime = System.currentTimeMillis(); - + private static final String CONSUMER_DIRECTORY = "/tmp/consumer-test"; private static final String TEST_UPDATED_INVERTED_INDEX_QUERY = "SELECT COUNT(*) FROM mytable WHERE DivActualElapsedTime = 305"; private static final List<String> UPDATED_INVERTED_INDEX_COLUMNS = Arrays.asList("FlightNum", "Origin", "Quarter", "DivActualElapsedTime"); + private static final long RANDOM_SEED = System.currentTimeMillis(); + private static final Random RANDOM = new Random(RANDOM_SEED); + + private final boolean _isDirectAlloc = RANDOM.nextBoolean(); + private final boolean _isConsumerDirConfigured = RANDOM.nextBoolean(); + private final boolean _enableSplitCommit = RANDOM.nextBoolean(); + private final boolean _enableLeadControllerResource = RANDOM.nextBoolean(); + private final long _startTime = System.currentTimeMillis(); @BeforeClass @Override public void setUp() throws Exception { // TODO Avoid printing to stdout. Instead, we need to add the seed to every assert in this (and super-classes) - System.out.println("========== Using random seed value " + RANDOM_SEED); + System.out.println(String.format( + "Using random seed: %s, isDirectAlloc: %s, isConsumerDirConfigured: %s, enableSplitCommit: %s, enableLeadControllerResource: %s", + RANDOM_SEED, _isDirectAlloc, _isConsumerDirConfigured, _enableSplitCommit, _enableLeadControllerResource)); + // Remove the consumer directory File consumerDirectory = new File(CONSUMER_DIRECTORY); if (consumerDirectory.exists()) { @@ -84,7 +82,9 @@ public class LLCRealtimeClusterIntegrationTest extends RealtimeClusterIntegratio public void startController() { ControllerConf controllerConfig = getDefaultControllerConfiguration(); controllerConfig.setHLCTablesAllowed(false); + controllerConfig.setSplitCommit(_enableSplitCommit); startController(controllerConfig); + enableResourceConfigForLeadControllerResource(_enableLeadControllerResource); } @Override @@ -105,24 +105,26 @@ public class LLCRealtimeClusterIntegrationTest extends RealtimeClusterIntegratio if (_isConsumerDirConfigured) { configuration.setProperty(CommonConstants.Server.CONFIG_OF_CONSUMER_DIR, CONSUMER_DIRECTORY); } + if (_enableSplitCommit) { + configuration.setProperty(CommonConstants.Server.CONFIG_OF_ENABLE_SPLIT_COMMIT, true); + configuration.setProperty(CommonConstants.Server.CONFIG_OF_ENABLE_COMMIT_END_WITH_METADATA, true); + } } @Test public void testConsumerDirectoryExists() { File consumerDirectory = new File(CONSUMER_DIRECTORY, "mytable_REALTIME"); - Assert.assertEquals(consumerDirectory.exists(), _isConsumerDirConfigured, + assertEquals(consumerDirectory.exists(), _isConsumerDirConfigured, "The off heap consumer directory does not exist"); } @Test - public void testSegmentFlushSize() - throws Exception { - + public void testSegmentFlushSize() { String zkSegmentsPath = "/SEGMENTS/" + TableNameBuilder.REALTIME.tableNameWithType(getTableName()); List<String> segmentNames = _propertyStore.getChildNames(zkSegmentsPath, 0); for (String segmentName : segmentNames) { ZNRecord znRecord = _propertyStore.get(zkSegmentsPath + "/" + segmentName, null, 0); - Assert.assertEquals(znRecord.getSimpleField(CommonConstants.Segment.FLUSH_THRESHOLD_SIZE), + assertEquals(znRecord.getSimpleField(CommonConstants.Segment.FLUSH_THRESHOLD_SIZE), Integer.toString(getRealtimeSegmentFlushSize() / getNumKafkaPartitions()), "Segment: " + segmentName + " does not have the expected flush size"); } @@ -131,45 +133,36 @@ public class LLCRealtimeClusterIntegrationTest extends RealtimeClusterIntegratio @Test public void testInvertedIndexTriggering() throws Exception { - - final long numTotalDocs = getCountStarResult(); + long numTotalDocs = getCountStarResult(); JsonNode queryResponse = postQuery(TEST_UPDATED_INVERTED_INDEX_QUERY); - Assert.assertEquals(queryResponse.get("totalDocs").asLong(), numTotalDocs); + assertEquals(queryResponse.get("totalDocs").asLong(), numTotalDocs); // TODO: investigate why assert for a specific value fails intermittently - Assert.assertNotSame(queryResponse.get("numEntriesScannedInFilter").asLong(), 0); + assertEquals(queryResponse.get("numEntriesScannedInFilter").asLong(), 0L); updateRealtimeTableConfig(getTableName(), UPDATED_INVERTED_INDEX_COLUMNS, null); - sendPostRequest(_controllerRequestURLBuilder.forTableReload(getTableName(), "realtime"), null); - TestUtils.waitForCondition(new Function<Void, Boolean>() { - @Override - public Boolean apply(@javax.annotation.Nullable Void aVoid) { - try { - JsonNode queryResponse = postQuery(TEST_UPDATED_INVERTED_INDEX_QUERY); - // Total docs should not change during reload - Assert.assertEquals(queryResponse.get("totalDocs").asLong(), numTotalDocs); - Assert.assertEquals(queryResponse.get("numConsumingSegmentsQueried").asLong(), 2); - Assert.assertTrue(queryResponse.get("minConsumingFreshnessTimeMs").asLong() > _startTime); - Assert.assertTrue(queryResponse.get("minConsumingFreshnessTimeMs").asLong() < System.currentTimeMillis()); - return queryResponse.get("numEntriesScannedInFilter").asLong() == 0; - } catch (Exception e) { - throw new RuntimeException(e); - } + TestUtils.waitForCondition(aVoid -> { + try { + JsonNode queryResponse1 = postQuery(TEST_UPDATED_INVERTED_INDEX_QUERY); + // Total docs should not change during reload + assertEquals(queryResponse1.get("totalDocs").asLong(), numTotalDocs); + assertEquals(queryResponse1.get("numConsumingSegmentsQueried").asLong(), 2); + assertTrue(queryResponse1.get("minConsumingFreshnessTimeMs").asLong() > _startTime); + assertTrue(queryResponse1.get("minConsumingFreshnessTimeMs").asLong() < System.currentTimeMillis()); + return queryResponse1.get("numEntriesScannedInFilter").asLong() == 0; + } catch (Exception e) { + throw new RuntimeException(e); } }, 600_000L, "Failed to generate inverted index"); } - @Test - public void testAddHLCTableShouldFail() { + @Test(expectedExceptions = IOException.class) + public void testAddHLCTableShouldFail() + throws IOException { TableConfig tableConfig = new TableConfig.Builder(TableType.REALTIME).setTableName("testTable") .setStreamConfigs(Collections.singletonMap("stream.kafka.consumer.type", "HIGHLEVEL")).build(); - try { - sendPostRequest(_controllerRequestURLBuilder.forTableCreate(), tableConfig.toJsonConfigString()); - Assert.fail(); - } catch (IOException e) { - // Expected - } + sendPostRequest(_controllerRequestURLBuilder.forTableCreate(), tableConfig.toJsonConfigString()); } } diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/LLCRealtimeClusterSplitCommitIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/LLCRealtimeClusterSplitCommitIntegrationTest.java deleted file mode 100644 index 454cdb4..0000000 --- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/LLCRealtimeClusterSplitCommitIntegrationTest.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pinot.integration.tests; - -import org.apache.commons.configuration.Configuration; -import org.apache.pinot.common.utils.CommonConstants; -import org.apache.pinot.controller.ControllerConf; - - -/** - * Integration test that extends LLCRealtimeClusterIntegrationTest but with split commit enabled. - */ -public class LLCRealtimeClusterSplitCommitIntegrationTest extends LLCRealtimeClusterIntegrationTest { - @Override - public void startController() { - ControllerConf controllerConfig = getDefaultControllerConfiguration(); - controllerConfig.setSplitCommit(true); - startController(controllerConfig); - } - - @Override - public void startServer() { - Configuration serverConfig = getDefaultServerConfiguration(); - serverConfig.setProperty(CommonConstants.Server.CONFIG_OF_ENABLE_SPLIT_COMMIT, true); - serverConfig.setProperty(CommonConstants.Server.CONFIG_OF_ENABLE_COMMIT_END_WITH_METADATA, true); - startServer(serverConfig); - } -} diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/LLCRealtimeClusterWithEnabledLeadControllerResourceIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/LLCRealtimeClusterWithEnabledLeadControllerResourceIntegrationTest.java deleted file mode 100644 index 681e235..0000000 --- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/LLCRealtimeClusterWithEnabledLeadControllerResourceIntegrationTest.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pinot.integration.tests; - -import org.apache.pinot.controller.ControllerConf; - - -public class LLCRealtimeClusterWithEnabledLeadControllerResourceIntegrationTest extends LLCRealtimeClusterIntegrationTest { - @Override - public void startController() { - ControllerConf controllerConfig = getDefaultControllerConfiguration(); - startController(controllerConfig); - enableResourceConfigForLeadControllerResource(true); - } -} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
