This is an automated email from the ASF dual-hosted git repository.
vinoth pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 04fc86b Turn on embedded server for all client tests
04fc86b is described below
commit 04fc86b43d0afc39b76a7d85e82e8b94dee42923
Author: Balaji Varadarajan <[email protected]>
AuthorDate: Wed Jun 12 16:50:13 2019 -0700
Turn on embedded server for all client tests
---
.../java/com/uber/hoodie/TestAsyncCompaction.java | 11 +++--
...tAsyncCompactionWithEmbeddedTimelineServer.java | 56 ---------------------
.../TestCleanerWithEmbeddedTimelineServer.java | 47 ------------------
.../java/com/uber/hoodie/TestHoodieClientBase.java | 14 +++++-
.../TestHoodieClientOnCopyOnWriteStorage.java | 4 --
.../TestHoodieReadClientWithEmbeddedServer.java | 47 ------------------
.../TestHoodieWriteClientWithEmbeddedServer.java | 57 ----------------------
.../src/test/java/com/uber/hoodie/TestMultiFS.java | 4 +-
.../uber/hoodie/TestMultiFSWithEmbeddedServer.java | 38 ---------------
9 files changed, 21 insertions(+), 257 deletions(-)
diff --git
a/hoodie-client/src/test/java/com/uber/hoodie/TestAsyncCompaction.java
b/hoodie-client/src/test/java/com/uber/hoodie/TestAsyncCompaction.java
index 038e1ac..a86edc4 100644
--- a/hoodie-client/src/test/java/com/uber/hoodie/TestAsyncCompaction.java
+++ b/hoodie-client/src/test/java/com/uber/hoodie/TestAsyncCompaction.java
@@ -37,6 +37,8 @@ import com.uber.hoodie.common.table.HoodieTableMetaClient;
import com.uber.hoodie.common.table.HoodieTimeline;
import com.uber.hoodie.common.table.timeline.HoodieInstant;
import com.uber.hoodie.common.table.timeline.HoodieInstant.State;
+import com.uber.hoodie.common.table.view.FileSystemViewStorageConfig;
+import com.uber.hoodie.common.table.view.FileSystemViewStorageType;
import com.uber.hoodie.common.table.view.HoodieTableFileSystemView;
import com.uber.hoodie.common.util.AvroUtils;
import com.uber.hoodie.common.util.CompactionUtils;
@@ -75,7 +77,10 @@ public class TestAsyncCompaction extends
TestHoodieClientBase {
.withMaxNumDeltaCommitsBeforeCompaction(1).build())
.withStorageConfig(HoodieStorageConfig.newBuilder().limitFileSize(1024
* 1024 * 1024).build())
.forTable("test-trip-table")
-
.withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.BLOOM).build());
+
.withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.BLOOM).build())
+ .withEmbeddedTimelineServerEnabled(true).withFileSystemViewConfig(
+
FileSystemViewStorageConfig.newBuilder().withStorageType(FileSystemViewStorageType.EMBEDDED_KV_STORE)
+ .build());
}
@Override
@@ -572,8 +577,4 @@ public class TestAsyncCompaction extends
TestHoodieClientBase {
protected HoodieTableType getTableType() {
return HoodieTableType.MERGE_ON_READ;
}
-
- protected HoodieTable getHoodieTable(HoodieTableMetaClient metaClient,
HoodieWriteConfig config) {
- return HoodieTable.getHoodieTable(metaClient, config, jsc);
- }
}
diff --git
a/hoodie-client/src/test/java/com/uber/hoodie/TestAsyncCompactionWithEmbeddedTimelineServer.java
b/hoodie-client/src/test/java/com/uber/hoodie/TestAsyncCompactionWithEmbeddedTimelineServer.java
deleted file mode 100644
index fe5f195..0000000
---
a/hoodie-client/src/test/java/com/uber/hoodie/TestAsyncCompactionWithEmbeddedTimelineServer.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.uber.hoodie;
-
-import com.uber.hoodie.common.table.HoodieTableMetaClient;
-import com.uber.hoodie.common.table.SyncableFileSystemView;
-import com.uber.hoodie.common.table.view.FileSystemViewStorageConfig;
-import com.uber.hoodie.common.table.view.FileSystemViewStorageType;
-import com.uber.hoodie.config.HoodieWriteConfig;
-import com.uber.hoodie.exception.HoodieException;
-import com.uber.hoodie.table.HoodieTable;
-import java.io.IOException;
-import org.junit.After;
-
-public class TestAsyncCompactionWithEmbeddedTimelineServer extends
TestAsyncCompaction {
-
- @Override
- protected HoodieWriteConfig.Builder getConfigBuilder(Boolean autoCommit) {
- HoodieWriteConfig.Builder builder = super.getConfigBuilder(autoCommit);
- try {
- return
builder.withEmbeddedTimelineServerEnabled(true).withFileSystemViewConfig(
-
FileSystemViewStorageConfig.newBuilder().withStorageType(FileSystemViewStorageType.EMBEDDED_KV_STORE)
- .build());
- } catch (Exception e) {
- throw new HoodieException(e);
- }
- }
-
- @Override
- protected HoodieTable getHoodieTable(HoodieTableMetaClient metaClient,
HoodieWriteConfig config) {
- HoodieTable table = HoodieTable.getHoodieTable(metaClient, config, jsc);
- ((SyncableFileSystemView) (table.getRTFileSystemView())).reset();
- return table;
- }
-
- @After
- public void tearDown() throws IOException {
- super.tearDown();
- }
-}
diff --git
a/hoodie-client/src/test/java/com/uber/hoodie/TestCleanerWithEmbeddedTimelineServer.java
b/hoodie-client/src/test/java/com/uber/hoodie/TestCleanerWithEmbeddedTimelineServer.java
deleted file mode 100644
index 3578544..0000000
---
a/hoodie-client/src/test/java/com/uber/hoodie/TestCleanerWithEmbeddedTimelineServer.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.uber.hoodie;
-
-import com.uber.hoodie.common.table.view.FileSystemViewStorageConfig;
-import com.uber.hoodie.common.table.view.FileSystemViewStorageType;
-import com.uber.hoodie.config.HoodieWriteConfig;
-import com.uber.hoodie.exception.HoodieException;
-import java.io.IOException;
-import org.junit.After;
-
-public class TestCleanerWithEmbeddedTimelineServer extends TestCleaner {
-
- @Override
- HoodieWriteConfig.Builder getConfigBuilder() {
- HoodieWriteConfig.Builder builder = super.getConfigBuilder();
- try {
- builder.withEmbeddedTimelineServerEnabled(true).withFileSystemViewConfig(
-
FileSystemViewStorageConfig.newBuilder().withStorageType(FileSystemViewStorageType.EMBEDDED_KV_STORE)
- .build());
- } catch (Exception e) {
- throw new HoodieException(e);
- }
- return builder;
- }
-
- @After
- public void tearDown() throws IOException {
- super.tearDown();
- }
-}
diff --git
a/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClientBase.java
b/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClientBase.java
index ae2e57c..850f45f 100644
--- a/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClientBase.java
+++ b/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClientBase.java
@@ -33,7 +33,10 @@ import com.uber.hoodie.common.model.HoodieTableType;
import com.uber.hoodie.common.model.HoodieTestUtils;
import com.uber.hoodie.common.table.HoodieTableMetaClient;
import com.uber.hoodie.common.table.HoodieTimeline;
+import com.uber.hoodie.common.table.SyncableFileSystemView;
import com.uber.hoodie.common.table.timeline.HoodieActiveTimeline;
+import com.uber.hoodie.common.table.view.FileSystemViewStorageConfig;
+import com.uber.hoodie.common.table.view.FileSystemViewStorageType;
import com.uber.hoodie.common.util.FSUtils;
import com.uber.hoodie.config.HoodieCompactionConfig;
import com.uber.hoodie.config.HoodieIndexConfig;
@@ -176,7 +179,16 @@ public class TestHoodieClientBase implements Serializable {
.withCompactionConfig(HoodieCompactionConfig.newBuilder().compactionSmallFileSize(1024
* 1024).build())
.withStorageConfig(HoodieStorageConfig.newBuilder().limitFileSize(1024
* 1024).build())
.forTable("test-trip-table")
-
.withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.BLOOM).build());
+
.withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.BLOOM).build())
+ .withEmbeddedTimelineServerEnabled(true).withFileSystemViewConfig(
+
FileSystemViewStorageConfig.newBuilder().withStorageType(FileSystemViewStorageType.EMBEDDED_KV_STORE)
+ .build());
+ }
+
+ protected HoodieTable getHoodieTable(HoodieTableMetaClient metaClient,
HoodieWriteConfig config) {
+ HoodieTable table = HoodieTable.getHoodieTable(metaClient, config, jsc);
+ ((SyncableFileSystemView) (table.getRTFileSystemView())).reset();
+ return table;
}
/**
diff --git
a/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClientOnCopyOnWriteStorage.java
b/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClientOnCopyOnWriteStorage.java
index 0c2fefb..6bcfcd9 100644
---
a/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClientOnCopyOnWriteStorage.java
+++
b/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClientOnCopyOnWriteStorage.java
@@ -446,10 +446,6 @@ public class TestHoodieClientOnCopyOnWriteStorage extends
TestHoodieClientBase {
assertEquals("Total inserts in commit3 must add up", keys3.size(),
numTotalInsertsInCommit3);
}
- protected HoodieTable getHoodieTable(HoodieTableMetaClient metaClient,
HoodieWriteConfig config) {
- return HoodieTable.getHoodieTable(metaClient, config, jsc);
- }
-
/**
* Test scenario of new file-group getting added during insert()
*/
diff --git
a/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieReadClientWithEmbeddedServer.java
b/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieReadClientWithEmbeddedServer.java
deleted file mode 100644
index a7936cd..0000000
---
a/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieReadClientWithEmbeddedServer.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.uber.hoodie;
-
-import com.uber.hoodie.common.table.view.FileSystemViewStorageConfig;
-import com.uber.hoodie.common.table.view.FileSystemViewStorageType;
-import com.uber.hoodie.config.HoodieWriteConfig;
-import com.uber.hoodie.exception.HoodieException;
-import java.io.IOException;
-import org.junit.After;
-
-public class TestHoodieReadClientWithEmbeddedServer extends
TestHoodieReadClient {
-
- @Override
- HoodieWriteConfig.Builder getConfigBuilder() {
- HoodieWriteConfig.Builder builder = super.getConfigBuilder();
- try {
- builder.withEmbeddedTimelineServerEnabled(true).withFileSystemViewConfig(
-
FileSystemViewStorageConfig.newBuilder().withStorageType(FileSystemViewStorageType.EMBEDDED_KV_STORE)
- .build());
- } catch (Exception e) {
- throw new HoodieException(e);
- }
- return builder;
- }
-
- @After
- public void tearDown() throws IOException {
- super.tearDown();
- }
-}
diff --git
a/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieWriteClientWithEmbeddedServer.java
b/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieWriteClientWithEmbeddedServer.java
deleted file mode 100644
index 1789413..0000000
---
a/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieWriteClientWithEmbeddedServer.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.uber.hoodie;
-
-import com.uber.hoodie.common.table.HoodieTableMetaClient;
-import com.uber.hoodie.common.table.SyncableFileSystemView;
-import com.uber.hoodie.common.table.view.FileSystemViewStorageConfig;
-import com.uber.hoodie.common.table.view.FileSystemViewStorageType;
-import com.uber.hoodie.config.HoodieWriteConfig;
-import com.uber.hoodie.exception.HoodieException;
-import com.uber.hoodie.table.HoodieTable;
-import java.io.IOException;
-import org.junit.After;
-
-public class TestHoodieWriteClientWithEmbeddedServer extends
TestHoodieClientOnCopyOnWriteStorage {
-
- @Override
- HoodieWriteConfig.Builder getConfigBuilder() {
- HoodieWriteConfig.Builder builder = super.getConfigBuilder();
- try {
- builder.withEmbeddedTimelineServerEnabled(true).withFileSystemViewConfig(
-
FileSystemViewStorageConfig.newBuilder().withStorageType(FileSystemViewStorageType.EMBEDDED_KV_STORE)
- .build());
- } catch (Exception e) {
- throw new HoodieException(e);
- }
- return builder;
- }
-
- @Override
- protected HoodieTable getHoodieTable(HoodieTableMetaClient metaClient,
HoodieWriteConfig config) {
- HoodieTable table = HoodieTable.getHoodieTable(metaClient, config, jsc);
- ((SyncableFileSystemView) (table.getRTFileSystemView())).reset();
- return table;
- }
-
- @After
- public void tearDown() throws IOException {
- super.tearDown();
- }
-}
diff --git a/hoodie-client/src/test/java/com/uber/hoodie/TestMultiFS.java
b/hoodie-client/src/test/java/com/uber/hoodie/TestMultiFS.java
index 2388c6a..2ae6c93 100644
--- a/hoodie-client/src/test/java/com/uber/hoodie/TestMultiFS.java
+++ b/hoodie-client/src/test/java/com/uber/hoodie/TestMultiFS.java
@@ -99,10 +99,10 @@ public class TestMultiFS implements Serializable {
}
protected HoodieWriteConfig getHoodieWriteConfig(String basePath) {
- return HoodieWriteConfig.newBuilder().withPath(basePath)
+ return
HoodieWriteConfig.newBuilder().withPath(basePath).withEmbeddedTimelineServerEnabled(true)
.withSchema(HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA).withParallelism(2, 2)
.forTable(tableName).withIndexConfig(
-
HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.BLOOM).build()).build();
+
HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.BLOOM).build()).build();
}
@Test
diff --git
a/hoodie-client/src/test/java/com/uber/hoodie/TestMultiFSWithEmbeddedServer.java
b/hoodie-client/src/test/java/com/uber/hoodie/TestMultiFSWithEmbeddedServer.java
deleted file mode 100644
index e096fca..0000000
---
a/hoodie-client/src/test/java/com/uber/hoodie/TestMultiFSWithEmbeddedServer.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.uber.hoodie;
-
-import com.uber.hoodie.common.HoodieTestDataGenerator;
-import com.uber.hoodie.config.HoodieIndexConfig;
-import com.uber.hoodie.config.HoodieWriteConfig;
-import com.uber.hoodie.index.HoodieIndex;
-
-/**
- * Tests MultiFS with embedded timeline server enabled
- */
-public class TestMultiFSWithEmbeddedServer extends TestMultiFS {
-
- @Override
- protected HoodieWriteConfig getHoodieWriteConfig(String basePath) {
- return
HoodieWriteConfig.newBuilder().withPath(basePath).withEmbeddedTimelineServerEnabled(true)
-
.withSchema(HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA).withParallelism(2, 2)
- .forTable(tableName).withIndexConfig(
-
HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.BLOOM).build()).build();
- }
-}
