prasannarajaperumal commented on code in PR #5064:
URL: https://github.com/apache/hudi/pull/5064#discussion_r963422812
##########
hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetaServerConfig.java:
##########
@@ -28,65 +28,65 @@
@ConfigClassProperty(name = "Metastore Configs",
groupName = ConfigGroups.Names.WRITE_CLIENT,
description = "Configurations used by the Hudi Metastore.")
-public class HoodieMetastoreConfig extends HoodieConfig {
+public class HoodieMetaServerConfig extends HoodieConfig {
- public static final String METASTORE_PREFIX = "hoodie.metastore";
+ public static final String META_SERVER_PREFIX = "hoodie.metaserver";
- public static final ConfigProperty<Boolean> METASTORE_ENABLE = ConfigProperty
- .key(METASTORE_PREFIX + ".enable")
+ public static final ConfigProperty<Boolean> META_SERVER_ENABLE =
ConfigProperty
+ .key(META_SERVER_PREFIX + ".enable")
.defaultValue(false)
.withDocumentation("Use metastore server to store hoodie table
metadata");
- public static final ConfigProperty<String> METASTORE_URLS = ConfigProperty
- .key(METASTORE_PREFIX + ".uris")
+ public static final ConfigProperty<String> META_SERVER_URLS = ConfigProperty
+ .key(META_SERVER_PREFIX + ".uris")
.defaultValue("thrift://localhost:9090")
.withDocumentation("Metastore server uris");
- public static final ConfigProperty<Integer> METASTORE_CONNECTION_RETRIES =
ConfigProperty
- .key(METASTORE_PREFIX + ".connect.retries")
+ public static final ConfigProperty<Integer> META_SERVER_CONNECTION_RETRIES =
ConfigProperty
+ .key(META_SERVER_PREFIX + ".connect.retries")
.defaultValue(3)
.withDocumentation("Number of retries while opening a connection to
metastore");
- public static final ConfigProperty<Integer> METASTORE_CONNECTION_RETRY_DELAY
= ConfigProperty
- .key(METASTORE_PREFIX + ".connect.retry.delay")
+ public static final ConfigProperty<Integer>
META_SERVER_CONNECTION_RETRY_DELAY = ConfigProperty
+ .key(META_SERVER_PREFIX + ".connect.retry.delay")
.defaultValue(1)
.withDocumentation("Number of seconds for the client to wait between
consecutive connection attempts");
- public static HoodieMetastoreConfig.Builder newBuilder() {
- return new HoodieMetastoreConfig.Builder();
+ public static HoodieMetaServerConfig.Builder newBuilder() {
+ return new HoodieMetaServerConfig.Builder();
}
- public boolean enableMetastore() {
- return getBoolean(METASTORE_ENABLE);
+ public boolean enableMetaServer() {
Review Comment:
isMetaServerEnabled? enableMetaServer sounds like a method to enable
metaserver
##########
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java:
##########
@@ -2124,7 +2124,7 @@ public HoodieStorageLayout.LayoutType getLayoutType() {
* Metastore configs.
*/
public boolean isMetastoreEnabled() {
Review Comment:
isMetaServerEnabled
##########
hudi-metaserver/src/main/java/org/apache/hudi/common/table/HoodieTableMetaServerClient.java:
##########
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.common.table;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hudi.common.config.HoodieMetaServerConfig;
+import org.apache.hudi.common.fs.ConsistencyGuardConfig;
+import org.apache.hudi.common.fs.FileSystemRetryConfig;
+import org.apache.hudi.common.model.HoodieTableType;
+import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.table.timeline.HoodieMetaServerBasedTimeline;
+import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.exception.HoodieException;
+import org.apache.hudi.metaserver.client.HoodieMetaServerClient;
+import org.apache.hudi.metaserver.client.HoodieMetaServerClientProxy;
+import org.apache.hudi.metaserver.thrift.NoSuchObjectException;
+import org.apache.hudi.metaserver.thrift.Table;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * HoodieTableMetaClient implementation for hoodie table whose metadata is
stored in the hoodie meta server.
+ */
+public class HoodieTableMetaServerClient extends HoodieTableMetaClient {
Review Comment:
We should now abstract HoodieTableMetaClient into an interface and define
APIs as step 1. We will have multiple implementations of this
FileBasedMetaClient (Default) and RemoteMetaClient. RemoteMetaClient will fetch
this from a remote server.
##########
hudi-metaserver/src/main/java/org/apache/hudi/common/table/timeline/HoodieMetaServerBasedTimeline.java:
##########
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.common.table.timeline;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hudi.common.config.HoodieMetaServerConfig;
+import org.apache.hudi.common.table.HoodieTableConfig;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.ValidationUtils;
+import org.apache.hudi.exception.HoodieException;
+import org.apache.hudi.metaserver.client.HoodieMetaServerClient;
+import org.apache.hudi.metaserver.client.HoodieMetaServerClientProxy;
+
+/**
+ * Active timeline for hoodie table whose metadata is stored in the hoodie
meta server instead of file system.
+ */
+public class HoodieMetaServerBasedTimeline extends HoodieActiveTimeline {
+ private String databaseName;
+ private String tableName;
+
+ private HoodieMetaServerClient metaServerClient;
+
+ public HoodieMetaServerBasedTimeline(HoodieTableMetaClient metaClient,
HoodieMetaServerConfig config) {
Review Comment:
Default constructor in HoodieActiveTimeline should only be used for
Serialization/De-Serialization. HoodieMetaServerBasedTimeline should not extend
HoodieActiveTimeline.
##########
hudi-metaserver/src/main/java/org/apache/hudi/common/table/timeline/HoodieMetaServerBasedTimeline.java:
##########
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.common.table.timeline;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hudi.common.config.HoodieMetaServerConfig;
+import org.apache.hudi.common.table.HoodieTableConfig;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.ValidationUtils;
+import org.apache.hudi.exception.HoodieException;
+import org.apache.hudi.metaserver.client.HoodieMetaServerClient;
+import org.apache.hudi.metaserver.client.HoodieMetaServerClientProxy;
+
+/**
+ * Active timeline for hoodie table whose metadata is stored in the hoodie
meta server instead of file system.
+ */
+public class HoodieMetaServerBasedTimeline extends HoodieActiveTimeline {
Review Comment:
HoodieActiveTimeline should be an Interface again here.
FileBasedActiveTimeline vs RemoteActiveTimeline - similar to above comment.
Also see below on constructor.
##########
hudi-metaserver/src/main/java/org/apache/hudi/common/table/view/HoodieMetaServerFileSystemView.java:
##########
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.common.table.view;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hudi.common.config.HoodieMetaServerConfig;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.timeline.HoodieTimeline;
+import org.apache.hudi.metaserver.client.HoodieMetaServerClient;
+import org.apache.hudi.metaserver.client.HoodieMetaServerClientProxy;
+
+import java.io.IOException;
+
+/**
+ * TableFileSystemView Implementations based on in-memory storage and
+ * is specifically for hoodie table whose metadata is stored in the hoodie
meta server.
+ */
+public class HoodieMetaServerFileSystemView extends HoodieTableFileSystemView {
+ private String databaseName;
+ private String tableName;
+
+ private HoodieMetaServerClient metaServerClient;
+
+ public HoodieMetaServerFileSystemView(HoodieTableMetaClient metaClient,
+ HoodieTimeline visibleActiveTimeline,
HoodieMetaServerConfig config) {
+ super(metaClient, visibleActiveTimeline);
+ this.metaServerClient = HoodieMetaServerClientProxy.getProxy(config);
+ this.databaseName = metaClient.getTableConfig().getDatabaseName();
+ this.tableName = metaClient.getTableConfig().getTableName();
+ }
+
+ protected FileStatus[] listPartition(Path partitionPath) throws IOException {
+ // TODO: support get snapshot from meta server
+ return new FileStatus[0];
Review Comment:
Why check-in something that is not implemented? If this is something to be
done in the future, track in jira and leave this out of this commit.
##########
.gitignore:
##########
@@ -1,6 +1,7 @@
# Directories #
/build/
target/
+hudi-metastore/src/main/thrift/gen-java/
Review Comment:
Do not generate sources into src/main/java. Generated sources in maven
should go into target/generated-sources/
We should not need a exclude in gitignore other than target/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
