This is an automated email from the ASF dual-hosted git repository.
mchades pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 4d8e0f39d [#4259] feat(core): support custom rest api (#4261)
4d8e0f39d is described below
commit 4d8e0f39db578af589427b9c2e5420f345c9b8da
Author: mchades <[email protected]>
AuthorDate: Wed Jul 31 21:02:57 2024 +0800
[#4259] feat(core): support custom rest api (#4261)
### What changes were proposed in this pull request?
- add static method `fromString` for `Catalog.Type` and `Namespace`
- make `GravitinoEnv` extensional
- make GravitinoServer use passed in `GravitinoEnv`
- support custom rest api
### Why are the changes needed?
Fix: #4259
### Does this PR introduce _any_ user-facing change?
yes, users now can use custom REST API packages
### How was this patch tested?
test locally
---
.../main/java/org/apache/gravitino/Catalog.java | 22 +++++++++++++++-
.../main/java/org/apache/gravitino/Namespace.java | 11 ++++++++
.../org/apache/gravitino/dto/rel/TableDTO.java | 4 +--
.../main/java/org/apache/gravitino/Configs.java | 9 +++++++
.../java/org/apache/gravitino/GravitinoEnv.java | 24 ++++++++++++++++--
.../apache/gravitino/config/ConfigConstants.java | 7 ++++++
.../org/apache/gravitino/lock/TreeLockUtils.java | 2 +-
docs/gravitino-server-config.md | 29 +++++++++++-----------
.../apache/gravitino/server/GravitinoServer.java | 17 ++++++++++---
.../gravitino/server/TestGravitinoServer.java | 3 ++-
10 files changed, 103 insertions(+), 25 deletions(-)
diff --git a/api/src/main/java/org/apache/gravitino/Catalog.java
b/api/src/main/java/org/apache/gravitino/Catalog.java
index bbcc0cb7d..052a04d94 100644
--- a/api/src/main/java/org/apache/gravitino/Catalog.java
+++ b/api/src/main/java/org/apache/gravitino/Catalog.java
@@ -18,6 +18,7 @@
*/
package org.apache.gravitino;
+import java.util.Locale;
import java.util.Map;
import org.apache.gravitino.annotation.Evolving;
import org.apache.gravitino.file.FilesetCatalog;
@@ -45,7 +46,26 @@ public interface Catalog extends Auditable {
MESSAGING,
/** Catalog Type for test only. */
- UNSUPPORTED
+ UNSUPPORTED;
+
+ /**
+ * Convert the string (case-insensitive) to the catalog type.
+ *
+ * @param type The string to convert
+ * @return The catalog type
+ */
+ public static Type fromString(String type) {
+ switch (type.toLowerCase(Locale.ROOT)) {
+ case "relational":
+ return RELATIONAL;
+ case "fileset":
+ return FILESET;
+ case "messaging":
+ return MESSAGING;
+ default:
+ throw new IllegalArgumentException("Unknown catalog type: " + type);
+ }
+ }
}
/** The cloud that the catalog is running on. Used by the catalog property
`cloud.name`. */
diff --git a/api/src/main/java/org/apache/gravitino/Namespace.java
b/api/src/main/java/org/apache/gravitino/Namespace.java
index 341fdc604..bdd48a2ef 100644
--- a/api/src/main/java/org/apache/gravitino/Namespace.java
+++ b/api/src/main/java/org/apache/gravitino/Namespace.java
@@ -65,6 +65,17 @@ public class Namespace {
return new Namespace(levels);
}
+ /**
+ * Create a namespace with the given string with levels separated by dots.
+ *
+ * @param namespace The namespace string
+ * @return A namespace with the given levels
+ */
+ public static Namespace fromString(String namespace) {
+ // todo: escape the dots in the levels if needed
+ return new Namespace(namespace.split("\\."));
+ }
+
private Namespace(String[] levels) {
this.levels = levels;
}
diff --git a/common/src/main/java/org/apache/gravitino/dto/rel/TableDTO.java
b/common/src/main/java/org/apache/gravitino/dto/rel/TableDTO.java
index c893c0b69..9a6843a3f 100644
--- a/common/src/main/java/org/apache/gravitino/dto/rel/TableDTO.java
+++ b/common/src/main/java/org/apache/gravitino/dto/rel/TableDTO.java
@@ -21,6 +21,7 @@ package org.apache.gravitino.dto.rel;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
import java.util.Map;
+import lombok.EqualsAndHashCode;
import org.apache.gravitino.dto.AuditDTO;
import org.apache.gravitino.dto.rel.indexes.IndexDTO;
import org.apache.gravitino.dto.rel.partitioning.Partitioning;
@@ -32,6 +33,7 @@ import
org.apache.gravitino.rel.expressions.transforms.Transform;
import org.apache.gravitino.rel.indexes.Index;
/** Represents a Table DTO (Data Transfer Object). */
+@EqualsAndHashCode
public class TableDTO implements Table {
@JsonProperty("name")
@@ -293,8 +295,6 @@ public class TableDTO implements Table {
*/
public TableDTO build() {
Preconditions.checkArgument(name != null && !name.isEmpty(), "name
cannot be null or empty");
- Preconditions.checkArgument(
- columns != null && columns.length > 0, "columns cannot be null or
empty");
Preconditions.checkArgument(audit != null, "audit cannot be null");
return new TableDTO(
diff --git a/core/src/main/java/org/apache/gravitino/Configs.java
b/core/src/main/java/org/apache/gravitino/Configs.java
index 4cbfebe01..fa65d399a 100644
--- a/core/src/main/java/org/apache/gravitino/Configs.java
+++ b/core/src/main/java/org/apache/gravitino/Configs.java
@@ -20,6 +20,7 @@ package org.apache.gravitino;
import com.google.common.collect.Lists;
import java.io.File;
+import java.util.Collections;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.apache.gravitino.config.ConfigBuilder;
@@ -340,4 +341,12 @@ public class Configs {
.version(ConfigConstants.VERSION_0_5_1)
.intConf()
.createWithDefault(DEFAULT_METRICS_TIME_SLIDING_WINDOW_SECONDS);
+
+ public static final ConfigEntry<List<String>> REST_API_EXTENSION_PACKAGES =
+ new ConfigBuilder("gravitino.server.rest.extensionPackages")
+ .doc("Comma-separated list of REST API packages to expand")
+ .version(ConfigConstants.VERSION_0_6_0)
+ .stringConf()
+ .toSequence()
+ .createWithDefault(Collections.emptyList());
}
diff --git a/core/src/main/java/org/apache/gravitino/GravitinoEnv.java
b/core/src/main/java/org/apache/gravitino/GravitinoEnv.java
index aa267f225..a44aebb5b 100644
--- a/core/src/main/java/org/apache/gravitino/GravitinoEnv.java
+++ b/core/src/main/java/org/apache/gravitino/GravitinoEnv.java
@@ -102,7 +102,7 @@ public class GravitinoEnv {
private TagManager tagManager;
private EventBus eventBus;
- private GravitinoEnv() {}
+ protected GravitinoEnv() {}
private static class InstanceHolder {
private static final GravitinoEnv INSTANCE = new GravitinoEnv();
@@ -226,6 +226,26 @@ public class GravitinoEnv {
return idGenerator;
}
+ /**
+ * Get the CatalogManager associated with the Gravitino environment.
+ *
+ * @return The CatalogManager instance.
+ */
+ public CatalogManager catalogManager() {
+ Preconditions.checkArgument(catalogManager != null, "GravitinoEnv is not
initialized.");
+ return catalogManager;
+ }
+
+ /**
+ * Get the EventBus associated with the Gravitino environment.
+ *
+ * @return The EventBus instance.
+ */
+ public EventBus eventBus() {
+ Preconditions.checkArgument(eventBus != null, "GravitinoEnv is not
initialized.");
+ return eventBus;
+ }
+
/**
* Get the MetricsSystem associated with the Gravitino environment.
*
@@ -235,7 +255,7 @@ public class GravitinoEnv {
return metricsSystem;
}
- public LockManager getLockManager() {
+ public LockManager lockManager() {
return lockManager;
}
diff --git
a/core/src/main/java/org/apache/gravitino/config/ConfigConstants.java
b/core/src/main/java/org/apache/gravitino/config/ConfigConstants.java
index 7060fd363..fd6efc70f 100644
--- a/core/src/main/java/org/apache/gravitino/config/ConfigConstants.java
+++ b/core/src/main/java/org/apache/gravitino/config/ConfigConstants.java
@@ -38,18 +38,25 @@ public final class ConfigConstants {
/** The version number for the 0.1.0 release. */
public static final String VERSION_0_1_0 = "0.1.0";
+
/** The version number for the 0.2.0 release. */
public static final String VERSION_0_2_0 = "0.2.0";
+
/** The version number for the 0.3.0 release. */
public static final String VERSION_0_3_0 = "0.3.0";
+
/** The version number for the 0.4.0 release. */
public static final String VERSION_0_4_0 = "0.4.0";
+
/** The version number for the 0.5.0 release. */
public static final String VERSION_0_5_0 = "0.5.0";
+
/** The version number for the 0.5.1 release. */
public static final String VERSION_0_5_1 = "0.5.1";
+
/** The version number for the 0.5.2 release. */
public static final String VERSION_0_5_2 = "0.5.2";
+
/** The version number for the 0.6.0 release. */
public static final String VERSION_0_6_0 = "0.6.0";
}
diff --git a/core/src/main/java/org/apache/gravitino/lock/TreeLockUtils.java
b/core/src/main/java/org/apache/gravitino/lock/TreeLockUtils.java
index 03814eebf..8f25f183d 100644
--- a/core/src/main/java/org/apache/gravitino/lock/TreeLockUtils.java
+++ b/core/src/main/java/org/apache/gravitino/lock/TreeLockUtils.java
@@ -43,7 +43,7 @@ public class TreeLockUtils {
*/
public static <R, E extends Exception> R doWithTreeLock(
NameIdentifier identifier, LockType lockType, Executable<R, E>
executable) throws E {
- TreeLock lock =
GravitinoEnv.getInstance().getLockManager().createTreeLock(identifier);
+ TreeLock lock =
GravitinoEnv.getInstance().lockManager().createTreeLock(identifier);
try {
lock.lock(lockType);
return executable.execute();
diff --git a/docs/gravitino-server-config.md b/docs/gravitino-server-config.md
index e6633f628..88e473ded 100644
--- a/docs/gravitino-server-config.md
+++ b/docs/gravitino-server-config.md
@@ -17,25 +17,26 @@ Apache Gravitino supports several configurations:
## Apache Gravitino server configurations
You can customize the Gravitino server by editing the configuration file
`gravitino.conf` in the `conf` directory. The default values are sufficient for
most use cases.
-We strongly recommend that you read the following sections to understand the
configuration file so you can change the default values to suit your specific
situation and usage scenario.
+We strongly recommend that you read the following sections to understand the
configuration file, so you can change the default values to suit your specific
situation and usage scenario.
The `gravitino.conf` file lists the configuration items in the following
table. It groups those items into the following categories:
### Apache Gravitino HTTP Server configuration
-| Configuration item | Description
| Default value
| Required | Since version |
-|-------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------|----------|---------------|
-| `gravitino.server.webserver.host` | The host of the
Gravitino server.
| `0.0.0.0`
| No | 0.1.0 |
-| `gravitino.server.webserver.httpPort` | The port on which
the Gravitino server listens for incoming connections.
| `8090`
| No | 0.1.0 |
-| `gravitino.server.webserver.minThreads` | The minimum number
of threads in the thread pool used by the Jetty webserver. `minThreads` is 8 if
the value is less than 8.
| `Math.max(Math.min(Runtime.getRuntime().availableProcessors() * 2, 100),
8)` | No | 0.2.0 |
-| `gravitino.server.webserver.maxThreads` | The maximum number
of threads in the thread pool used by the Jetty webserver. `maxThreads` is 8 if
the value is less than 8, and `maxThreads` must be great or equal to
`minThreads`. | `Math.max(Runtime.getRuntime().availableProcessors() * 4, 400)`
| No | 0.1.0 |
-| `gravitino.server.webserver.threadPoolWorkQueueSize` | The size of the
queue in the thread pool used by the Jetty webserver.
| `100`
| No | 0.1.0 |
-| `gravitino.server.webserver.stopTimeout` | Time in milliseconds
to gracefully shut down the Jetty webserver, for more, please see
`org.eclipse.jetty.server.Server#setStopTimeout`.
| `30000`
| No | 0.2.0 |
-| `gravitino.server.webserver.idleTimeout` | The timeout in
milliseconds of idle connections.
| `30000`
| No | 0.2.0 |
-| `gravitino.server.webserver.requestHeaderSize` | Maximum size of HTTP
requests.
| `131072`
| No | 0.1.0 |
-| `gravitino.server.webserver.responseHeaderSize` | Maximum size of HTTP
responses.
| `131072`
| No | 0.1.0 |
-| `gravitino.server.shutdown.timeout` | Time in milliseconds
to gracefully shut down of the Gravitino webserver.
| `3000`
| No | 0.2.0 |
-| `gravitino.server.webserver.customFilters` | Comma-separated list
of filter class names to apply to the API.
| (none)
| No | 0.4.0 |
+| Configuration item | Description
| Default value
| Required | Since version |
+|------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------|----------|---------------|
+| `gravitino.server.webserver.host` | The host of the
Gravitino server.
| `0.0.0.0`
| No | 0.1.0 |
+| `gravitino.server.webserver.httpPort` | The port on which the
Gravitino server listens for incoming connections.
| `8090`
| No | 0.1.0 |
+| `gravitino.server.webserver.minThreads` | The minimum number of
threads in the thread pool used by the Jetty webserver. `minThreads` is 8 if
the value is less than 8.
| `Math.max(Math.min(Runtime.getRuntime().availableProcessors() * 2, 100),
8)` | No | 0.2.0 |
+| `gravitino.server.webserver.maxThreads` | The maximum number of
threads in the thread pool used by the Jetty webserver. `maxThreads` is 8 if
the value is less than 8, and `maxThreads` must be great or equal to
`minThreads`. | `Math.max(Runtime.getRuntime().availableProcessors() * 4, 400)`
| No | 0.1.0 |
+| `gravitino.server.webserver.threadPoolWorkQueueSize` | The size of the queue
in the thread pool used by the Jetty webserver.
| `100`
| No | 0.1.0 |
+| `gravitino.server.webserver.stopTimeout` | Time in milliseconds
to gracefully shut down the Jetty webserver, for more, please see
`org.eclipse.jetty.server.Server#setStopTimeout`.
| `30000`
| No | 0.2.0 |
+| `gravitino.server.webserver.idleTimeout` | The timeout in
milliseconds of idle connections.
| `30000`
| No | 0.2.0 |
+| `gravitino.server.webserver.requestHeaderSize` | Maximum size of HTTP
requests.
| `131072`
| No | 0.1.0 |
+| `gravitino.server.webserver.responseHeaderSize` | Maximum size of HTTP
responses.
| `131072`
| No | 0.1.0 |
+| `gravitino.server.shutdown.timeout` | Time in milliseconds
to gracefully shut down of the Gravitino webserver.
| `3000`
| No | 0.2.0 |
+| `gravitino.server.webserver.customFilters` | Comma-separated list
of filter class names to apply to the API.
| (none)
| No | 0.4.0 |
+| `gravitino.server.rest.extensionPackages` | Comma-separated list
of REST API packages to expand
| (none)
| No | 0.6.0 |
The filter in the customFilters should be a standard javax servlet filter.
You can also specify filter parameters by setting configuration entries of the
form `gravitino.server.webserver.<class name of filter>.param.<param
name>=<value>`.
diff --git
a/server/src/main/java/org/apache/gravitino/server/GravitinoServer.java
b/server/src/main/java/org/apache/gravitino/server/GravitinoServer.java
index 43e36bab9..3232c293b 100644
--- a/server/src/main/java/org/apache/gravitino/server/GravitinoServer.java
+++ b/server/src/main/java/org/apache/gravitino/server/GravitinoServer.java
@@ -18,7 +18,9 @@
*/
package org.apache.gravitino.server;
+import com.google.common.collect.Lists;
import java.io.File;
+import java.util.List;
import java.util.Properties;
import javax.servlet.Servlet;
import org.apache.gravitino.Configs;
@@ -70,10 +72,10 @@ public class GravitinoServer extends ResourceConfig {
private final GravitinoEnv gravitinoEnv;
- public GravitinoServer(ServerConfig config) {
+ public GravitinoServer(ServerConfig config, GravitinoEnv gravitinoEnv) {
serverConfig = config;
server = new JettyServer();
- gravitinoEnv = GravitinoEnv.getInstance();
+ this.gravitinoEnv = gravitinoEnv;
}
public void initialize() {
@@ -89,8 +91,15 @@ public class GravitinoServer extends ResourceConfig {
initializeRestApi();
}
+ public ServerConfig serverConfig() {
+ return serverConfig;
+ }
+
private void initializeRestApi() {
- packages("org.apache.gravitino.server.web.rest");
+ List<String> restApiPackages =
Lists.newArrayList("org.apache.gravitino.server.web.rest");
+
restApiPackages.addAll(serverConfig.get(Configs.REST_API_EXTENSION_PACKAGES));
+ packages(restApiPackages.toArray(new String[0]));
+
boolean enableAuthorization =
serverConfig.get(Configs.ENABLE_AUTHORIZATION);
register(
new AbstractBinder() {
@@ -150,7 +159,7 @@ public class GravitinoServer extends ResourceConfig {
LOG.info("Starting Gravitino Server");
String confPath = System.getenv("GRAVITINO_TEST") == null ? "" : args[0];
ServerConfig serverConfig = loadConfig(confPath);
- GravitinoServer server = new GravitinoServer(serverConfig);
+ GravitinoServer server = new GravitinoServer(serverConfig,
GravitinoEnv.getInstance());
server.initialize();
try {
diff --git
a/server/src/test/java/org/apache/gravitino/server/TestGravitinoServer.java
b/server/src/test/java/org/apache/gravitino/server/TestGravitinoServer.java
index e124d1c28..c11e48f3b 100644
--- a/server/src/test/java/org/apache/gravitino/server/TestGravitinoServer.java
+++ b/server/src/test/java/org/apache/gravitino/server/TestGravitinoServer.java
@@ -28,6 +28,7 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.UUID;
import org.apache.commons.io.FileUtils;
+import org.apache.gravitino.GravitinoEnv;
import org.apache.gravitino.auxiliary.AuxiliaryServiceManager;
import org.apache.gravitino.rest.RESTUtils;
import org.apache.gravitino.server.web.JettyServerConfig;
@@ -76,7 +77,7 @@ public class TestGravitinoServer {
} catch (Exception e) {
// Ignore
}
- gravitinoServer = new GravitinoServer(spyServerConfig);
+ gravitinoServer = new GravitinoServer(spyServerConfig,
GravitinoEnv.getInstance());
}
@AfterAll
