This is an automated email from the ASF dual-hosted git repository.
pingtimeout pushed a commit to branch benchmarks-ppc
in repository https://gitbox.apache.org/repos/asf/polaris-tools.git
commit b6cf1dcd94835af39a4041ada1e4aac8cf60afb5
Author: Pierre Laporte <pie...@pingtimeout.fr>
AuthorDate: Thu Apr 3 13:29:18 2025 +0200
Update authentication to match that of Dremio+PPC
---
benchmarks/README.md | 20 +++++++---------
.../src/gatling/resources/benchmark-defaults.conf | 16 ++++++-------
.../benchmarks/actions/AuthenticationActions.scala | 26 ++++++++++----------
.../benchmarks/parameters/BenchmarkConfig.scala | 6 ++---
.../parameters/ConnectionParameters.scala | 28 +++++++++++++++-------
.../benchmarks/simulations/CreateCommits.scala | 16 ++++++++-----
.../benchmarks/simulations/CreateTreeDataset.scala | 20 +++++++++-------
.../benchmarks/simulations/ReadTreeDataset.scala | 20 +++++++++-------
.../simulations/ReadUpdateTreeDataset.scala | 14 +++++++----
9 files changed, 95 insertions(+), 71 deletions(-)
diff --git a/benchmarks/README.md b/benchmarks/README.md
index 4622bc1..fa6a27c 100644
--- a/benchmarks/README.md
+++ b/benchmarks/README.md
@@ -60,12 +60,12 @@ Connection settings are configured under `http` and `auth`:
```hocon
http {
- base-url = "http://localhost:8181"; # Service URL
+ polaris-base-url = "http://localhost:8181"; # URL to the Polaris server that
powers PPC
+ dremio-base-url = "http://localhost:9047"; # URL to the Dremio server
}
auth {
- client-id = null # Required: OAuth2 client ID
- client-secret = null # Required: OAuth2 client secret
+ dremioPat = null # Required: Personal access token
for Dremio authentication
}
```
@@ -90,12 +90,12 @@ To customize the benchmark settings, create your own
`application.conf` file and
Example `application.conf`:
```hocon
auth {
- client-id = "your-client-id"
- client-secret = "your-client-secret"
+ dremioPat = "your-personal-access-token"
}
http {
- base-url = "http://your-polaris-instance:8181";
+ polaris-base-url = "http://your-polaris-instance:8181";
+ dremio-base-url = "http://your-dremio-instance:9047";
}
workload {
@@ -134,18 +134,16 @@ Reports generated in:
./benchmarks/build/reports/gatling/<simulation-name>/index
### Example Polaris server startup
-For repeated testing and benchmarking purposes it's convenient to have fixed
client-ID + client-secret combinations. **The following example is ONLY for
testing and benchmarking against an airgapped Polaris instance**
+For repeated testing and benchmarking purposes it's convenient to have a
personal access token ready.
```bash
-# Start Polaris with the fixed client-ID/secret admin/admin
-# DO NEVER EVER USE THE FOLLOWING FOR ANY NON-AIRGAPPED POLARIS INSTANCE !!
+# Start Polaris with the default configuration
./gradlew :polaris-quarkus-server:quarkusBuild && java \
- -Dpolaris.bootstrap.credentials=POLARIS,admin,admin \
-Djava.security.manager=allow \
-jar quarkus/server/build/quarkus-app/quarkus-run.jar
```
-With the above you can run the benchmarks using a configuration file with
`client-id = "admin"` and `client-secret = "admin"` - meant only for
convenience in a fully airgapped system.
+After starting the server, generate a personal access token through the Dremio
UI or API and use it in your `application.conf` file with the `dremioPat`
property.
# Test Dataset
diff --git a/benchmarks/src/gatling/resources/benchmark-defaults.conf
b/benchmarks/src/gatling/resources/benchmark-defaults.conf
index dced98c..584a4e0 100644
--- a/benchmarks/src/gatling/resources/benchmark-defaults.conf
+++ b/benchmarks/src/gatling/resources/benchmark-defaults.conf
@@ -19,20 +19,20 @@
# HTTP connection settings
http {
- # Base URL of the Polaris service
+ # URL to the Polaris server that powers PPC
# Default: "http://localhost:8181";
- base-url = "http://localhost:8181";
+ polaris-base-url = "http://localhost:8181";
+
+ # URL to the Dremio server
+ # Default: "http://localhost:9047";
+ dremio-base-url = "http://localhost:9047";
}
# Authentication settings
auth {
- # OAuth2 client ID for authentication
- # Required: Must be provided in configuration
- client-id = null
-
- # OAuth2 client secret for authentication
+ # Personal access token for authentication against the Dremio server
# Required: Must be provided in configuration
- client-secret = null
+ dremioPat = null
}
# Dataset tree structure configuration
diff --git
a/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/actions/AuthenticationActions.scala
b/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/actions/AuthenticationActions.scala
index a65d2b8..ecbdfce 100644
---
a/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/actions/AuthenticationActions.scala
+++
b/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/actions/AuthenticationActions.scala
@@ -51,35 +51,35 @@ case class AuthenticationActions(
/**
* Creates a Gatling Feeder that provides authentication credentials. The
feeder continuously
- * supplies client ID and client secret from the connection parameters for
use in authentication
+ * supplies the personal access token from the connection parameters for use
in authentication
* requests.
*
- * @return An iterator providing client credentials
+ * @return An iterator providing authentication credentials
*/
def feeder(): Feeder[String] = Iterator.continually(
Map(
- "clientId" -> cp.clientId,
- "clientSecret" -> cp.clientSecret
+ "dremioPat" -> cp.dremioPat
)
)
/**
- * Authenticates using client credentials and saves the access token as a
session attribute. The
- * credentials are defined in the [[AuthenticationActions.feeder]]. This
operation performs an
- * OAuth2 client credentials flow, requesting full principal roles, and
stores the received access
- * token in both the Gatling session and the shared AtomicReference.
+ * Authenticates using a personal access token and saves the OAuth token as
a session attribute.
+ * The Dremio PAT is defined in the [[AuthenticationActions.feeder]].
*
* There is no limit to the maximum number of users that can authenticate
concurrently.
*/
val authenticateAndSaveAccessToken: ChainBuilder =
retryOnHttpStatus(maxRetries, retryableHttpCodes, "Authenticate")(
http("Authenticate")
- .post("/api/catalog/v1/oauth/tokens")
+ .post("/oauth/token")
.header("Content-Type", "application/x-www-form-urlencoded")
- .formParam("grant_type", "client_credentials")
- .formParam("client_id", "#{clientId}")
- .formParam("client_secret", "#{clientSecret}")
- .formParam("scope", "PRINCIPAL_ROLE:ALL")
+ .formParam("grant_type",
"urn:ietf:params:oauth:grant-type:token-exchange")
+ .formParam(
+ "subject_token_type",
+ "urn:ietf:params:oauth:token-type:dremio:personal-access-token"
+ )
+ .formParam("subject_token", "#{dremioPat}")
+ .formParam("scope", "dremio.all")
.saveHttpStatusCode()
.check(status.is(200))
.check(jsonPath("$.access_token").saveAs("accessToken"))
diff --git
a/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/parameters/BenchmarkConfig.scala
b/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/parameters/BenchmarkConfig.scala
index 40d841b..fc820ce 100644
---
a/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/parameters/BenchmarkConfig.scala
+++
b/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/parameters/BenchmarkConfig.scala
@@ -33,9 +33,9 @@ object BenchmarkConfig {
val workload: Config = config.getConfig("workload")
val connectionParams = ConnectionParameters(
- auth.getString("client-id"),
- auth.getString("client-secret"),
- http.getString("base-url")
+ auth.getString("dremioPat"),
+ http.getString("polaris-base-url"),
+ http.getString("dremio-base-url")
)
val workloadParams = {
diff --git
a/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/parameters/ConnectionParameters.scala
b/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/parameters/ConnectionParameters.scala
index 67c8cc1..bc61a9e 100644
---
a/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/parameters/ConnectionParameters.scala
+++
b/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/parameters/ConnectionParameters.scala
@@ -22,16 +22,26 @@ package org.apache.polaris.benchmarks.parameters
/**
* Case class to hold the connection parameters for the benchmark.
*
- * @param clientId The client ID for authentication.
- * @param clientSecret The client secret for authentication.
- * @param baseUrl The base URL of the Polaris service.
+ * @param dremioPat The personal access token for authentication against the
Dremio server.
+ * @param polarisBaseUrl The URL to the Polaris server that powers PPC.
+ * @param dremioBaseUrl The URL to the Dremio server.
*/
-case class ConnectionParameters(clientId: String, clientSecret: String,
baseUrl: String) {
- require(clientId != null && clientId.nonEmpty, "Client ID cannot be null or
empty")
- require(clientSecret != null && clientSecret.nonEmpty, "Client secret cannot
be null or empty")
- require(baseUrl != null && baseUrl.nonEmpty, "Base URL cannot be null or
empty")
+case class ConnectionParameters(dremioPat: String, polarisBaseUrl: String,
dremioBaseUrl: String) {
+ require(dremioPat != null && dremioPat.nonEmpty, "Personal access token
cannot be null or empty")
require(
- baseUrl.startsWith("http://";) || baseUrl.startsWith("https://";),
- "Base URL must start with http:// or https://";
+ polarisBaseUrl != null && polarisBaseUrl.nonEmpty,
+ "Polaris base URL cannot be null or empty"
+ )
+ require(
+ polarisBaseUrl.startsWith("http://";) ||
polarisBaseUrl.startsWith("https://";),
+ "Polaris base URL must start with http:// or https://";
+ )
+ require(
+ dremioBaseUrl != null && dremioBaseUrl.nonEmpty,
+ "Dremio base URL cannot be null or empty"
+ )
+ require(
+ dremioBaseUrl.startsWith("http://";) ||
dremioBaseUrl.startsWith("https://";),
+ "Dremio base URL must start with http:// or https://";
)
}
diff --git
a/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/simulations/CreateCommits.scala
b/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/simulations/CreateCommits.scala
index cd8e7c9..2811cba 100644
---
a/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/simulations/CreateCommits.scala
+++
b/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/simulations/CreateCommits.scala
@@ -109,8 +109,12 @@ class CreateCommits extends Simulation {
.feed(viewActions.propertyUpdateFeeder())
.exec(viewActions.updateView)
- private val httpProtocol = http
- .baseUrl(cp.baseUrl)
+ private val dremioHttpProtocol = http
+ .baseUrl(cp.dremioBaseUrl)
+ .acceptHeader("application/json")
+ .contentTypeHeader("application/json")
+ private val polarisHttpProtocol = http
+ .baseUrl(cp.polarisBaseUrl)
.acceptHeader("application/json")
.contentTypeHeader("application/json")
@@ -118,7 +122,7 @@ class CreateCommits extends Simulation {
private val viewCommitsThroughput = wp.createCommits.viewCommitsThroughput
private val durationInMinutes = wp.createCommits.durationInMinutes
setUp(
-
continuouslyRefreshOauthToken.inject(atOnceUsers(1)).protocols(httpProtocol),
+
continuouslyRefreshOauthToken.inject(atOnceUsers(1)).protocols(dremioHttpProtocol),
waitForAuthentication
.inject(atOnceUsers(1))
.andThen(
@@ -126,13 +130,13 @@ class CreateCommits extends Simulation {
.inject(
constantUsersPerSec(tableCommitsThroughput).during(durationInMinutes.minutes)
)
- .protocols(httpProtocol),
+ .protocols(polarisHttpProtocol),
viewUpdateScenario
.inject(
constantUsersPerSec(viewCommitsThroughput).during(durationInMinutes.minutes)
)
- .protocols(httpProtocol)
+ .protocols(polarisHttpProtocol)
)
-
.andThen(stopRefreshingToken.inject(atOnceUsers(1)).protocols(httpProtocol))
+
.andThen(stopRefreshingToken.inject(atOnceUsers(1)).protocols(dremioHttpProtocol))
)
}
diff --git
a/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/simulations/CreateTreeDataset.scala
b/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/simulations/CreateTreeDataset.scala
index 6f39a09..0b41e9e 100644
---
a/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/simulations/CreateTreeDataset.scala
+++
b/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/simulations/CreateTreeDataset.scala
@@ -145,8 +145,12 @@ class CreateTreeDataset extends Simulation {
//
--------------------------------------------------------------------------------
// Build up the HTTP protocol configuration and set up the simulation
//
--------------------------------------------------------------------------------
- private val httpProtocol = http
- .baseUrl(cp.baseUrl)
+ private val dremioHttpProtocol = http
+ .baseUrl(cp.dremioBaseUrl)
+ .acceptHeader("application/json")
+ .contentTypeHeader("application/json")
+ private val polarisHttpProtocol = http
+ .baseUrl(cp.polarisBaseUrl)
.acceptHeader("application/json")
.contentTypeHeader("application/json")
.disableCaching
@@ -156,20 +160,20 @@ class CreateTreeDataset extends Simulation {
private val viewThroughput = wp.createTreeDataset.viewThroughput
setUp(
-
continuouslyRefreshOauthToken.inject(atOnceUsers(1)).protocols(httpProtocol),
+
continuouslyRefreshOauthToken.inject(atOnceUsers(1)).protocols(dremioHttpProtocol),
waitForAuthentication
.inject(atOnceUsers(1))
- .andThen(createCatalogs.inject(atOnceUsers(1)).protocols(httpProtocol))
+
.andThen(createCatalogs.inject(atOnceUsers(1)).protocols(polarisHttpProtocol))
.andThen(
createNamespaces
.inject(
constantUsersPerSec(1).during(1.seconds),
constantUsersPerSec(dp.nsWidth - 1).during(dp.nsDepth.seconds)
)
- .protocols(httpProtocol)
+ .protocols(polarisHttpProtocol)
)
-
.andThen(createTables.inject(atOnceUsers(tableThroughput)).protocols(httpProtocol))
-
.andThen(createViews.inject(atOnceUsers(viewThroughput)).protocols(httpProtocol))
-
.andThen(stopRefreshingToken.inject(atOnceUsers(1)).protocols(httpProtocol))
+
.andThen(createTables.inject(atOnceUsers(tableThroughput)).protocols(polarisHttpProtocol))
+
.andThen(createViews.inject(atOnceUsers(viewThroughput)).protocols(polarisHttpProtocol))
+
.andThen(stopRefreshingToken.inject(atOnceUsers(1)).protocols(dremioHttpProtocol))
)
}
diff --git
a/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/simulations/ReadTreeDataset.scala
b/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/simulations/ReadTreeDataset.scala
index 379f6e3..0f78e7e 100644
---
a/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/simulations/ReadTreeDataset.scala
+++
b/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/simulations/ReadTreeDataset.scala
@@ -148,8 +148,12 @@ class ReadTreeDataset extends Simulation {
//
--------------------------------------------------------------------------------
// Build up the HTTP protocol configuration and set up the simulation
//
--------------------------------------------------------------------------------
- private val httpProtocol = http
- .baseUrl(cp.baseUrl)
+ private val dremioHttpProtocol = http
+ .baseUrl(cp.dremioBaseUrl)
+ .acceptHeader("application/json")
+ .contentTypeHeader("application/json")
+ private val polarisHttpProtocol = http
+ .baseUrl(cp.polarisBaseUrl)
.acceptHeader("application/json")
.contentTypeHeader("application/json")
.disableCaching
@@ -160,13 +164,13 @@ class ReadTreeDataset extends Simulation {
private val viewThroughput = wp.readTreeDataset.viewThroughput
setUp(
-
continuouslyRefreshOauthToken.inject(atOnceUsers(1)).protocols(httpProtocol),
+
continuouslyRefreshOauthToken.inject(atOnceUsers(1)).protocols(dremioHttpProtocol),
waitForAuthentication
.inject(atOnceUsers(1))
- .andThen(verifyCatalogs.inject(atOnceUsers(1)).protocols(httpProtocol))
-
.andThen(verifyNamespaces.inject(atOnceUsers(namespaceThroughput)).protocols(httpProtocol))
-
.andThen(verifyTables.inject(atOnceUsers(tableThroughput)).protocols(httpProtocol))
-
.andThen(verifyViews.inject(atOnceUsers(viewThroughput)).protocols(httpProtocol))
-
.andThen(stopRefreshingToken.inject(atOnceUsers(1)).protocols(httpProtocol))
+
.andThen(verifyCatalogs.inject(atOnceUsers(1)).protocols(polarisHttpProtocol))
+
.andThen(verifyNamespaces.inject(atOnceUsers(namespaceThroughput)).protocols(polarisHttpProtocol))
+
.andThen(verifyTables.inject(atOnceUsers(tableThroughput)).protocols(polarisHttpProtocol))
+
.andThen(verifyViews.inject(atOnceUsers(viewThroughput)).protocols(polarisHttpProtocol))
+
.andThen(stopRefreshingToken.inject(atOnceUsers(1)).protocols(dremioHttpProtocol))
)
}
diff --git
a/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/simulations/ReadUpdateTreeDataset.scala
b/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/simulations/ReadUpdateTreeDataset.scala
index 2304920..c205b0d 100644
---
a/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/simulations/ReadUpdateTreeDataset.scala
+++
b/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/simulations/ReadUpdateTreeDataset.scala
@@ -137,8 +137,12 @@ class ReadUpdateTreeDataset extends Simulation {
//
--------------------------------------------------------------------------------
// Build up the HTTP protocol configuration and set up the simulation
//
--------------------------------------------------------------------------------
- private val httpProtocol = http
- .baseUrl(cp.baseUrl)
+ private val dremioHttpProtocol = http
+ .baseUrl(cp.dremioBaseUrl)
+ .acceptHeader("application/json")
+ .contentTypeHeader("application/json")
+ private val polarisHttpProtocol = http
+ .baseUrl(cp.polarisBaseUrl)
.acceptHeader("application/json")
.contentTypeHeader("application/json")
.disableCaching
@@ -148,7 +152,7 @@ class ReadUpdateTreeDataset extends Simulation {
private val durationInMinutes = wp.readUpdateTreeDataset.durationInMinutes
setUp(
-
continuouslyRefreshOauthToken.inject(atOnceUsers(1)).protocols(httpProtocol),
+
continuouslyRefreshOauthToken.inject(atOnceUsers(1)).protocols(dremioHttpProtocol),
waitForAuthentication
.inject(atOnceUsers(1))
.andThen(
@@ -156,8 +160,8 @@ class ReadUpdateTreeDataset extends Simulation {
.inject(
constantUsersPerSec(throughput).during(durationInMinutes.minutes).randomized
)
- .protocols(httpProtocol)
+ .protocols(polarisHttpProtocol)
)
-
.andThen(stopRefreshingToken.inject(atOnceUsers(1)).protocols(httpProtocol))
+
.andThen(stopRefreshingToken.inject(atOnceUsers(1)).protocols(dremioHttpProtocol))
)
}
