This is an automated email from the ASF dual-hosted git repository.
pingtimeout pushed a commit to branch benchmarks-ppc
in repository https://gitbox.apache.org/repos/asf/polaris-tools.git
commit bbaa9bec1857969ea4764936ba501518af597816
Author: Pierre Laporte <pie...@pingtimeout.fr>
AuthorDate: Thu Apr 3 13:29:36 2025 +0200
Remove catalog actions as they are not available in PPC
---
benchmarks/README.md | 1 -
.../src/gatling/resources/benchmark-defaults.conf | 4 -
.../benchmarks/actions/CatalogActions.scala | 135 ---------------------
.../benchmarks/actions/NamespaceActions.scala | 5 +-
.../polaris/benchmarks/actions/TableActions.scala | 5 +-
.../polaris/benchmarks/actions/ViewActions.scala | 6 +-
.../benchmarks/parameters/BenchmarkConfig.scala | 1 -
.../benchmarks/parameters/DatasetParameters.scala | 3 -
.../benchmarks/simulations/CreateTreeDataset.scala | 15 ---
.../benchmarks/simulations/ReadTreeDataset.scala | 15 ---
.../simulations/ReadUpdateTreeDataset.scala | 1 -
11 files changed, 6 insertions(+), 185 deletions(-)
diff --git a/benchmarks/README.md b/benchmarks/README.md
index ca388fb..66df1cf 100644
--- a/benchmarks/README.md
+++ b/benchmarks/README.md
@@ -38,7 +38,6 @@ These parameters must be consistent across all benchmarks and
are configured und
```hocon
dataset.tree {
- num-catalogs = 1 # Number of catalogs to create
namespace-width = 2 # Width of the namespace tree
namespace-depth = 4 # Depth of the namespace tree
tables-per-namespace = 5 # Tables per namespace
diff --git a/benchmarks/src/gatling/resources/benchmark-defaults.conf
b/benchmarks/src/gatling/resources/benchmark-defaults.conf
index 91c768b..e90d3ef 100644
--- a/benchmarks/src/gatling/resources/benchmark-defaults.conf
+++ b/benchmarks/src/gatling/resources/benchmark-defaults.conf
@@ -37,10 +37,6 @@ auth {
# Dataset tree structure configuration
dataset.tree {
- # Number of catalogs to create. Only the first catalog (C_0) will contain
the test dataset.
- # Default: 1
- num-catalogs = 1
-
# Width of the namespace tree (N). Each namespace will have exactly N
children
# For N > 1: Total namespaces = (N^D - 1)/(N - 1)
# For N = 1: Total namespaces = D
diff --git
a/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/actions/CatalogActions.scala
b/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/actions/CatalogActions.scala
deleted file mode 100644
index 8878b5a..0000000
---
a/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/actions/CatalogActions.scala
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.polaris.benchmarks.actions
-
-import io.gatling.core.Predef._
-import io.gatling.core.feeder.Feeder
-import io.gatling.core.structure.ChainBuilder
-import io.gatling.http.Predef._
-import org.apache.polaris.benchmarks.RetryOnHttpCodes.{
- retryOnHttpStatus,
- HttpRequestBuilderWithStatusSave
-}
-import org.apache.polaris.benchmarks.parameters.DatasetParameters
-import org.slf4j.LoggerFactory
-
-import java.util.concurrent.atomic.AtomicReference
-
-/**
- * Actions for performance testing catalog operations in Apache Iceberg. This
class provides methods
- * to create and fetch catalogs.
- *
- * @param dp Dataset parameters controlling the dataset generation
- * @param accessToken Reference to the authentication token for API requests
- * @param maxRetries Maximum number of retry attempts for failed operations
- * @param retryableHttpCodes HTTP status codes that should trigger a retry
- */
-case class CatalogActions(
- dp: DatasetParameters,
- accessToken: AtomicReference[String],
- maxRetries: Int = 10,
- retryableHttpCodes: Set[Int] = Set(409, 500)
-) {
- private val logger = LoggerFactory.getLogger(getClass)
-
- /**
- * Creates a Gatling Feeder that generates catalog names and their default
storage locations. Each
- * catalog will be named "C_n" where n is a sequential number, and will have
a corresponding
- * storage location under the configured base path.
- *
- * @return An iterator providing catalog names and their storage locations
- */
- def feeder(): Feeder[String] = Iterator
- .from(0)
- .map { i =>
- val catalogName = s"C_$i"
- Map(
- "catalogName" -> catalogName,
- "defaultBaseLocation" -> s"${dp.defaultBaseLocation}/$catalogName"
- )
- }
- .take(dp.numCatalogs)
-
- /**
- * Creates a new Iceberg catalog with FILE storage type. The catalog is
created as an INTERNAL
- * type with a name and a default base location that are defined in the
[[CatalogActions.feeder]].
- * This represents the fundamental operation of establishing a new catalog
in an Iceberg
- * deployment.
- *
- * There is no limit to the number of users that can create catalogs
concurrently.
- */
- val createCatalog: ChainBuilder =
- retryOnHttpStatus(maxRetries, retryableHttpCodes, "Create catalog")(
- http("Create Catalog")
- .post("/api/management/v1/catalogs")
- .header("Authorization", "Bearer #{accessToken}")
- .header("Content-Type", "application/json")
- .body(
- StringBody(
- """{
- | "catalog": {
- | "type": "INTERNAL",
- | "name": "#{catalogName}",
- | "properties": {
- | "default-base-location": "#{defaultBaseLocation}"
- | },
- | "storageConfigInfo": {
- | "storageType": "FILE"
- | }
- | }
- |}""".stripMargin
- )
- )
- .saveHttpStatusCode()
- .check(status.is(201))
- )
-
- /**
- * Retrieves details of a specific Iceberg catalog by name. The catalog name
is defined in the
- * [[CatalogActions.feeder]]. Some basic properties are verified, like the
catalog type, storage
- * settings, and base location.
- *
- * There is no limit to the number of users that can fetch catalogs
concurrently.
- */
- val fetchCatalog: ChainBuilder = exec(
- http("Fetch Catalog")
- .get("/api/management/v1/catalogs/#{catalogName}")
- .header("Authorization", "Bearer #{accessToken}")
- .header("Content-Type", "application/json")
- .check(status.is(200))
- .check(jsonPath("$.type").is("INTERNAL"))
- .check(jsonPath("$.name").is("#{catalogName}"))
-
.check(jsonPath("$.properties.default-base-location").is("#{defaultBaseLocation}"))
- .check(jsonPath("$.storageConfigInfo.storageType").is("FILE"))
-
.check(jsonPath("$.storageConfigInfo.allowedLocations[0]").is("#{defaultBaseLocation}"))
- )
-
- /**
- * Lists all available Iceberg catalogs in the deployment. This operation
does not rely on any
- * feeder data.
- */
- val fetchAllCatalogs: ChainBuilder = exec(
- http("Fetch all Catalogs")
- .get("/api/management/v1/catalogs")
- .header("Authorization", "Bearer #{accessToken}")
- .header("Content-Type", "application/json")
- .check(status.is(200))
- )
-}
diff --git
a/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/actions/NamespaceActions.scala
b/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/actions/NamespaceActions.scala
index 5872618..17b265a 100644
---
a/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/actions/NamespaceActions.scala
+++
b/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/actions/NamespaceActions.scala
@@ -67,7 +67,7 @@ case class NamespaceActions(
.pathToRoot(namespaceId)
.map(ordinal => s"NS_$ordinal")
Map(
- "catalogName" -> "C_0",
+ "catalogName" -> "default",
"namespaceId" -> tableId,
"namespacePath" -> namespacePath,
"namespaceJsonPath" -> Json.toJson(namespacePath).toString(),
@@ -105,10 +105,9 @@ case class NamespaceActions(
*/
def namespaceFetchFeeder(): Feeder[Any] = namespaceCreationFeeder()
.map { row =>
- val catalogName = row("catalogName").asInstanceOf[String]
val namespaceUnixPath =
row("namespacePath").asInstanceOf[Seq[String]].mkString("/")
val location = Map(
- "location" ->
s"${dp.defaultBaseLocation}/$catalogName/$namespaceUnixPath"
+ "location" -> s"${dp.defaultBaseLocation}/$namespaceUnixPath"
)
row ++ Map(
"location" -> location
diff --git
a/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/actions/TableActions.scala
b/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/actions/TableActions.scala
index 67e0ac0..cccd53f 100644
---
a/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/actions/TableActions.scala
+++
b/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/actions/TableActions.scala
@@ -71,7 +71,7 @@ case class TableActions(
.map { j =>
val tableId = positionInLevel * dp.numTablesPerNs + j
Map(
- "catalogName" -> "C_0",
+ "catalogName" -> "default",
"parentNamespacePath" -> parentNamespacePath,
"multipartNamespace" -> parentNamespacePath.mkString("%1F"),
"tableName" -> s"T_$tableId"
@@ -130,7 +130,6 @@ case class TableActions(
*/
def tableFetchFeeder(): Feeder[Any] = tableIdentityFeeder()
.map { row =>
- val catalogName: String = row("catalogName").asInstanceOf[String]
val parentNamespacePath: Seq[String] =
row("parentNamespacePath").asInstanceOf[Seq[String]]
val tableName: String = row("tableName").asInstanceOf[String]
val initialProperties: Map[String, String] = (0 until
dp.numTableProperties)
@@ -138,7 +137,7 @@ case class TableActions(
.toMap
row ++ Map(
"initialProperties" -> initialProperties,
- "location" ->
s"${dp.defaultBaseLocation}/$catalogName/${parentNamespacePath.mkString("/")}/$tableName"
+ "location" ->
s"${dp.defaultBaseLocation}/${parentNamespacePath.mkString("/")}/$tableName"
)
}
diff --git
a/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/actions/ViewActions.scala
b/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/actions/ViewActions.scala
index 99ff175..f0938cd 100644
---
a/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/actions/ViewActions.scala
+++
b/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/actions/ViewActions.scala
@@ -51,7 +51,6 @@ case class ViewActions(
*/
def viewIdentityFeeder(): Feeder[Any] =
dp.nAryTree.lastLevelOrdinals.iterator
.flatMap { namespaceId =>
- val catalogId = 0
val parentNamespacePath: Seq[String] = dp.nAryTree
.pathToRoot(namespaceId)
.map(ordinal => s"NS_$ordinal")
@@ -62,7 +61,7 @@ case class ViewActions(
// Ensure the view ID matches that of the associated table
val viewId = positionInLevel * dp.numTablesPerNs + j
Map(
- "catalogName" -> s"C_$catalogId",
+ "catalogName" -> s"default",
"parentNamespacePath" -> parentNamespacePath,
"multipartNamespace" -> parentNamespacePath.mkString("%1F"),
"viewName" -> s"V_$viewId",
@@ -122,7 +121,6 @@ case class ViewActions(
*/
def viewFetchFeeder(): Feeder[Any] = viewCreationFeeder()
.map { row =>
- val catalogName: String = row("catalogName").asInstanceOf[String]
val parentNamespacePath: Seq[String] =
row("parentNamespacePath").asInstanceOf[Seq[String]]
val viewName: String = row("viewName").asInstanceOf[String]
val initialProperties: Map[String, String] = (0 until
dp.numViewProperties)
@@ -130,7 +128,7 @@ case class ViewActions(
.toMap
row ++ Map(
"initialProperties" -> initialProperties,
- "location" ->
s"${dp.defaultBaseLocation}/$catalogName/${parentNamespacePath.mkString("/")}/$viewName"
+ "location" ->
s"${dp.defaultBaseLocation}/${parentNamespacePath.mkString("/")}/$viewName"
)
}
diff --git
a/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/parameters/BenchmarkConfig.scala
b/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/parameters/BenchmarkConfig.scala
index 97b23b9..e5af748 100644
---
a/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/parameters/BenchmarkConfig.scala
+++
b/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/parameters/BenchmarkConfig.scala
@@ -68,7 +68,6 @@ object BenchmarkConfig {
}
val datasetParams = DatasetParameters(
- dataset.getInt("num-catalogs"),
dataset.getString("default-base-location"),
dataset.getInt("namespace-width"),
dataset.getInt("namespace-depth"),
diff --git
a/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/parameters/DatasetParameters.scala
b/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/parameters/DatasetParameters.scala
index 894cee2..4d1e306 100644
---
a/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/parameters/DatasetParameters.scala
+++
b/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/parameters/DatasetParameters.scala
@@ -24,7 +24,6 @@ import org.apache.polaris.benchmarks.NAryTreeBuilder
/**
* Case class to hold the dataset parameters for the benchmark.
*
- * @param numCatalogs The number of catalogs to create.
* @param defaultBaseLocation The default base location for the datasets.
* @param nsWidth The width of the namespace n-ary tree.
* @param nsDepth The depth of the namespace n-ary tree.
@@ -39,7 +38,6 @@ import org.apache.polaris.benchmarks.NAryTreeBuilder
* @param numViewProperties The number of view properties to create.
*/
case class DatasetParameters(
- numCatalogs: Int,
defaultBaseLocation: String,
nsWidth: Int,
nsDepth: Int,
@@ -59,7 +57,6 @@ case class DatasetParameters(
val numTables: Int = if (numTablesMax <= 0) maxPossibleTables else
numTablesMax
val numViews: Int = if (numViewsMax <= 0) maxPossibleViews else numViewsMax
- require(numCatalogs > 0, "Number of catalogs must be positive")
require(defaultBaseLocation.nonEmpty, "Base location cannot be empty")
require(nsWidth > 0, "Namespace width must be positive")
require(nsDepth > 0, "Namespace depth must be positive")
diff --git
a/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/simulations/CreateTreeDataset.scala
b/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/simulations/CreateTreeDataset.scala
index 0b41e9e..8fbc0e7 100644
---
a/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/simulations/CreateTreeDataset.scala
+++
b/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/simulations/CreateTreeDataset.scala
@@ -56,7 +56,6 @@ class CreateTreeDataset extends Simulation {
private val shouldRefreshToken: AtomicBoolean = new AtomicBoolean(true)
private val authenticationActions = AuthenticationActions(cp, accessToken,
5, Set(500))
- private val catalogActions = CatalogActions(dp, accessToken, 0, Set())
private val namespaceActions = NamespaceActions(dp, wp, accessToken, 5,
Set(500))
private val tableActions = TableActions(dp, wp, accessToken, 5, Set(500))
private val viewActions = ViewActions(dp, wp, accessToken, 5, Set(500))
@@ -93,19 +92,6 @@ class CreateTreeDataset extends Simulation {
session
}
- //
--------------------------------------------------------------------------------
- // Workload: Create catalogs
- //
--------------------------------------------------------------------------------
- val createCatalogs: ScenarioBuilder =
- scenario("Create catalogs using the Polaris Management REST API")
- .exec(authenticationActions.restoreAccessTokenInSession)
- .asLongAs(session =>
- createdCatalogs.getAndIncrement() < dp.numCatalogs &&
session.contains("accessToken")
- )(
- feed(catalogActions.feeder())
- .exec(catalogActions.createCatalog)
- )
-
//
--------------------------------------------------------------------------------
// Workload: Create namespaces
//
--------------------------------------------------------------------------------
@@ -163,7 +149,6 @@ class CreateTreeDataset extends Simulation {
continuouslyRefreshOauthToken.inject(atOnceUsers(1)).protocols(dremioHttpProtocol),
waitForAuthentication
.inject(atOnceUsers(1))
-
.andThen(createCatalogs.inject(atOnceUsers(1)).protocols(polarisHttpProtocol))
.andThen(
createNamespaces
.inject(
diff --git
a/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/simulations/ReadTreeDataset.scala
b/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/simulations/ReadTreeDataset.scala
index 0f78e7e..054255a 100644
---
a/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/simulations/ReadTreeDataset.scala
+++
b/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/simulations/ReadTreeDataset.scala
@@ -54,12 +54,10 @@ class ReadTreeDataset extends Simulation {
private val shouldRefreshToken: AtomicBoolean = new AtomicBoolean(true)
private val authenticationActions = AuthenticationActions(cp, accessToken)
- private val catalogActions = CatalogActions(dp, accessToken)
private val namespaceActions = NamespaceActions(dp, wp, accessToken)
private val tableActions = TableActions(dp, wp, accessToken)
private val viewActions = ViewActions(dp, wp, accessToken)
- private val verifiedCatalogs = new AtomicInteger()
private val verifiedNamespaces = new AtomicInteger()
private val verifiedTables = new AtomicInteger()
private val verifiedViews = new AtomicInteger()
@@ -91,18 +89,6 @@ class ReadTreeDataset extends Simulation {
session
}
- //
--------------------------------------------------------------------------------
- // Workload: Verify each catalog
- //
--------------------------------------------------------------------------------
- private val verifyCatalogs = scenario("Verify catalogs using the Polaris
Management REST API")
- .exec(authenticationActions.restoreAccessTokenInSession)
- .asLongAs(session =>
- verifiedCatalogs.getAndIncrement() < dp.numCatalogs &&
session.contains("accessToken")
- )(
- feed(catalogActions.feeder())
- .exec(catalogActions.fetchCatalog)
- )
-
//
--------------------------------------------------------------------------------
// Workload: Verify namespaces
//
--------------------------------------------------------------------------------
@@ -167,7 +153,6 @@ class ReadTreeDataset extends Simulation {
continuouslyRefreshOauthToken.inject(atOnceUsers(1)).protocols(dremioHttpProtocol),
waitForAuthentication
.inject(atOnceUsers(1))
-
.andThen(verifyCatalogs.inject(atOnceUsers(1)).protocols(polarisHttpProtocol))
.andThen(verifyNamespaces.inject(atOnceUsers(namespaceThroughput)).protocols(polarisHttpProtocol))
.andThen(verifyTables.inject(atOnceUsers(tableThroughput)).protocols(polarisHttpProtocol))
.andThen(verifyViews.inject(atOnceUsers(viewThroughput)).protocols(polarisHttpProtocol))
diff --git
a/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/simulations/ReadUpdateTreeDataset.scala
b/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/simulations/ReadUpdateTreeDataset.scala
index c205b0d..13b373e 100644
---
a/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/simulations/ReadUpdateTreeDataset.scala
+++
b/benchmarks/src/gatling/scala/org/apache/polaris/benchmarks/simulations/ReadUpdateTreeDataset.scala
@@ -58,7 +58,6 @@ class ReadUpdateTreeDataset extends Simulation {
private val shouldRefreshToken: AtomicBoolean = new AtomicBoolean(true)
private val authActions = AuthenticationActions(cp, accessToken)
- private val catActions = CatalogActions(dp, accessToken)
private val nsActions = NamespaceActions(dp, wp, accessToken)
private val tblActions = TableActions(dp, wp, accessToken)
private val viewActions = ViewActions(dp, wp, accessToken)
