This is an automated email from the ASF dual-hosted git repository.
janhoy pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr-orbit.git
The following commit(s) were added to refs/heads/main by this push:
new 9b1199b3 Remove OpenSearch-specific cloud_provider module (#15) (#26)
9b1199b3 is described below
commit 9b1199b3942224baa7ca90a07fe40926d8bb2fe9
Author: Jan Høydahl <[email protected]>
AuthorDate: Tue May 26 00:07:47 2026 +0200
Remove OpenSearch-specific cloud_provider module (#15) (#26)
---
osbenchmark/cloud_provider/__init__.py | 20 ---
osbenchmark/cloud_provider/cloud_provider.py | 44 -----
osbenchmark/cloud_provider/factory.py | 43 -----
osbenchmark/cloud_provider/vendors/__init__.py | 18 --
osbenchmark/cloud_provider/vendors/aws.py | 188 ---------------------
.../vendors => utils}/s3_data_producer.py | 0
osbenchmark/workload/params.py | 2 +-
7 files changed, 1 insertion(+), 314 deletions(-)
diff --git a/osbenchmark/cloud_provider/__init__.py
b/osbenchmark/cloud_provider/__init__.py
deleted file mode 100644
index 8a1ceaf6..00000000
--- a/osbenchmark/cloud_provider/__init__.py
+++ /dev/null
@@ -1,20 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-#
-# The OpenSearch Contributors require contributions made to
-# this file be licensed under the Apache-2.0 license or a
-# compatible open source license.
-# Modifications Copyright OpenSearch Contributors. See
-# GitHub history for details.
-# Licensed to Elasticsearch B.V. under one or more contributor
-# license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright
-# ownership. Elasticsearch B.V. licenses this file to you under
-# the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-
-from .cloud_provider import CloudProvider
-from .factory import CloudProviderFactory
-from .vendors import AWSProvider
diff --git a/osbenchmark/cloud_provider/cloud_provider.py
b/osbenchmark/cloud_provider/cloud_provider.py
deleted file mode 100644
index 2d7f8eb0..00000000
--- a/osbenchmark/cloud_provider/cloud_provider.py
+++ /dev/null
@@ -1,44 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-#
-# The OpenSearch Contributors require contributions made to
-# this file be licensed under the Apache-2.0 license or a
-# compatible open source license.
-# Modifications Copyright OpenSearch Contributors. See
-# GitHub history for details.
-# Licensed to Elasticsearch B.V. under one or more contributor
-# license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright
-# ownership. Elasticsearch B.V. licenses this file to you under
-# the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-
-from abc import ABC, abstractmethod
-
-class CloudProvider(ABC):
-
- @abstractmethod
- def validate_client_options(self, client_options: dict) -> bool:
- pass
-
- @abstractmethod
- def validate_config_for_metrics(self, config) -> bool:
- pass
-
- @abstractmethod
- def mask_client_options(self, masked_client_options: dict, client_options:
dict) -> dict:
- pass
-
- @abstractmethod
- def parse_log_in_params(self, client_options=None, config=None,
for_metrics_datastore=False) -> dict:
- pass
-
- @abstractmethod
- def update_client_options_for_metrics(self, client_options) -> dict:
- pass
-
- @abstractmethod
- def create_client(self, hosts, client_options, client_class=None,
use_async=False):
- pass
diff --git a/osbenchmark/cloud_provider/factory.py
b/osbenchmark/cloud_provider/factory.py
deleted file mode 100644
index 1ea09fc5..00000000
--- a/osbenchmark/cloud_provider/factory.py
+++ /dev/null
@@ -1,43 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-#
-# The OpenSearch Contributors require contributions made to
-# this file be licensed under the Apache-2.0 license or a
-# compatible open source license.
-# Modifications Copyright OpenSearch Contributors. See
-# GitHub history for details.
-# Licensed to Elasticsearch B.V. under one or more contributor
-# license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright
-# ownership. Elasticsearch B.V. licenses this file to you under
-# the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-
-from typing import Optional
-
-from osbenchmark.cloud_provider.cloud_provider import CloudProvider
-from osbenchmark.cloud_provider.vendors import aws
-
-class CloudProviderFactory:
-
- providers = [
- aws.AWSProvider()
- ]
-
- @classmethod
- def get_provider_from_client_options(cls, client_options) ->
Optional[CloudProvider]:
- for provider in cls.providers:
- if provider.validate_client_options(client_options):
- return provider
-
- return None
-
- @classmethod
- def get_provider_from_config(cls, config) -> Optional[CloudProvider]:
- for provider in cls.providers:
- if provider.validate_config_for_metrics(config):
- return provider
-
- return None
diff --git a/osbenchmark/cloud_provider/vendors/__init__.py
b/osbenchmark/cloud_provider/vendors/__init__.py
deleted file mode 100644
index 51b201d7..00000000
--- a/osbenchmark/cloud_provider/vendors/__init__.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-#
-# The OpenSearch Contributors require contributions made to
-# this file be licensed under the Apache-2.0 license or a
-# compatible open source license.
-# Modifications Copyright OpenSearch Contributors. See
-# GitHub history for details.
-# Licensed to Elasticsearch B.V. under one or more contributor
-# license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright
-# ownership. Elasticsearch B.V. licenses this file to you under
-# the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-
-from .aws import AWSProvider
diff --git a/osbenchmark/cloud_provider/vendors/aws.py
b/osbenchmark/cloud_provider/vendors/aws.py
deleted file mode 100644
index 05fabf8a..00000000
--- a/osbenchmark/cloud_provider/vendors/aws.py
+++ /dev/null
@@ -1,188 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-#
-# The OpenSearch Contributors require contributions made to
-# this file be licensed under the Apache-2.0 license or a
-# compatible open source license.
-# Modifications Copyright OpenSearch Contributors. See
-# GitHub history for details.
-# Licensed to Elasticsearch B.V. under one or more contributor
-# license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright
-# ownership. Elasticsearch B.V. licenses this file to you under
-# the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-import os
-import logging
-
-from osbenchmark import exceptions
-from ..cloud_provider import CloudProvider
-
-class AWSProvider(CloudProvider):
- AVAILABLE_SERVICES = ['es', 'aoss']
- VALID_CONFIG_SETTINGS = ['config', 'environment', 'session']
-
- def __init__(self):
- self.aws_log_in_config = {}
- self.aws_metrics_log_in_config = {}
- self.logger = logging.getLogger(__name__)
-
- def validate_client_options(self, client_options) -> bool:
- return "amazon_aws_log_in" in client_options
-
- def validate_config_for_metrics(self, config) -> bool:
- metrics_amazon_aws_log_in = config.opts("reporting",
"datastore.amazon_aws_log_in",
- default_value=None,
mandatory=False)
-
- if metrics_amazon_aws_log_in in AWSProvider.VALID_CONFIG_SETTINGS:
- return True
-
- return False
-
- def mask_client_options(self, masked_client_options, client_options) ->
dict:
- masked_client_options["aws_access_key_id"] = "*****"
- masked_client_options["aws_secret_access_key"] = "*****"
- # session_token is optional and used only for role based access
- if self.aws_log_in_config.get("aws_session_token", None):
- masked_client_options["aws_session_token"] = "*****"
-
- return masked_client_options
-
- def parse_log_in_params(self, client_options=None, config=None,
for_metrics_datastore=False) -> dict:
- if for_metrics_datastore:
- # This is meant for the situation where benchmark.ini specifies
datastore with AWS credentials
- if config is None:
- raise exceptions.ConfigError("Missing config when parsing log
in params for metrics.")
-
- metrics_amazon_aws_log_in = config.opts("reporting",
"datastore.amazon_aws_log_in",
- default_value=None,
mandatory=False)
-
- metrics_aws_access_key_id = None
- metrics_aws_secret_access_key = None
- metrics_aws_session_token = None
- metrics_aws_region = None
- metrics_aws_service = None
-
- if metrics_amazon_aws_log_in == 'config':
- metrics_aws_access_key_id = config.opts("reporting",
"datastore.aws_access_key_id",
-
default_value=None, mandatory=False)
- metrics_aws_secret_access_key = config.opts("reporting",
"datastore.aws_secret_access_key",
-
default_value=None, mandatory=False)
- metrics_aws_session_token = config.opts("reporting",
"datastore.aws_session_token",
-
default_value=None, mandatory=False)
- metrics_aws_region = config.opts("reporting",
"datastore.region",
- default_value=None,
mandatory=False)
- metrics_aws_service = config.opts("reporting",
"datastore.service",
- default_value=None,
mandatory=False)
- elif metrics_amazon_aws_log_in == 'environment':
- metrics_aws_access_key_id =
os.getenv("OSB_DATASTORE_AWS_ACCESS_KEY_ID", default=None)
- metrics_aws_secret_access_key =
os.getenv("OSB_DATASTORE_AWS_SECRET_ACCESS_KEY", default=None)
- metrics_aws_session_token =
os.getenv("OSB_DATASTORE_AWS_SESSION_TOKEN", default=None)
- metrics_aws_region = os.getenv("OSB_DATASTORE_REGION",
default=None)
- metrics_aws_service = os.getenv("OSB_DATASTORE_SERVICE",
default=None)
-
- if metrics_amazon_aws_log_in is not None:
- if (
- not metrics_aws_access_key_id or
- not metrics_aws_secret_access_key or
- not metrics_aws_region or
- not metrics_aws_service
- ):
- if metrics_amazon_aws_log_in == 'environment':
- missing_aws_credentials_message = "Missing AWS
credentials through " \
-
"OSB_DATASTORE_AWS_ACCESS_KEY_ID, " \
-
"OSB_DATASTORE_AWS_SECRET_ACCESS_KEY, " \
- "OSB_DATASTORE_REGION,
OSB_DATASTORE_SERVICE " \
- "environment
variables."
- elif metrics_amazon_aws_log_in == 'config':
- missing_aws_credentials_message = "Missing AWS
credentials through datastore.aws_access_key_id, " \
-
"datastore.aws_secret_access_key, datastore.region, " \
- "datastore.service in
the config file."
- else:
- missing_aws_credentials_message =
"datastore.amazon_aws_log_in can only be one of " \
- "'environment' or
'config'"
- raise
exceptions.ConfigError(missing_aws_credentials_message) from None
-
- if metrics_aws_service not in AWSProvider.AVAILABLE_SERVICES:
- raise exceptions.ConfigError(f"datastore.service can only
be one of {AWSProvider.AVAILABLE_SERVICES}") from None
-
- self.aws_metrics_log_in_config['metrics_aws_log_in_choice'] =
metrics_amazon_aws_log_in
- self.aws_metrics_log_in_config['metrics_aws_access_key_id'] =
metrics_aws_access_key_id
- self.aws_metrics_log_in_config['metrics_aws_secret_access_key'] =
metrics_aws_secret_access_key
- self.aws_metrics_log_in_config['metrics_aws_session_token'] =
metrics_aws_session_token
- self.aws_metrics_log_in_config['metrics_aws_service'] =
metrics_aws_service
- self.aws_metrics_log_in_config['metrics_aws_region'] =
metrics_aws_region
-
- else:
- def validate_for_environment_and_client_options():
- # Validate aws_log_in_config
- required_fields = ["aws_access_key_id",
"aws_secret_access_key", "service", "region"]
- for field in required_fields:
- if not self.aws_log_in_config[field]:
- msg = "Invalid AWS log in parameters, required inputs
are aws_access_key_id, \
- aws_secret_access_key, service and region."
- self.logger.error(msg)
- raise exceptions.SystemSetupError(msg)
-
- # This is for all other client use-cases
- if client_options is None:
- raise exceptions.ConfigurationError("Missing client options
when parsing log in params")
-
- # AWS log in : option 1) pass in parameters from os environment
variables
- if client_options["amazon_aws_log_in"] == "environment":
- self.aws_log_in_config["aws_access_key_id"] =
os.environ.get("OSB_AWS_ACCESS_KEY_ID")
- self.aws_log_in_config["aws_secret_access_key"] =
os.environ.get("OSB_AWS_SECRET_ACCESS_KEY")
- self.aws_log_in_config["region"] = os.environ.get("OSB_REGION")
- self.aws_log_in_config["service"] =
os.environ.get("OSB_SERVICE")
- # optional: applicable only for role-based access
- self.aws_log_in_config["aws_session_token"] =
os.environ.get("OSB_AWS_SESSION_TOKEN")
- validate_for_environment_and_client_options()
-
- # AWS log in : option 2) parameters are passed in from command line
- elif client_options["amazon_aws_log_in"] == "client_option":
- self.aws_log_in_config["aws_access_key_id"] =
client_options.get("aws_access_key_id")
- self.aws_log_in_config["aws_secret_access_key"] =
client_options.get("aws_secret_access_key")
- self.aws_log_in_config["region"] = client_options.get("region")
- self.aws_log_in_config["service"] =
client_options.get("service")
- # optional: applicable only for role-based access
- self.aws_log_in_config["aws_session_token"] =
client_options.get("aws_session_token")
- validate_for_environment_and_client_options()
-
- # AWS log in: option 3) parameters are passed in from command line
but for session
- elif client_options["amazon_aws_log_in"] == "session":
- self.aws_log_in_config["region"] = client_options.get("region")
- self.aws_log_in_config["service"] =
client_options.get("service")
-
- # Validate session differently from environment and
client_option
- if client_options["amazon_aws_log_in"] == "session" and not
self.aws_log_in_config["region"]:
- self.logger.error("region is mandatory parameter for
session client.")
- raise exceptions.SystemSetupError(
- "region is mandatory parameter for session client."
- )
-
- if self.aws_log_in_config["service"] not in
AWSProvider.AVAILABLE_SERVICES:
- self.logger.error("Service for AWS log in should be one %s",
AWSProvider.AVAILABLE_SERVICES)
- raise exceptions.SystemSetupError(
- "Cannot specify service as '{}'. Accepted values are
{}.".format(
- self.aws_log_in_config["service"],
- AWSProvider.AVAILABLE_SERVICES)
- )
-
- def update_client_options_for_metrics(self, client_options):
- if self.aws_metrics_log_in_config['metrics_aws_log_in_choice'] is not
None:
- client_options["amazon_aws_log_in"] = 'client_option'
- client_options["aws_access_key_id"] =
self.aws_metrics_log_in_config['metrics_aws_access_key_id']
- client_options["aws_secret_access_key"] =
self.aws_metrics_log_in_config['metrics_aws_secret_access_key']
- client_options["service"] =
self.aws_metrics_log_in_config['metrics_aws_service']
- client_options["region"] =
self.aws_metrics_log_in_config['metrics_aws_region']
-
- if self.aws_metrics_log_in_config['metrics_aws_session_token']:
- client_options["aws_session_token"] =
self.aws_metrics_log_in_config['metrics_aws_session_token']
-
- return client_options
-
- def create_client(self, hosts, client_options, client_class=None,
use_async=False):
- raise exceptions.SystemSetupError("AWS OpenSearch client creation is
not supported in this build. "
- "This benchmark tool targets Apache
Solr.")
diff --git a/osbenchmark/cloud_provider/vendors/s3_data_producer.py
b/osbenchmark/utils/s3_data_producer.py
similarity index 100%
rename from osbenchmark/cloud_provider/vendors/s3_data_producer.py
rename to osbenchmark/utils/s3_data_producer.py
diff --git a/osbenchmark/workload/params.py b/osbenchmark/workload/params.py
index 0dec7774..a5a53d3d 100644
--- a/osbenchmark/workload/params.py
+++ b/osbenchmark/workload/params.py
@@ -882,7 +882,7 @@ class Slice:
client_options_obj = IngestionManager.config.opts("client", "options")
client_options = getattr(client_options_obj, "all_client_options", {})
# pylint: disable = import-outside-toplevel
- from osbenchmark.cloud_provider.vendors.s3_data_producer import
S3DataProducer
+ from osbenchmark.utils.s3_data_producer import S3DataProducer
bucket = re.sub('^s3://', "", Slice.base_url)
keys = Slice.document_file
producer = S3DataProducer(bucket, keys, client_options, Slice.data_dir)
