This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 4c9deff333 [ASTERIXDB-3423][CONF] Add compiler property for COPY TO
buffer size
4c9deff333 is described below
commit 4c9deff333b8f88d6aa542d93381542d5e3819e6
Author: Ali Alsuliman <[email protected]>
AuthorDate: Thu Jun 13 01:03:56 2024 +0300
[ASTERIXDB-3423][CONF] Add compiler property for COPY TO buffer size
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Add property 'COMPILER_COPY_TO_WRITE_BUFFER_SIZE' to configure
COPY TO write buffer size. Default is 8MB with a minimum of 5MB.
- Enforce a minimum of 5MB for 'CLOUD_WRITE_BUFFER_SIZE'.
- Remove property 'COMPILER_ENABLE_DB_RESOLUTION'.
Change-Id: I50bebc4b8b683889855cb5dd048ab27d7193ff76
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18370
Integration-Tests: Jenkins <[email protected]>
Reviewed-by: Ali Alsuliman <[email protected]>
Reviewed-by: Murtadha Hubail <[email protected]>
Tested-by: Ali Alsuliman <[email protected]>
---
.../apache/asterix/hyracks/bootstrap/CCApplication.java | 5 +----
.../apache/asterix/hyracks/bootstrap/NCApplication.java | 5 +----
.../api/cluster_state_1/cluster_state_1.1.regexadm | 1 +
.../cluster_state_1_full/cluster_state_1_full.1.regexadm | 1 +
.../cluster_state_1_less/cluster_state_1_less.1.regexadm | 1 +
.../asterix/cloud/clients/aws/s3/S3ClientConfig.java | 4 +---
.../writer/AbstractCloudExternalFileWriterFactory.java | 2 ++
.../cloud/writer/S3ExternalFileWriterFactory.java | 2 +-
.../apache/asterix/common/config/CloudProperties.java | 7 +++++--
.../apache/asterix/common/config/CompilerProperties.java | 16 +++++++++++-----
.../metadata/provider/ExternalWriterProvider.java | 4 +++-
.../runtime/writer/ExternalFileWriterConfiguration.java | 8 +++++++-
12 files changed, 35 insertions(+), 21 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
index 8cd24873cc..f19dea6059 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
@@ -73,7 +73,6 @@ import
org.apache.asterix.common.cluster.IGlobalRecoveryManager;
import org.apache.asterix.common.cluster.IGlobalTxManager;
import org.apache.asterix.common.config.AsterixExtension;
import org.apache.asterix.common.config.CloudProperties;
-import org.apache.asterix.common.config.CompilerProperties;
import org.apache.asterix.common.config.ExtensionProperties;
import org.apache.asterix.common.config.ExternalProperties;
import org.apache.asterix.common.config.GlobalConfig;
@@ -171,10 +170,8 @@ public class CCApplication extends BaseCCApplication {
new
ReplicationProperties(PropertiesAccessor.getInstance(ccServiceCtx.getAppConfig()));
INcLifecycleCoordinator lifecycleCoordinator =
createNcLifeCycleCoordinator(repProp.isReplicationEnabled());
componentProvider = new StorageComponentProvider();
- boolean isDbResolutionEnabled =
-
ccServiceCtx.getAppConfig().getBoolean(CompilerProperties.Option.COMPILER_ENABLE_DB_RESOLUTION);
boolean cloudDeployment =
ccServiceCtx.getAppConfig().getBoolean(CLOUD_DEPLOYMENT);
- boolean useDatabaseResolution = cloudDeployment &&
isDbResolutionEnabled;
+ boolean useDatabaseResolution = cloudDeployment;
INamespaceResolver namespaceResolver =
createNamespaceResolver(useDatabaseResolution);
INamespacePathResolver namespacePathResolver = new
NamespacePathResolver(useDatabaseResolution);
ccExtensionManager = new CCExtensionManager(new
ArrayList<>(getExtensions()), namespaceResolver, ccServiceCtx);
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
index d02dc4fc09..0ff8796975 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
@@ -61,7 +61,6 @@ import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.api.IPropertiesFactory;
import org.apache.asterix.common.api.IReceptionistFactory;
import org.apache.asterix.common.config.AsterixExtension;
-import org.apache.asterix.common.config.CompilerProperties;
import org.apache.asterix.common.config.ExtensionProperties;
import org.apache.asterix.common.config.ExternalProperties;
import org.apache.asterix.common.config.GlobalConfig;
@@ -160,10 +159,8 @@ public class NCApplication extends BaseNCApplication {
}
MetadataBuiltinFunctions.init();
- boolean isDbResolutionEnabled =
-
ncServiceCtx.getAppConfig().getBoolean(CompilerProperties.Option.COMPILER_ENABLE_DB_RESOLUTION);
boolean cloudDeployment =
ncServiceCtx.getAppConfig().getBoolean(CLOUD_DEPLOYMENT);
- boolean useDatabaseResolution = cloudDeployment &&
isDbResolutionEnabled;
+ boolean useDatabaseResolution = cloudDeployment;
INamespaceResolver namespaceResolver =
createNamespaceResolver(useDatabaseResolution);
NamespacePathResolver namespacePathResolver = new
NamespacePathResolver(useDatabaseResolution);
ncExtensionManager = new NCExtensionManager(new
ArrayList<>(getExtensions()), cloudDeployment,
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
index 93b92bc0df..107de0079a 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
@@ -29,6 +29,7 @@
"compiler.batch.lookup" : true,
"compiler.cbo" : true,
"compiler.column.filter" : true,
+ "compiler.copy.to.write.buffer.size" : 8388608,
"compiler\.external\.field\.pushdown" : true,
"compiler.forcejoinorder" : false,
"compiler\.framesize" : 32768,
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
index a5e70a5fcb..bdeedd67f7 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
@@ -29,6 +29,7 @@
"compiler.batch.lookup" : true,
"compiler.cbo" : true,
"compiler.column.filter" : true,
+ "compiler.copy.to.write.buffer.size" : 8388608,
"compiler\.external\.field\.pushdown" : true,
"compiler.forcejoinorder" : false,
"compiler\.framesize" : 32768,
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
index cc1db052bf..96fa23ca42 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
@@ -29,6 +29,7 @@
"compiler.batch.lookup" : true,
"compiler.cbo" : true,
"compiler.column.filter" : true,
+ "compiler.copy.to.write.buffer.size" : 8388608,
"compiler\.external\.field\.pushdown" : true,
"compiler.forcejoinorder" : false,
"compiler\.framesize" : 32768,
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
index c98e0e8cc4..4357b37680 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
@@ -22,7 +22,6 @@ import java.util.Map;
import org.apache.asterix.common.config.CloudProperties;
import org.apache.asterix.external.util.aws.s3.S3Constants;
-import org.apache.hyracks.util.StorageUtil;
import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
@@ -55,7 +54,7 @@ public final class S3ClientConfig {
cloudProperties.getProfilerLogInterval(),
cloudProperties.getWriteBufferSize());
}
- public static S3ClientConfig of(Map<String, String> configuration) {
+ public static S3ClientConfig of(Map<String, String> configuration, int
writeBufferSize) {
// Used to determine local vs. actual S3
String endPoint =
configuration.getOrDefault(S3Constants.SERVICE_END_POINT_FIELD_NAME, "");
// Disabled
@@ -65,7 +64,6 @@ public final class S3ClientConfig {
String region = "";
String prefix = "";
boolean anonymousAuth = false;
- int writeBufferSize = StorageUtil.getIntSizeInBytes(5,
StorageUtil.StorageUnit.MEGABYTE);
return new S3ClientConfig(region, endPoint, prefix, anonymousAuth,
profilerLogInterval, writeBufferSize);
}
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/AbstractCloudExternalFileWriterFactory.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/AbstractCloudExternalFileWriterFactory.java
index d73957e481..589ee7904d 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/AbstractCloudExternalFileWriterFactory.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/AbstractCloudExternalFileWriterFactory.java
@@ -50,12 +50,14 @@ abstract class AbstractCloudExternalFileWriterFactory
implements IExternalFileWr
protected final Map<String, String> configuration;
protected final SourceLocation pathSourceLocation;
protected final String staticPath;
+ protected final int writeBufferSize;
protected transient ICloudClient cloudClient;
AbstractCloudExternalFileWriterFactory(ExternalFileWriterConfiguration
externalConfig) {
configuration = externalConfig.getConfiguration();
pathSourceLocation = externalConfig.getPathSourceLocation();
staticPath = externalConfig.getStaticPath();
+ writeBufferSize = externalConfig.getWriteBufferSize();
}
abstract ICloudClient createCloudClient() throws CompilationException;
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriterFactory.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriterFactory.java
index dcaf488c10..e07acc0246 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriterFactory.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriterFactory.java
@@ -62,7 +62,7 @@ public final class S3ExternalFileWriterFactory extends
AbstractCloudExternalFile
@Override
ICloudClient createCloudClient() throws CompilationException {
- S3ClientConfig config = S3ClientConfig.of(configuration);
+ S3ClientConfig config = S3ClientConfig.of(configuration,
writeBufferSize);
return new S3CloudClient(config,
S3Utils.buildAwsS3Client(configuration),
ICloudGuardian.NoOpCloudGuardian.INSTANCE);
}
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
index 5c612dc718..7e7ac69867 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
@@ -24,6 +24,7 @@ import static
org.apache.hyracks.control.common.config.OptionTypes.LONG_BYTE_UNI
import static
org.apache.hyracks.control.common.config.OptionTypes.NONNEGATIVE_INTEGER;
import static
org.apache.hyracks.control.common.config.OptionTypes.POSITIVE_INTEGER;
import static org.apache.hyracks.control.common.config.OptionTypes.STRING;
+import static
org.apache.hyracks.control.common.config.OptionTypes.getRangedIntegerType;
import static org.apache.hyracks.util.StorageUtil.StorageUnit.GIGABYTE;
import java.util.concurrent.TimeUnit;
@@ -57,7 +58,9 @@ public class CloudProperties extends AbstractProperties {
CLOUD_STORAGE_DEBUG_MODE_ENABLED(BOOLEAN, false),
CLOUD_STORAGE_DEBUG_SWEEP_THRESHOLD_SIZE(LONG_BYTE_UNIT,
StorageUtil.getLongSizeInBytes(1, GIGABYTE)),
CLOUD_PROFILER_LOG_INTERVAL(NONNEGATIVE_INTEGER, 5),
- CLOUD_WRITE_BUFFER_SIZE(POSITIVE_INTEGER,
StorageUtil.getIntSizeInBytes(8, StorageUtil.StorageUnit.MEGABYTE));
+ CLOUD_WRITE_BUFFER_SIZE(
+ getRangedIntegerType(5, Integer.MAX_VALUE),
+ StorageUtil.getIntSizeInBytes(8,
StorageUtil.StorageUnit.MEGABYTE));
private final IOptionType interpreter;
private final Object defaultValue;
@@ -142,7 +145,7 @@ public class CloudProperties extends AbstractProperties {
+ " the profiler is disabled by default). The
minimum is 1 minute."
+ " NOTE: Enabling the profiler could perturb the
performance of cloud requests";
case CLOUD_WRITE_BUFFER_SIZE:
- return "The write buffer size in bytes. (default: 8MB)";
+ return "The write buffer size in bytes. (default: 8MB,
min: 5MB)";
default:
throw new IllegalStateException("NYI: " + this);
}
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
index 042516e0c9..cfe7ce8d5c 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java
@@ -25,6 +25,7 @@ import static
org.apache.hyracks.control.common.config.OptionTypes.LONG_BYTE_UNI
import static
org.apache.hyracks.control.common.config.OptionTypes.NONNEGATIVE_INTEGER;
import static
org.apache.hyracks.control.common.config.OptionTypes.POSITIVE_INTEGER;
import static org.apache.hyracks.control.common.config.OptionTypes.STRING;
+import static
org.apache.hyracks.control.common.config.OptionTypes.getRangedIntegerType;
import static org.apache.hyracks.util.StorageUtil.StorageUnit.KILOBYTE;
import static org.apache.hyracks.util.StorageUtil.StorageUnit.MEGABYTE;
@@ -124,12 +125,14 @@ public class CompilerProperties extends
AbstractProperties {
BOOLEAN,
AlgebricksConfig.COLUMN_FILTER_DEFAULT,
"Enable/disable the use of column min/max filters"),
- //TODO(DB): remove after
- COMPILER_ENABLE_DB_RESOLUTION(BOOLEAN, true, "Enable/disable the
resolution of namespaces to database"),
COMPILER_RUNTIME_MEMORY_OVERHEAD(
NONNEGATIVE_INTEGER,
5,
- "A percentage of the job's required memory to be added to
account for runtime memory overhead");
+ "A percentage of the job's required memory to be added to
account for runtime memory overhead"),
+ COMPILER_COPY_TO_WRITE_BUFFER_SIZE(
+ getRangedIntegerType(5, Integer.MAX_VALUE),
+ StorageUtil.getIntSizeInBytes(8,
StorageUtil.StorageUnit.MEGABYTE),
+ "The COPY TO write buffer size in bytes. (default: 8MB, min:
5MB)");
private final IOptionType type;
private final Object defaultValue;
@@ -163,8 +166,7 @@ public class CompilerProperties extends AbstractProperties {
@Override
public boolean hidden() {
- return this == COMPILER_EXTERNALSCANMEMORY || this ==
COMPILER_CBOTEST
- || this == COMPILER_ENABLE_DB_RESOLUTION;
+ return this == COMPILER_EXTERNALSCANMEMORY || this ==
COMPILER_CBOTEST;
}
}
@@ -323,4 +325,8 @@ public class CompilerProperties extends AbstractProperties {
public int getRuntimeMemoryOverheadPercentage() {
return accessor.getInt(Option.COMPILER_RUNTIME_MEMORY_OVERHEAD);
}
+
+ public int getCopyToWriteBufferSize() {
+ return accessor.getInt(Option.COMPILER_COPY_TO_WRITE_BUFFER_SIZE);
+ }
}
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java
index 8ca428b70a..b8583d0c0d 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java
@@ -86,8 +86,10 @@ public class ExternalWriterProvider {
IWriteDataSink sink, String staticPath, SourceLocation
pathExpressionLocation) {
Map<String, String> params = sink.getConfiguration();
boolean singleNodeCluster = isSingleNodeCluster(appCtx);
+ int copyToWriteBufferSize =
appCtx.getCompilerProperties().getCopyToWriteBufferSize();
- return new ExternalFileWriterConfiguration(params,
pathExpressionLocation, staticPath, singleNodeCluster);
+ return new ExternalFileWriterConfiguration(params,
pathExpressionLocation, staticPath, singleNodeCluster,
+ copyToWriteBufferSize);
}
private static boolean isSingleNodeCluster(ICcApplicationContext appCtx) {
diff --git
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/ExternalFileWriterConfiguration.java
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/ExternalFileWriterConfiguration.java
index b62a07aef8..080fa82423 100644
---
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/ExternalFileWriterConfiguration.java
+++
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/ExternalFileWriterConfiguration.java
@@ -27,13 +27,15 @@ public final class ExternalFileWriterConfiguration {
private final SourceLocation pathSourceLocation;
private final String staticPath;
private final boolean singleNodeCluster;
+ private final int writeBufferSize;
public ExternalFileWriterConfiguration(Map<String, String> configuration,
SourceLocation pathSourceLocation,
- String staticPath, boolean singleNodeCluster) {
+ String staticPath, boolean singleNodeCluster, int writeBufferSize)
{
this.configuration = configuration;
this.pathSourceLocation = pathSourceLocation;
this.staticPath = staticPath;
this.singleNodeCluster = singleNodeCluster;
+ this.writeBufferSize = writeBufferSize;
}
public Map<String, String> getConfiguration() {
@@ -51,4 +53,8 @@ public final class ExternalFileWriterConfiguration {
public boolean isSingleNodeCluster() {
return singleNodeCluster;
}
+
+ public int getWriteBufferSize() {
+ return writeBufferSize;
+ }
}
