>From Ali Alsuliman <[email protected]>: Ali Alsuliman has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18370 )
Change subject: [ASTERIXDB-3423][CONF] Add compiler property for COPY TO buffer size ...................................................................... [ASTERIXDB-3423][CONF] Add compiler property for COPY TO buffer size - user model changes: no - storage format changes: no - interface changes: no Details: - Add property 'COMPILER_COPY_TO_WRITE_BUFFER_SIZE' to configure COPY TO write buffer size. Default is 8MB with a minimum of 5MB. - Enforce a minimum of 5MB for 'CLOUD_WRITE_BUFFER_SIZE'. - Remove property 'COMPILER_ENABLE_DB_RESOLUTION'. Change-Id: I50bebc4b8b683889855cb5dd048ab27d7193ff76 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18370 Integration-Tests: Jenkins <[email protected]> Reviewed-by: Ali Alsuliman <[email protected]> Reviewed-by: Murtadha Hubail <[email protected]> Tested-by: Ali Alsuliman <[email protected]> --- M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/AbstractCloudExternalFileWriterFactory.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriterFactory.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm M asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java M asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/ExternalFileWriterConfiguration.java 12 files changed, 59 insertions(+), 21 deletions(-) Approvals: Murtadha Hubail: Looks good to me, approved Ali Alsuliman: Looks good to me, but someone else must approve; Verified Jenkins: Verified Objections: Anon. E. Moose #1000171: Violations found diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java index 8cd2487..f19dea6 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java @@ -73,7 +73,6 @@ import org.apache.asterix.common.cluster.IGlobalTxManager; import org.apache.asterix.common.config.AsterixExtension; import org.apache.asterix.common.config.CloudProperties; -import org.apache.asterix.common.config.CompilerProperties; import org.apache.asterix.common.config.ExtensionProperties; import org.apache.asterix.common.config.ExternalProperties; import org.apache.asterix.common.config.GlobalConfig; @@ -171,10 +170,8 @@ new ReplicationProperties(PropertiesAccessor.getInstance(ccServiceCtx.getAppConfig())); INcLifecycleCoordinator lifecycleCoordinator = createNcLifeCycleCoordinator(repProp.isReplicationEnabled()); componentProvider = new StorageComponentProvider(); - boolean isDbResolutionEnabled = - ccServiceCtx.getAppConfig().getBoolean(CompilerProperties.Option.COMPILER_ENABLE_DB_RESOLUTION); boolean cloudDeployment = ccServiceCtx.getAppConfig().getBoolean(CLOUD_DEPLOYMENT); - boolean useDatabaseResolution = cloudDeployment && isDbResolutionEnabled; + boolean useDatabaseResolution = cloudDeployment; INamespaceResolver namespaceResolver = createNamespaceResolver(useDatabaseResolution); INamespacePathResolver namespacePathResolver = new NamespacePathResolver(useDatabaseResolution); ccExtensionManager = new CCExtensionManager(new ArrayList<>(getExtensions()), namespaceResolver, ccServiceCtx); diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java index d02dc4f..0ff8796 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java @@ -61,7 +61,6 @@ import org.apache.asterix.common.api.IPropertiesFactory; import org.apache.asterix.common.api.IReceptionistFactory; import org.apache.asterix.common.config.AsterixExtension; -import org.apache.asterix.common.config.CompilerProperties; import org.apache.asterix.common.config.ExtensionProperties; import org.apache.asterix.common.config.ExternalProperties; import org.apache.asterix.common.config.GlobalConfig; @@ -160,10 +159,8 @@ } MetadataBuiltinFunctions.init(); - boolean isDbResolutionEnabled = - ncServiceCtx.getAppConfig().getBoolean(CompilerProperties.Option.COMPILER_ENABLE_DB_RESOLUTION); boolean cloudDeployment = ncServiceCtx.getAppConfig().getBoolean(CLOUD_DEPLOYMENT); - boolean useDatabaseResolution = cloudDeployment && isDbResolutionEnabled; + boolean useDatabaseResolution = cloudDeployment; INamespaceResolver namespaceResolver = createNamespaceResolver(useDatabaseResolution); NamespacePathResolver namespacePathResolver = new NamespacePathResolver(useDatabaseResolution); ncExtensionManager = new NCExtensionManager(new ArrayList<>(getExtensions()), cloudDeployment, diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm index 93b92bc..107de00 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm @@ -29,6 +29,7 @@ "compiler.batch.lookup" : true, "compiler.cbo" : true, "compiler.column.filter" : true, + "compiler.copy.to.write.buffer.size" : 8388608, "compiler\.external\.field\.pushdown" : true, "compiler.forcejoinorder" : false, "compiler\.framesize" : 32768, diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm index a5e70a5..bdeedd6 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm @@ -29,6 +29,7 @@ "compiler.batch.lookup" : true, "compiler.cbo" : true, "compiler.column.filter" : true, + "compiler.copy.to.write.buffer.size" : 8388608, "compiler\.external\.field\.pushdown" : true, "compiler.forcejoinorder" : false, "compiler\.framesize" : 32768, diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm index cc1db05..96fa23c 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm @@ -29,6 +29,7 @@ "compiler.batch.lookup" : true, "compiler.cbo" : true, "compiler.column.filter" : true, + "compiler.copy.to.write.buffer.size" : 8388608, "compiler\.external\.field\.pushdown" : true, "compiler.forcejoinorder" : false, "compiler\.framesize" : 32768, diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java index c98e0e8..4357b37 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java @@ -22,7 +22,6 @@ import org.apache.asterix.common.config.CloudProperties; import org.apache.asterix.external.util.aws.s3.S3Constants; -import org.apache.hyracks.util.StorageUtil; import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; @@ -55,7 +54,7 @@ cloudProperties.getProfilerLogInterval(), cloudProperties.getWriteBufferSize()); } - public static S3ClientConfig of(Map<String, String> configuration) { + public static S3ClientConfig of(Map<String, String> configuration, int writeBufferSize) { // Used to determine local vs. actual S3 String endPoint = configuration.getOrDefault(S3Constants.SERVICE_END_POINT_FIELD_NAME, ""); // Disabled @@ -65,7 +64,6 @@ String region = ""; String prefix = ""; boolean anonymousAuth = false; - int writeBufferSize = StorageUtil.getIntSizeInBytes(5, StorageUtil.StorageUnit.MEGABYTE); return new S3ClientConfig(region, endPoint, prefix, anonymousAuth, profilerLogInterval, writeBufferSize); } diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/AbstractCloudExternalFileWriterFactory.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/AbstractCloudExternalFileWriterFactory.java index d73957e..589ee79 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/AbstractCloudExternalFileWriterFactory.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/AbstractCloudExternalFileWriterFactory.java @@ -50,12 +50,14 @@ protected final Map<String, String> configuration; protected final SourceLocation pathSourceLocation; protected final String staticPath; + protected final int writeBufferSize; protected transient ICloudClient cloudClient; AbstractCloudExternalFileWriterFactory(ExternalFileWriterConfiguration externalConfig) { configuration = externalConfig.getConfiguration(); pathSourceLocation = externalConfig.getPathSourceLocation(); staticPath = externalConfig.getStaticPath(); + writeBufferSize = externalConfig.getWriteBufferSize(); } abstract ICloudClient createCloudClient() throws CompilationException; diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriterFactory.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriterFactory.java index dcaf488..e07acc0 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriterFactory.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriterFactory.java @@ -62,7 +62,7 @@ @Override ICloudClient createCloudClient() throws CompilationException { - S3ClientConfig config = S3ClientConfig.of(configuration); + S3ClientConfig config = S3ClientConfig.of(configuration, writeBufferSize); return new S3CloudClient(config, S3Utils.buildAwsS3Client(configuration), ICloudGuardian.NoOpCloudGuardian.INSTANCE); } diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java index 5c612dc..7e7ac69 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java @@ -24,6 +24,7 @@ import static org.apache.hyracks.control.common.config.OptionTypes.NONNEGATIVE_INTEGER; import static org.apache.hyracks.control.common.config.OptionTypes.POSITIVE_INTEGER; import static org.apache.hyracks.control.common.config.OptionTypes.STRING; +import static org.apache.hyracks.control.common.config.OptionTypes.getRangedIntegerType; import static org.apache.hyracks.util.StorageUtil.StorageUnit.GIGABYTE; import java.util.concurrent.TimeUnit; @@ -57,7 +58,9 @@ CLOUD_STORAGE_DEBUG_MODE_ENABLED(BOOLEAN, false), CLOUD_STORAGE_DEBUG_SWEEP_THRESHOLD_SIZE(LONG_BYTE_UNIT, StorageUtil.getLongSizeInBytes(1, GIGABYTE)), CLOUD_PROFILER_LOG_INTERVAL(NONNEGATIVE_INTEGER, 5), - CLOUD_WRITE_BUFFER_SIZE(POSITIVE_INTEGER, StorageUtil.getIntSizeInBytes(8, StorageUtil.StorageUnit.MEGABYTE)); + CLOUD_WRITE_BUFFER_SIZE( + getRangedIntegerType(5, Integer.MAX_VALUE), + StorageUtil.getIntSizeInBytes(8, StorageUtil.StorageUnit.MEGABYTE)); private final IOptionType interpreter; private final Object defaultValue; @@ -142,7 +145,7 @@ + " the profiler is disabled by default). The minimum is 1 minute." + " NOTE: Enabling the profiler could perturb the performance of cloud requests"; case CLOUD_WRITE_BUFFER_SIZE: - return "The write buffer size in bytes. (default: 8MB)"; + return "The write buffer size in bytes. (default: 8MB, min: 5MB)"; default: throw new IllegalStateException("NYI: " + this); } diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java index 042516e..cfe7ce8 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java @@ -25,6 +25,7 @@ import static org.apache.hyracks.control.common.config.OptionTypes.NONNEGATIVE_INTEGER; import static org.apache.hyracks.control.common.config.OptionTypes.POSITIVE_INTEGER; import static org.apache.hyracks.control.common.config.OptionTypes.STRING; +import static org.apache.hyracks.control.common.config.OptionTypes.getRangedIntegerType; import static org.apache.hyracks.util.StorageUtil.StorageUnit.KILOBYTE; import static org.apache.hyracks.util.StorageUtil.StorageUnit.MEGABYTE; @@ -124,12 +125,14 @@ BOOLEAN, AlgebricksConfig.COLUMN_FILTER_DEFAULT, "Enable/disable the use of column min/max filters"), - //TODO(DB): remove after - COMPILER_ENABLE_DB_RESOLUTION(BOOLEAN, true, "Enable/disable the resolution of namespaces to database"), COMPILER_RUNTIME_MEMORY_OVERHEAD( NONNEGATIVE_INTEGER, 5, - "A percentage of the job's required memory to be added to account for runtime memory overhead"); + "A percentage of the job's required memory to be added to account for runtime memory overhead"), + COMPILER_COPY_TO_WRITE_BUFFER_SIZE( + getRangedIntegerType(5, Integer.MAX_VALUE), + StorageUtil.getIntSizeInBytes(8, StorageUtil.StorageUnit.MEGABYTE), + "The COPY TO write buffer size in bytes. (default: 8MB, min: 5MB)"); private final IOptionType type; private final Object defaultValue; @@ -163,8 +166,7 @@ @Override public boolean hidden() { - return this == COMPILER_EXTERNALSCANMEMORY || this == COMPILER_CBOTEST - || this == COMPILER_ENABLE_DB_RESOLUTION; + return this == COMPILER_EXTERNALSCANMEMORY || this == COMPILER_CBOTEST; } } @@ -323,4 +325,8 @@ public int getRuntimeMemoryOverheadPercentage() { return accessor.getInt(Option.COMPILER_RUNTIME_MEMORY_OVERHEAD); } + + public int getCopyToWriteBufferSize() { + return accessor.getInt(Option.COMPILER_COPY_TO_WRITE_BUFFER_SIZE); + } } diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java index 8ca428b..b8583d0 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java @@ -86,8 +86,10 @@ IWriteDataSink sink, String staticPath, SourceLocation pathExpressionLocation) { Map<String, String> params = sink.getConfiguration(); boolean singleNodeCluster = isSingleNodeCluster(appCtx); + int copyToWriteBufferSize = appCtx.getCompilerProperties().getCopyToWriteBufferSize(); - return new ExternalFileWriterConfiguration(params, pathExpressionLocation, staticPath, singleNodeCluster); + return new ExternalFileWriterConfiguration(params, pathExpressionLocation, staticPath, singleNodeCluster, + copyToWriteBufferSize); } private static boolean isSingleNodeCluster(ICcApplicationContext appCtx) { diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/ExternalFileWriterConfiguration.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/ExternalFileWriterConfiguration.java index b62a07a..080fa82 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/ExternalFileWriterConfiguration.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/ExternalFileWriterConfiguration.java @@ -27,13 +27,15 @@ private final SourceLocation pathSourceLocation; private final String staticPath; private final boolean singleNodeCluster; + private final int writeBufferSize; public ExternalFileWriterConfiguration(Map<String, String> configuration, SourceLocation pathSourceLocation, - String staticPath, boolean singleNodeCluster) { + String staticPath, boolean singleNodeCluster, int writeBufferSize) { this.configuration = configuration; this.pathSourceLocation = pathSourceLocation; this.staticPath = staticPath; this.singleNodeCluster = singleNodeCluster; + this.writeBufferSize = writeBufferSize; } public Map<String, String> getConfiguration() { @@ -51,4 +53,8 @@ public boolean isSingleNodeCluster() { return singleNodeCluster; } + + public int getWriteBufferSize() { + return writeBufferSize; + } } -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18370 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I50bebc4b8b683889855cb5dd048ab27d7193ff76 Gerrit-Change-Number: 18370 Gerrit-PatchSet: 2 Gerrit-Owner: Ali Alsuliman <[email protected]> Gerrit-Reviewer: Ali Alsuliman <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Michael Blow <[email protected]> Gerrit-Reviewer: Murtadha Hubail <[email protected]> Gerrit-Reviewer: Wail Alkowaileet <[email protected]> Gerrit-MessageType: merged
