>From Hussain Towaileb <[email protected]>: Hussain Towaileb has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18197 )
Change subject: [NO ISSUE]: Add more validations + cleanup + address comments ...................................................................... [NO ISSUE]: Add more validations + cleanup + address comments Change-Id: I6e511e2ec13edf7458a809826860124507a51038 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18197 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Hussain Towaileb <[email protected]> Reviewed-by: Wail Alkowaileet <[email protected]> --- R asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/TextualExternalDatabasePrinterFactory.java M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/LocalFSExternalFileWriterFactory.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/TextualExternalFilePrinter.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/SinkWritePOperator.java A asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/TextualExternalDatabasePrinter.java M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java M hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java M asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriterFactory.java M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/TextualExternalFilePrinterFactory.java R asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/ExternalFileWriterConfiguration.java M asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/IExternalFileWriterFactoryProvider.java M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java R asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/AbstractTextualExternalPrinter.java M asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java 18 files changed, 110 insertions(+), 83 deletions(-) Approvals: Wail Alkowaileet: Looks good to me, approved Hussain Towaileb: Looks good to me, but someone else must approve Jenkins: Verified; Verified Anon. E. Moose #1000171: diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java index 8025202..01b8527 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java @@ -681,6 +681,9 @@ public boolean isAutogenerated() { return autogenerated; } - } + public boolean isFileStoreSink() { + return keyExpressions.isEmpty() && !autogenerated; + } + } } diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java index 0949478..077b39b 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java @@ -361,7 +361,7 @@ return translate(expr, outputDatasetName, (ICompiledDmlStatement) stmt, null, resultMetadata); } - public ILogicalPlan translateCopyTo(Query expr, CompiledStatements.ICompiledStatement stmt, + private ILogicalPlan translateCopyTo(Query expr, CompiledStatements.ICompiledStatement stmt, IResultMetadata resultMetadata) throws AlgebricksException { CompiledStatements.CompiledCopyToStatement copyTo = (CompiledStatements.CompiledCopyToStatement) stmt; MutableObject<ILogicalOperator> base = new MutableObject<>(new EmptyTupleSourceOperator()); @@ -423,7 +423,7 @@ // astPathExpressions has at least one expression see CopyToStatement constructor List<Expression> astPathExpressions = copyTo.getPathExpressions(); ILogicalExpression fullPathExpr = null; - String separator = getExternalWriterSeparator(copyTo.getAdapter()); + String separator = getSeparator(copyTo.getAdapter(), copyTo.isFileStoreSink()); List<Mutable<ILogicalExpression>> pathExprs = new ArrayList<>(astPathExpressions.size()); Pair<ILogicalExpression, Mutable<ILogicalOperator>> pathExprPair; for (int i = 0; i < astPathExpressions.size(); i++) { @@ -483,8 +483,8 @@ return new ALogicalPlanImpl(globalPlanRoots); } - protected String getExternalWriterSeparator(String adapter) { - return String.valueOf(ExternalWriterProvider.getSeparator(adapter)); + private String getSeparator(String adapter, boolean isFileStore) { + return isFileStore ? String.valueOf(ExternalWriterProvider.getSeparator(adapter)) : ""; } public ILogicalPlan translate(Query expr, String outputDatasetName, ICompiledDmlStatement stmt, diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriterFactory.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriterFactory.java index 4477b1f..cb0a6e7 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriterFactory.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriterFactory.java @@ -36,7 +36,7 @@ import org.apache.asterix.common.exceptions.ErrorCode; import org.apache.asterix.external.util.ExternalDataConstants; import org.apache.asterix.external.util.aws.s3.S3Utils; -import org.apache.asterix.runtime.writer.ExternalWriterConfiguration; +import org.apache.asterix.runtime.writer.ExternalFileWriterConfiguration; import org.apache.asterix.runtime.writer.IExternalFileWriter; import org.apache.asterix.runtime.writer.IExternalFileWriterFactory; import org.apache.asterix.runtime.writer.IExternalFileWriterFactoryProvider; @@ -61,7 +61,7 @@ static final char SEPARATOR = '/'; public static final IExternalFileWriterFactoryProvider PROVIDER = new IExternalFileWriterFactoryProvider() { @Override - public IExternalFileWriterFactory create(ExternalWriterConfiguration configuration) { + public IExternalFileWriterFactory create(ExternalFileWriterConfiguration configuration) { return new S3ExternalFileWriterFactory(configuration); } @@ -75,7 +75,7 @@ private final String staticPath; private transient S3CloudClient cloudClient; - private S3ExternalFileWriterFactory(ExternalWriterConfiguration externalConfig) { + private S3ExternalFileWriterFactory(ExternalFileWriterConfiguration externalConfig) { configuration = externalConfig.getConfiguration(); pathSourceLocation = externalConfig.getPathSourceLocation(); staticPath = externalConfig.getStaticPath(); diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java index ef54ba8..b508ce5 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java @@ -95,6 +95,7 @@ WRITE_PATH_LENGTH_EXCEEDS_MAX_LENGTH(65), TYPE_MISMATCH_EXTRA_FIELD(66), UNSUPPORTED_COLUMN_TYPE(67), + INVALID_KEY_TYPE(68), UNSUPPORTED_JRE(100), @@ -295,6 +296,7 @@ UNSUPPORTED_COLUMN_LSM_FILTER(1192), UNKNOWN_STORAGE_FORMAT(1193), UNSUPPORTED_INDEX_IN_COLUMNAR_FORMAT(1194), + COMPOSITE_KEY_NOT_SUPPORTED(1195), // Feed errors DATAFLOW_ILLEGAL_STATE(3001), diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties index 45b6185..3621447 100644 --- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties +++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties @@ -102,6 +102,7 @@ 65 = Length of the file path '%1$s' exceeds the maximum length of '%2$s bytes' allowed in %3$s 66 = Type mismatch: including an extra field %1$s 67 = Type(s) '%1$s' are not supported in columnar storage format. Supported types are %2$s +68 = Invalid key type. Expected '%1$s', found '%2$s'. 100 = Unsupported JRE: %1$s @@ -297,6 +298,7 @@ 1192 = Filters are not supported with columnar storage format 1193 = Unknown storage format '%1$s' 1194 = Index type '%1$s' is not supported with columnar storage format +1195 = Composite key is not allowed, only single key must be specified. # Feed Errors 3001 = Illegal state. diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/LocalFSExternalFileWriterFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/LocalFSExternalFileWriterFactory.java index 73f34f1..bdefaa8 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/LocalFSExternalFileWriterFactory.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/LocalFSExternalFileWriterFactory.java @@ -23,7 +23,7 @@ import org.apache.asterix.common.exceptions.CompilationException; import org.apache.asterix.common.exceptions.ErrorCode; import org.apache.asterix.common.exceptions.RuntimeDataException; -import org.apache.asterix.runtime.writer.ExternalWriterConfiguration; +import org.apache.asterix.runtime.writer.ExternalFileWriterConfiguration; import org.apache.asterix.runtime.writer.IExternalFileWriter; import org.apache.asterix.runtime.writer.IExternalFileWriterFactory; import org.apache.asterix.runtime.writer.IExternalFileWriterFactoryProvider; @@ -38,7 +38,7 @@ private static final char SEPARATOR = File.separatorChar; public static final IExternalFileWriterFactoryProvider PROVIDER = new IExternalFileWriterFactoryProvider() { @Override - public IExternalFileWriterFactory create(ExternalWriterConfiguration configuration) { + public IExternalFileWriterFactory create(ExternalFileWriterConfiguration configuration) { return new LocalFSExternalFileWriterFactory(configuration); } @@ -54,7 +54,7 @@ private final String staticPath; private boolean validated; - private LocalFSExternalFileWriterFactory(ExternalWriterConfiguration externalConfig) { + private LocalFSExternalFileWriterFactory(ExternalFileWriterConfiguration externalConfig) { pathSourceLocation = externalConfig.getPathSourceLocation(); singleNodeCluster = externalConfig.isSingleNodeCluster(); staticPath = externalConfig.getStaticPath(); diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/TextualExternalPrinter.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/AbstractTextualExternalPrinter.java similarity index 63% rename from asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/TextualExternalPrinter.java rename to asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/AbstractTextualExternalPrinter.java index 537af2e..8d7d60f 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/TextualExternalPrinter.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/AbstractTextualExternalPrinter.java @@ -21,18 +21,26 @@ import java.io.OutputStream; import java.io.PrintStream; +import org.apache.asterix.external.writer.compressor.IExternalFileCompressStreamFactory; +import org.apache.asterix.external.writer.compressor.NoOpExternalFileCompressStreamFactory; import org.apache.asterix.runtime.writer.IExternalPrinter; import org.apache.hyracks.algebricks.data.IPrinter; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.data.std.api.IValueReference; -final class TextualExternalPrinter implements IExternalPrinter { - private final IPrinter printer; - private TextualOutputStreamDelegate delegate; - private PrintStream printStream; +public abstract class AbstractTextualExternalPrinter implements IExternalPrinter { + final IPrinter printer; + final IExternalFileCompressStreamFactory compressStreamFactory; + TextualOutputStreamDelegate delegate; + PrintStream printStream; - TextualExternalPrinter(IPrinter printer) { + AbstractTextualExternalPrinter(IPrinter printer) { + this(printer, NoOpExternalFileCompressStreamFactory.INSTANCE); + } + + AbstractTextualExternalPrinter(IPrinter printer, final IExternalFileCompressStreamFactory compressStreamFactory) { this.printer = printer; + this.compressStreamFactory = compressStreamFactory; } @Override @@ -41,17 +49,23 @@ } @Override - public void newStream(OutputStream outputStream) { - delegate = new TextualOutputStreamDelegate(outputStream); + public void newStream(OutputStream outputStream) throws HyracksDataException { + if (printStream != null) { + close(); + } + delegate = new TextualOutputStreamDelegate(compressStreamFactory.createStream(outputStream)); printStream = new PrintStream(delegate); } @Override public void print(IValueReference value) throws HyracksDataException { printer.print(value.getByteArray(), value.getStartOffset(), value.getLength(), printStream); + afterPrint(); delegate.checkError(); } + abstract void afterPrint() throws HyracksDataException; + @Override public void close() throws HyracksDataException { if (printStream != null) { diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/TextualExternalDatabasePrinter.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/TextualExternalDatabasePrinter.java new file mode 100644 index 0000000..12c8d5a --- /dev/null +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/TextualExternalDatabasePrinter.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.external.writer.printer; + +import org.apache.hyracks.algebricks.data.IPrinter; + +final class TextualExternalDatabasePrinter extends AbstractTextualExternalPrinter { + + TextualExternalDatabasePrinter(IPrinter printer) { + super(printer); + } + + @Override + void afterPrint() { + } +} diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/TextualExternalPrinterFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/TextualExternalDatabasePrinterFactory.java similarity index 83% rename from asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/TextualExternalPrinterFactory.java rename to asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/TextualExternalDatabasePrinterFactory.java index d779793c..49bd6d8 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/TextualExternalPrinterFactory.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/TextualExternalDatabasePrinterFactory.java @@ -22,16 +22,16 @@ import org.apache.asterix.runtime.writer.IExternalPrinterFactory; import org.apache.hyracks.algebricks.data.IPrinterFactory; -public class TextualExternalPrinterFactory implements IExternalPrinterFactory { +public class TextualExternalDatabasePrinterFactory implements IExternalPrinterFactory { private static final long serialVersionUID = 9155959967258587588L; protected final IPrinterFactory printerFactory; - public TextualExternalPrinterFactory(IPrinterFactory printerFactory) { + public TextualExternalDatabasePrinterFactory(IPrinterFactory printerFactory) { this.printerFactory = printerFactory; } @Override public IExternalPrinter createPrinter() { - return new TextualExternalPrinter(printerFactory.createPrinter()); + return new TextualExternalDatabasePrinter(printerFactory.createPrinter()); } } diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/TextualExternalFilePrinter.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/TextualExternalFilePrinter.java index 57e7b58..5465098 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/TextualExternalFilePrinter.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/TextualExternalFilePrinter.java @@ -18,54 +18,17 @@ */ package org.apache.asterix.external.writer.printer; -import java.io.OutputStream; -import java.io.PrintStream; - import org.apache.asterix.external.writer.compressor.IExternalFileCompressStreamFactory; -import org.apache.asterix.runtime.writer.IExternalPrinter; import org.apache.hyracks.algebricks.data.IPrinter; -import org.apache.hyracks.api.exceptions.HyracksDataException; -import org.apache.hyracks.data.std.api.IValueReference; -final class TextualExternalFilePrinter implements IExternalPrinter { - private final IPrinter printer; - private final IExternalFileCompressStreamFactory compressStreamFactory; - private TextualOutputStreamDelegate delegate; - private PrintStream printStream; +final class TextualExternalFilePrinter extends AbstractTextualExternalPrinter { TextualExternalFilePrinter(IPrinter printer, IExternalFileCompressStreamFactory compressStreamFactory) { - this.printer = printer; - this.compressStreamFactory = compressStreamFactory; + super(printer, compressStreamFactory); } @Override - public void open() throws HyracksDataException { - printer.init(); - } - - @Override - public void newStream(OutputStream outputStream) throws HyracksDataException { - if (printStream != null) { - close(); - } - delegate = new TextualOutputStreamDelegate(compressStreamFactory.createStream(outputStream)); - printStream = new PrintStream(delegate); - } - - @Override - public void print(IValueReference value) throws HyracksDataException { - printer.print(value.getByteArray(), value.getStartOffset(), value.getLength(), printStream); + void afterPrint() { printStream.println(); - delegate.checkError(); - } - - @Override - public void close() throws HyracksDataException { - if (printStream != null) { - printStream.close(); - printStream = null; - delegate.checkError(); - delegate = null; - } } } diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/TextualExternalFilePrinterFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/TextualExternalFilePrinterFactory.java index e3d0a66..6bee7d7 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/TextualExternalFilePrinterFactory.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/TextualExternalFilePrinterFactory.java @@ -20,15 +20,17 @@ import org.apache.asterix.external.writer.compressor.IExternalFileCompressStreamFactory; import org.apache.asterix.runtime.writer.IExternalPrinter; +import org.apache.asterix.runtime.writer.IExternalPrinterFactory; import org.apache.hyracks.algebricks.data.IPrinterFactory; -public class TextualExternalFilePrinterFactory extends TextualExternalPrinterFactory { +public class TextualExternalFilePrinterFactory implements IExternalPrinterFactory { private static final long serialVersionUID = 8971234908711234L; + protected final IPrinterFactory printerFactory; private final IExternalFileCompressStreamFactory compressStreamFactory; public TextualExternalFilePrinterFactory(IPrinterFactory printerFactory, IExternalFileCompressStreamFactory compressStreamFactory) { - super(printerFactory); + this.printerFactory = printerFactory; this.compressStreamFactory = compressStreamFactory; } diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java index 340eb0a..416e32f 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java @@ -771,8 +771,8 @@ @Override public Pair<IPushRuntimeFactory, AlgebricksPartitionConstraint> getWriteDatabaseWithKeyRuntime(int sourceColumn, - IScalarEvaluatorFactory[] keyEvaluatorFactories, boolean autogenerated, IWriteDataSink sink, - RecordDescriptor inputDesc, Object sourceType) throws AlgebricksException { + IScalarEvaluatorFactory[] keyEvaluatorFactories, IWriteDataSink sink, RecordDescriptor inputDesc, + Object sourceType) throws AlgebricksException { throw new UnsupportedOperationException(); } diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java index 2d77a94..9253a48 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java @@ -30,7 +30,7 @@ import org.apache.asterix.external.writer.compressor.NoOpExternalFileCompressStreamFactory; import org.apache.asterix.external.writer.printer.TextualExternalFilePrinterFactory; import org.apache.asterix.formats.nontagged.CleanJSONPrinterFactoryProvider; -import org.apache.asterix.runtime.writer.ExternalWriterConfiguration; +import org.apache.asterix.runtime.writer.ExternalFileWriterConfiguration; import org.apache.asterix.runtime.writer.IExternalFileWriterFactory; import org.apache.asterix.runtime.writer.IExternalFileWriterFactoryProvider; import org.apache.asterix.runtime.writer.IExternalPrinterFactory; @@ -83,12 +83,12 @@ return Integer.parseInt(maxResultString); } - private static ExternalWriterConfiguration createConfiguration(ICcApplicationContext appCtx, IWriteDataSink sink, - String staticPath, SourceLocation pathExpressionLocation) { + private static ExternalFileWriterConfiguration createConfiguration(ICcApplicationContext appCtx, + IWriteDataSink sink, String staticPath, SourceLocation pathExpressionLocation) { Map<String, String> params = sink.getConfiguration(); boolean singleNodeCluster = isSingleNodeCluster(appCtx); - return new ExternalWriterConfiguration(params, pathExpressionLocation, staticPath, singleNodeCluster); + return new ExternalFileWriterConfiguration(params, pathExpressionLocation, staticPath, singleNodeCluster); } private static boolean isSingleNodeCluster(ICcApplicationContext appCtx) { diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/ExternalWriterConfiguration.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/ExternalFileWriterConfiguration.java similarity index 91% rename from asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/ExternalWriterConfiguration.java rename to asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/ExternalFileWriterConfiguration.java index fbb05ee..b62a07a 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/ExternalWriterConfiguration.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/ExternalFileWriterConfiguration.java @@ -22,13 +22,13 @@ import org.apache.hyracks.api.exceptions.SourceLocation; -public final class ExternalWriterConfiguration { +public final class ExternalFileWriterConfiguration { private final Map<String, String> configuration; private final SourceLocation pathSourceLocation; private final String staticPath; private final boolean singleNodeCluster; - public ExternalWriterConfiguration(Map<String, String> configuration, SourceLocation pathSourceLocation, + public ExternalFileWriterConfiguration(Map<String, String> configuration, SourceLocation pathSourceLocation, String staticPath, boolean singleNodeCluster) { this.configuration = configuration; this.pathSourceLocation = pathSourceLocation; diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/IExternalFileWriterFactoryProvider.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/IExternalFileWriterFactoryProvider.java index eabd2cb..5805f08 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/IExternalFileWriterFactoryProvider.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/writer/IExternalFileWriterFactoryProvider.java @@ -19,7 +19,7 @@ package org.apache.asterix.runtime.writer; public interface IExternalFileWriterFactoryProvider { - IExternalFileWriterFactory create(ExternalWriterConfiguration configuration); + IExternalFileWriterFactory create(ExternalFileWriterConfiguration configuration); char getSeparator(); } diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java index 5240e0c..11c8b81 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java @@ -66,8 +66,8 @@ throws AlgebricksException; Pair<IPushRuntimeFactory, AlgebricksPartitionConstraint> getWriteDatabaseWithKeyRuntime(int sourceColumn, - IScalarEvaluatorFactory[] keyEvaluatorFactories, boolean autogenerated, IWriteDataSink sink, - RecordDescriptor inputDesc, Object sourceType) throws AlgebricksException; + IScalarEvaluatorFactory[] keyEvaluatorFactories, IWriteDataSink sink, RecordDescriptor inputDesc, + Object sourceType) throws AlgebricksException; Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getResultHandleRuntime(IDataSink sink, int[] printColumns, IPrinterFactory[] printerFactories, IAWriterFactory writerFactory, diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/SinkWritePOperator.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/SinkWritePOperator.java index 71f5876..9c1879f 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/SinkWritePOperator.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/SinkWritePOperator.java @@ -62,16 +62,12 @@ private final LogicalVariable sourceVariable; private final List<LogicalVariable> partitionVariables; private final List<OrderColumn> orderColumns; - private final List<LogicalVariable> keyVariables; - private final boolean autogenerated; public SinkWritePOperator(LogicalVariable sourceVariable, List<LogicalVariable> partitionVariables, - List<OrderColumn> orderColumns, List<LogicalVariable> keyVariables, boolean autogenerated) { + List<OrderColumn> orderColumns) { this.sourceVariable = sourceVariable; this.partitionVariables = partitionVariables; this.orderColumns = orderColumns; - this.keyVariables = keyVariables; - this.autogenerated = autogenerated; } @Override @@ -174,8 +170,8 @@ typeEnv.getVarType(sourceVariable)); } else { - runtimeAndConstraints = mp.getWriteDatabaseWithKeyRuntime(sourceColumn, keyEvalFactories, autogenerated, - writeDataSink, inputDesc, typeEnv.getVarType(sourceVariable)); + runtimeAndConstraints = mp.getWriteDatabaseWithKeyRuntime(sourceColumn, keyEvalFactories, writeDataSink, + inputDesc, typeEnv.getVarType(sourceVariable)); } IPushRuntimeFactory runtime = runtimeAndConstraints.first; diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java index 3ea2631..9fd4cd9 100644 --- a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java +++ b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java @@ -409,8 +409,7 @@ ensureAllVariables(op.getPartitionExpressions(), v -> v); ensureAllVariables(op.getOrderExpressions(), Pair::getSecond); ensureAllVariables(op.getKeyExpressions(), v -> v); - return new SinkWritePOperator(op.getSourceVariable(), op.getPartitionVariables(), op.getOrderColumns(), - op.getKeyVariables(), op.getAutogenerated()); + return new SinkWritePOperator(op.getSourceVariable(), op.getPartitionVariables(), op.getOrderColumns()); } @Override -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18197 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I6e511e2ec13edf7458a809826860124507a51038 Gerrit-Change-Number: 18197 Gerrit-PatchSet: 2 Gerrit-Owner: Hussain Towaileb <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Hussain Towaileb <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Wail Alkowaileet <[email protected]> Gerrit-MessageType: merged
