This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 66aaf4151cba49a6fdb2c535911f6b75c7083ce3 Author: Ali Alsuliman <[email protected]> AuthorDate: Tue Dec 10 00:39:06 2019 -0800 [ASTERIXDB-2686][RT] Delete run files eagerly - user model changes: no - storage format changes: no - interface changes: no Details: Delete run files eagerly. Change-Id: Idc780b44b0554bf36e2378700aae1b38b8e57353 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/4403 Tested-by: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Reviewed-by: Till Westmann <[email protected]> --- .../join-ASTERIXDB-2686.1.ddl.sqlpp | 16 ++++++++++------ .../join-ASTERIXDB-2686.2.update.sqlpp | 11 +++++------ .../join-ASTERIXDB-2686.3.query.sqlpp | 10 ++++------ .../join-ASTERIXDB-2686.4.ddl.sqlpp | 9 +-------- .../misc/join-ASTERIXDB-2686/join-ASTERIXDB-2686.1.adm | 1 + .../src/test/resources/runtimets/testsuite_sqlpp.xml | 5 +++++ .../apache/hyracks/api/io/IWorkspaceFileFactory.java | 16 ++++++++++++++++ .../ExternalGroupWriteOperatorNodePushable.java | 3 +-- .../OptimizedHybridHashJoinOperatorDescriptor.java | 18 ++++++++++++++++++ .../std/sort/AbstractExternalSortRunMerger.java | 2 +- 10 files changed, 62 insertions(+), 29 deletions(-) diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IWorkspaceFileFactory.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/join-ASTERIXDB-2686/join-ASTERIXDB-2686.1.ddl.sqlpp similarity index 68% copy from hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IWorkspaceFileFactory.java copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/join-ASTERIXDB-2686/join-ASTERIXDB-2686.1.ddl.sqlpp index 9a35bb8..bafc59a 100644 --- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IWorkspaceFileFactory.java +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/join-ASTERIXDB-2686/join-ASTERIXDB-2686.1.ddl.sqlpp @@ -16,12 +16,16 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.hyracks.api.io; -import org.apache.hyracks.api.exceptions.HyracksDataException; +/* + * Description : Test that temporary run files generated are deleted as soon as possible and query runs fine + * Expected Result : Success + */ -public interface IWorkspaceFileFactory { - public FileReference createUnmanagedWorkspaceFile(String prefix) throws HyracksDataException; +drop dataverse test if exists; +create dataverse test; +use test; - public FileReference createManagedWorkspaceFile(String prefix) throws HyracksDataException; -} +create type openType as open {id: uuid}; +create dataset ds1(openType) primary key id autogenerated; +create dataset ds2(openType) primary key id autogenerated; \ No newline at end of file diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IWorkspaceFileFactory.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/join-ASTERIXDB-2686/join-ASTERIXDB-2686.2.update.sqlpp similarity index 70% copy from hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IWorkspaceFileFactory.java copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/join-ASTERIXDB-2686/join-ASTERIXDB-2686.2.update.sqlpp index 9a35bb8..9a695ff 100644 --- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IWorkspaceFileFactory.java +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/join-ASTERIXDB-2686/join-ASTERIXDB-2686.2.update.sqlpp @@ -16,12 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.hyracks.api.io; -import org.apache.hyracks.api.exceptions.HyracksDataException; +use test; -public interface IWorkspaceFileFactory { - public FileReference createUnmanagedWorkspaceFile(String prefix) throws HyracksDataException; +insert into ds1(from range(0,1000000) as data select value {"f1": 1}); +insert into ds2(from range(0,1000000) as data select value {"f1": 2}); - public FileReference createManagedWorkspaceFile(String prefix) throws HyracksDataException; -} +insert into ds1 {"f1": 3}; +insert into ds2 {"f1": 3}; \ No newline at end of file diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IWorkspaceFileFactory.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/join-ASTERIXDB-2686/join-ASTERIXDB-2686.3.query.sqlpp similarity index 70% copy from hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IWorkspaceFileFactory.java copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/join-ASTERIXDB-2686/join-ASTERIXDB-2686.3.query.sqlpp index 9a35bb8..f668266 100644 --- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IWorkspaceFileFactory.java +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/join-ASTERIXDB-2686/join-ASTERIXDB-2686.3.query.sqlpp @@ -16,12 +16,10 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.hyracks.api.io; -import org.apache.hyracks.api.exceptions.HyracksDataException; +use test; -public interface IWorkspaceFileFactory { - public FileReference createUnmanagedWorkspaceFile(String prefix) throws HyracksDataException; +set `compiler.joinmemory` "180KB"; - public FileReference createManagedWorkspaceFile(String prefix) throws HyracksDataException; -} +from ds1 join ds2 on ds1.f1 = ds2.f1 +select ds1.f1; \ No newline at end of file diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IWorkspaceFileFactory.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/join-ASTERIXDB-2686/join-ASTERIXDB-2686.4.ddl.sqlpp similarity index 70% copy from hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IWorkspaceFileFactory.java copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/join-ASTERIXDB-2686/join-ASTERIXDB-2686.4.ddl.sqlpp index 9a35bb8..548e632 100644 --- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IWorkspaceFileFactory.java +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/join-ASTERIXDB-2686/join-ASTERIXDB-2686.4.ddl.sqlpp @@ -16,12 +16,5 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.hyracks.api.io; -import org.apache.hyracks.api.exceptions.HyracksDataException; - -public interface IWorkspaceFileFactory { - public FileReference createUnmanagedWorkspaceFile(String prefix) throws HyracksDataException; - - public FileReference createManagedWorkspaceFile(String prefix) throws HyracksDataException; -} +drop dataverse test if exists; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/join-ASTERIXDB-2686/join-ASTERIXDB-2686.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/join-ASTERIXDB-2686/join-ASTERIXDB-2686.1.adm new file mode 100644 index 0000000..fca24df --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/join-ASTERIXDB-2686/join-ASTERIXDB-2686.1.adm @@ -0,0 +1 @@ +{ "f1": 3 } \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml index 929ecd5..817bab6 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml @@ -3801,6 +3801,11 @@ </compilation-unit> </test-case> <test-case FilePath="misc"> + <compilation-unit name="join-ASTERIXDB-2686"> + <output-dir compare="Text">join-ASTERIXDB-2686</output-dir> + </compilation-unit> + </test-case> + <test-case FilePath="misc"> <compilation-unit name="poll-dynamic"> <output-dir compare="Text">poll-dynamic</output-dir> </compilation-unit> diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IWorkspaceFileFactory.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IWorkspaceFileFactory.java index 9a35bb8..6859ec7 100644 --- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IWorkspaceFileFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IWorkspaceFileFactory.java @@ -18,10 +18,26 @@ */ package org.apache.hyracks.api.io; +import org.apache.hyracks.api.context.IHyracksJobletContext; +import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.exceptions.HyracksDataException; public interface IWorkspaceFileFactory { + + /** + * Creates an unmanaged file. Unmanaged files are not automatically deleted. The caller has to delete them. + * + * @param prefix a meaningful string to start the file name with. + */ public FileReference createUnmanagedWorkspaceFile(String prefix) throws HyracksDataException; + /** + * Creates a managed file. Managed files are automatically deleted at a certain point. This depends on which + * context the file is created in. Files created in the {@link IHyracksJobletContext job context} will be deleted + * as soon as the job is done. Files created in the {@link IHyracksTaskContext task context} will be deleted as + * soon as the task is done. + * + * @param prefix a meaningful string to start the file name with. + */ public FileReference createManagedWorkspaceFile(String prefix) throws HyracksDataException; } diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/group/external/ExternalGroupWriteOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/group/external/ExternalGroupWriteOperatorNodePushable.java index 999dbac..e8a1b76 100644 --- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/group/external/ExternalGroupWriteOperatorNodePushable.java +++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/group/external/ExternalGroupWriteOperatorNodePushable.java @@ -171,8 +171,7 @@ public class ExternalGroupWriteOperatorNodePushable extends AbstractUnaryOutputS @Override public RunFileWriter getRunFileWriter() throws HyracksDataException { - FileReference newRun = ctx.getJobletContext() - .createManagedWorkspaceFile(ExternalGroupOperatorDescriptor.class.getSimpleName()); + FileReference newRun = ctx.createManagedWorkspaceFile(ExternalGroupOperatorDescriptor.class.getSimpleName()); return new RunFileWriter(newRun, ctx.getIoManager()); } } diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java index e7984d6..6ba03c5 100644 --- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java +++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java @@ -442,6 +442,12 @@ public class OptimizedHybridHashJoinOperatorDescriptor extends AbstractOperatorD if (isLeftOuter && pReader != null) { appendNullToProbeTuples(pReader); } + if (bReader != null) { + bReader.close(); + } + if (pReader != null) { + pReader.close(); + } continue; } int bSize = state.hybridHJ.getBuildPartitionSizeInTup(pid); @@ -637,6 +643,12 @@ public class OptimizedHybridHashJoinOperatorDescriptor extends AbstractOperatorD // For the outer join, we don't reverse the role. appendNullToProbeTuples(rprfw); } + if (rbrfw != null) { + rbrfw.close(); + } + if (rprfw != null) { + rprfw.close(); + } continue; } @@ -661,6 +673,12 @@ public class OptimizedHybridHashJoinOperatorDescriptor extends AbstractOperatorD // For the outer join, we don't reverse the role. appendNullToProbeTuples(rprfw); } + if (rbrfw != null) { + rbrfw.close(); + } + if (rprfw != null) { + rprfw.close(); + } continue; } diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/AbstractExternalSortRunMerger.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/AbstractExternalSortRunMerger.java index 4a77b3c..f3e9320 100644 --- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/AbstractExternalSortRunMerger.java +++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/AbstractExternalSortRunMerger.java @@ -142,7 +142,7 @@ public abstract class AbstractExternalSortRunMerger { } finally { mergeResultWriter.close(); } - reader = mergeFileWriter.createReader(); + reader = mergeFileWriter.createDeleteOnCloseReader(); } runs.add(reader);
