>From Ian Maxon <[email protected]>:
Ian Maxon has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20835?usp=email )
Change subject: [ASTERIXDB-1888][RT] Make pre-sorted load not sort
......................................................................
[ASTERIXDB-1888][RT] Make pre-sorted load not sort
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
The hash merge connector is disabled by default because in certain
circumstances, it can cause a deadlock in large clusters. However,
for the application of LOAD, it can still be useful. It is better
to occasionally have to retry LOAD than it is to suffer with it
sorting enormous datasets for no good reason. So, in the case of
a LOAD statement with the pre-sorted keyword given, the proper
connector should be used.
Change-Id: I646539d88ea2bdc6da4fbf2b6e94e0a61c3125a9
---
M
asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
M
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-secondary-index-string-as-primary-key/scan-insert-inverted-index-ngram-secondary-index-string-as-primary-key.2.update.sqlpp
M
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index-string-as-primary-key/scan-insert-inverted-index-word-correlated-secondary-index-string-as-primary-key.2.update.sqlpp
M
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-secondary-index-string-as-primary-key/scan-insert-inverted-index-word-secondary-index-string-as-primary-key.2.update.sqlpp
M
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fuzzyjoin/dblp-string-as-primary-key/dblp-string-as-primary-key.2.update.sqlpp
M
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/union/union_opt_1/union_opt_1.2.update.sqlpp
M
hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EnforceStructuralPropertiesRule.java
7 files changed, 10 insertions(+), 7 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/35/20835/1
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
index 318c58e..949cb8f 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
@@ -19,6 +19,7 @@
package org.apache.asterix.translator;
import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
+import static
org.apache.hyracks.algebricks.rewriter.rules.EnforceStructuralPropertiesRule.HASH_MERGE;
import java.io.IOException;
import java.util.ArrayList;
@@ -290,6 +291,8 @@
orderColumns.add(new OrderColumn(pkVars.get(i),
OrderKind.ASC));
}
assign.setExplicitOrderingProperty(new
LocalOrderProperty(orderColumns));
+ //enable hash merge so the data doesn't have to be resorted
after partitioning
+ metadataProvider.getConfig().put(HASH_MERGE,"true");
}
}
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-secondary-index-string-as-primary-key/scan-insert-inverted-index-ngram-secondary-index-string-as-primary-key.2.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-secondary-index-string-as-primary-key/scan-insert-inverted-index-ngram-secondary-index-string-as-primary-key.2.update.sqlpp
index 826bcdb..cf06da5 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-secondary-index-string-as-primary-key/scan-insert-inverted-index-ngram-secondary-index-string-as-primary-key.2.update.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-ngram-secondary-index-string-as-primary-key/scan-insert-inverted-index-ngram-secondary-index-string-as-primary-key.2.update.sqlpp
@@ -26,5 +26,5 @@
use test;
-load dataset DBLP using localfs
((`path`=`asterix_nc1://data/dblp-small/dblp-small-id.txt`),(`format`=`delimited-text`),(`delimiter`=`:`))
pre-sorted;
+load dataset DBLP using localfs
((`path`=`asterix_nc1://data/dblp-small/dblp-small-id.txt`),(`format`=`delimited-text`),(`delimiter`=`:`));
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index-string-as-primary-key/scan-insert-inverted-index-word-correlated-secondary-index-string-as-primary-key.2.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index-string-as-primary-key/scan-insert-inverted-index-word-correlated-secondary-index-string-as-primary-key.2.update.sqlpp
index cf9a86a..407daa0 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index-string-as-primary-key/scan-insert-inverted-index-word-correlated-secondary-index-string-as-primary-key.2.update.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-correlated-secondary-index-string-as-primary-key/scan-insert-inverted-index-word-correlated-secondary-index-string-as-primary-key.2.update.sqlpp
@@ -21,7 +21,7 @@
use test;
-load dataset DBLP using localfs
((`path`=`asterix_nc1://data/dblp-small/dblp-small-id.txt`),(`format`=`delimited-text`),(`delimiter`=`:`))
pre-sorted;
+load dataset DBLP using localfs
((`path`=`asterix_nc1://data/dblp-small/dblp-small-id.txt`),(`format`=`delimited-text`),(`delimiter`=`:`));
insert into DBLP1
select element
{'id':o.id,'dblpid':o.dblpid,'title':o.title,'authors':o.authors,'misc':o.misc}
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-secondary-index-string-as-primary-key/scan-insert-inverted-index-word-secondary-index-string-as-primary-key.2.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-secondary-index-string-as-primary-key/scan-insert-inverted-index-word-secondary-index-string-as-primary-key.2.update.sqlpp
index b37beb2..7cd6b9d 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-secondary-index-string-as-primary-key/scan-insert-inverted-index-word-secondary-index-string-as-primary-key.2.update.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-insert-inverted-index-word-secondary-index-string-as-primary-key/scan-insert-inverted-index-word-secondary-index-string-as-primary-key.2.update.sqlpp
@@ -26,5 +26,5 @@
use test;
-load dataset DBLP using localfs
((`path`=`asterix_nc1://data/dblp-small/dblp-small-id.txt`),(`format`=`delimited-text`),(`delimiter`=`:`))
pre-sorted;
+load dataset DBLP using localfs
((`path`=`asterix_nc1://data/dblp-small/dblp-small-id.txt`),(`format`=`delimited-text`),(`delimiter`=`:`))
;
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fuzzyjoin/dblp-string-as-primary-key/dblp-string-as-primary-key.2.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fuzzyjoin/dblp-string-as-primary-key/dblp-string-as-primary-key.2.update.sqlpp
index 51c5bb1..1d91200 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fuzzyjoin/dblp-string-as-primary-key/dblp-string-as-primary-key.2.update.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/fuzzyjoin/dblp-string-as-primary-key/dblp-string-as-primary-key.2.update.sqlpp
@@ -19,5 +19,5 @@
use fuzzyjoin;
-load dataset DBLP using localfs
((`path`=`asterix_nc1://data/dblp-small/dblp-small-id.txt`),(`format`=`delimited-text`),(`delimiter`=`:`))
pre-sorted;
+load dataset DBLP using localfs
((`path`=`asterix_nc1://data/dblp-small/dblp-small-id.txt`),(`format`=`delimited-text`),(`delimiter`=`:`));
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/union/union_opt_1/union_opt_1.2.update.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/union/union_opt_1/union_opt_1.2.update.sqlpp
index 19d5778..1b8b12e 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/union/union_opt_1/union_opt_1.2.update.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/union/union_opt_1/union_opt_1.2.update.sqlpp
@@ -19,9 +19,9 @@
use test;
-load dataset onek1 using localfs
((`path`=`asterix_nc1://data/wisc/onektup.adm`),(`format`=`delimited-text`),(`delimiter`=`|`))
pre-sorted;
+load dataset onek1 using localfs
((`path`=`asterix_nc1://data/wisc/onektup.adm`),(`format`=`delimited-text`),(`delimiter`=`|`));
-load dataset onek2 using localfs
((`path`=`asterix_nc1://data/wisc/onektup.adm`),(`format`=`delimited-text`),(`delimiter`=`|`))
pre-sorted;
+load dataset onek2 using localfs
((`path`=`asterix_nc1://data/wisc/onektup.adm`),(`format`=`delimited-text`),(`delimiter`=`|`));
load dataset fbu1 using localfs
((`path`=`asterix_nc1://data/tinysocial/fbu.adm`),(`format`=`adm`));
diff --git
a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EnforceStructuralPropertiesRule.java
b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EnforceStructuralPropertiesRule.java
index f2ad988..add53d5 100644
---
a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EnforceStructuralPropertiesRule.java
+++
b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EnforceStructuralPropertiesRule.java
@@ -98,7 +98,7 @@
public class EnforceStructuralPropertiesRule implements IAlgebraicRewriteRule {
- private static final String HASH_MERGE = "hash_merge";
+ public static final String HASH_MERGE = "hash_merge";
private static final String TRUE_CONSTANT = "true";
private final FunctionIdentifier rangeMapFunction;
private final FunctionIdentifier localSamplingFun;
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20835?usp=email
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings?usp=email
Gerrit-MessageType: newchange
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: I646539d88ea2bdc6da4fbf2b6e94e0a61c3125a9
Gerrit-Change-Number: 20835
Gerrit-PatchSet: 1
Gerrit-Owner: Ian Maxon <[email protected]>
