This is an automated email from the ASF dual-hosted git repository.
dzamo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new 30104d9 DRILL-8118: Add Option to Allow Disk Use on Mongo Queries
(#2442)
30104d9 is described below
commit 30104d9faf3cf9fe3fcae4d109dc28f20f27f915
Author: Charles S. Givre <[email protected]>
AuthorDate: Sun Jan 30 06:26:39 2022 -0500
DRILL-8118: Add Option to Allow Disk Use on Mongo Queries (#2442)
* Initial commit
* Update
contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoRecordReader.java
Co-authored-by: luoc <[email protected]>
Co-authored-by: luoc <[email protected]>
---
.../java/org/apache/drill/exec/store/mongo/MongoRecordReader.java | 6 +++++-
.../apache/drill/exec/store/mongo/MongoStoragePluginConfig.java | 8 ++++++++
.../src/main/resources/bootstrap-storage-plugins.json | 1 +
.../java/org/apache/drill/exec/store/mongo/MongoTestBase.java | 2 +-
.../store/mongo/TestMongoStoragePluginUsesCredentialsStore.java | 2 +-
5 files changed, 16 insertions(+), 3 deletions(-)
diff --git
a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoRecordReader.java
b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoRecordReader.java
index 08219f6..93b2d16 100644
---
a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoRecordReader.java
+++
b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoRecordReader.java
@@ -198,7 +198,11 @@ public class MongoRecordReader extends
AbstractRecordReader {
if (!fields.isEmpty()) {
operations.add(Aggregates.project(fields));
}
- projection = collection.aggregate(operations);
+ if (plugin.getConfig().allowDiskUse()) {
+ projection = collection.aggregate(operations).allowDiskUse(true);
+ } else {
+ projection = collection.aggregate(operations);
+ }
} else {
projection = collection.find(filters).projection(fields);
}
diff --git
a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoStoragePluginConfig.java
b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoStoragePluginConfig.java
index be410ae..e1da24b 100644
---
a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoStoragePluginConfig.java
+++
b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoStoragePluginConfig.java
@@ -40,6 +40,8 @@ public class MongoStoragePluginConfig extends
AbstractSecuredStoragePluginConfig
private final String connection;
+ private final boolean allowDiskUse;
+
@JsonIgnore
private final ConnectionString clientURI;
@@ -51,12 +53,14 @@ public class MongoStoragePluginConfig extends
AbstractSecuredStoragePluginConfig
public MongoStoragePluginConfig(@JsonProperty("connection") String
connection,
@JsonProperty("pluginOptimizations") MongoPluginOptimizations
pluginOptimizations,
@JsonProperty("batchSize") Integer batchSize,
+ @JsonProperty("allowDiskUse") Boolean allowDiskUse,
@JsonProperty("credentialsProvider") CredentialsProvider
credentialsProvider) {
super(getCredentialsProvider(credentialsProvider), credentialsProvider ==
null);
this.connection = connection;
this.clientURI = new ConnectionString(connection);
this.pluginOptimizations = ObjectUtils.defaultIfNull(pluginOptimizations,
new MongoPluginOptimizations());
this.batchSize = batchSize != null ? batchSize : 100;
+ this.allowDiskUse = allowDiskUse != null && allowDiskUse;
}
public MongoPluginOptimizations getPluginOptimizations() {
@@ -76,6 +80,10 @@ public class MongoStoragePluginConfig extends
AbstractSecuredStoragePluginConfig
return batchSize;
}
+ public boolean allowDiskUse() {
+ return allowDiskUse;
+ }
+
private static CredentialsProvider
getCredentialsProvider(CredentialsProvider credentialsProvider) {
return credentialsProvider != null ? credentialsProvider :
PlainCredentialsProvider.EMPTY_CREDENTIALS_PROVIDER;
}
diff --git
a/contrib/storage-mongo/src/main/resources/bootstrap-storage-plugins.json
b/contrib/storage-mongo/src/main/resources/bootstrap-storage-plugins.json
index 2010e32..7e3cd42 100644
--- a/contrib/storage-mongo/src/main/resources/bootstrap-storage-plugins.json
+++ b/contrib/storage-mongo/src/main/resources/bootstrap-storage-plugins.json
@@ -3,6 +3,7 @@
"mongo" : {
"type":"mongo",
"connection":"mongodb://localhost:27017/",
+ "allowDiskUse": false,
"enabled": false
}
}
diff --git
a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestBase.java
b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestBase.java
index 8224470..4b3cc1a 100644
---
a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestBase.java
+++
b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestBase.java
@@ -39,7 +39,7 @@ public class MongoTestBase extends ClusterTest implements
MongoTestConstants {
private static void initMongoStoragePlugin(String connectionURI) throws
Exception {
MongoStoragePluginConfig storagePluginConfig = new
MongoStoragePluginConfig(connectionURI,
- null, 100, PlainCredentialsProvider.EMPTY_CREDENTIALS_PROVIDER);
+ null, 100, false, PlainCredentialsProvider.EMPTY_CREDENTIALS_PROVIDER);
storagePluginConfig.setEnabled(true);
pluginRegistry.put(MongoStoragePluginConfig.NAME, storagePluginConfig);
diff --git
a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/TestMongoStoragePluginUsesCredentialsStore.java
b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/TestMongoStoragePluginUsesCredentialsStore.java
index 298744b..7a09abe 100644
---
a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/TestMongoStoragePluginUsesCredentialsStore.java
+++
b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/TestMongoStoragePluginUsesCredentialsStore.java
@@ -35,7 +35,7 @@ public class TestMongoStoragePluginUsesCredentialsStore
extends BaseTest {
private void test(String expectedUserName, String expectedPassword, String
connection, String name) throws ExecutionSetupException {
MongoStoragePlugin plugin = new MongoStoragePlugin(
- new MongoStoragePluginConfig(connection, null, 100,
PlainCredentialsProvider.EMPTY_CREDENTIALS_PROVIDER),
+ new MongoStoragePluginConfig(connection, null, 100, false,
PlainCredentialsProvider.EMPTY_CREDENTIALS_PROVIDER),
null, name);
MongoClientImpl client = (MongoClientImpl) plugin.getClient();
MongoCredential cred = client.getSettings().getCredential();
