[
https://issues.apache.org/jira/browse/DRILL-8167?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17505998#comment-17505998
]
ASF GitHub Bot commented on DRILL-8167:
---------------------------------------
jnturton commented on a change in pull request #2494:
URL: https://github.com/apache/drill/pull/2494#discussion_r825601125
##########
File path:
exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader.java
##########
@@ -124,19 +139,76 @@ private JSONRecordReader(FragmentContext fragmentContext,
Path inputPath, JsonNo
this.embeddedContent = embeddedContent;
}
+ // If the config is null, create a temporary one with the global options.
+ if (config == null) {
+ this.config = new JSONFormatConfig(null,
+ embeddedContent == null &&
fragmentContext.getOptions().getOption(ExecConstants.JSON_READER_ALL_TEXT_MODE_VALIDATOR),
+ embeddedContent == null &&
fragmentContext.getOptions().getOption(ExecConstants.JSON_READ_NUMBERS_AS_DOUBLE_VALIDATOR),
+
fragmentContext.getOptions().getOption(ExecConstants.JSON_SKIP_MALFORMED_RECORDS_VALIDATOR),
+
fragmentContext.getOptions().getOption(ExecConstants.JSON_READER_ESCAPE_ANY_CHAR_VALIDATOR),
+
fragmentContext.getOptions().getOption(ExecConstants.JSON_READER_NAN_INF_NUMBERS_VALIDATOR));
+ } else {
+ this.config = config;
+ }
+
this.fileSystem = fileSystem;
this.fragmentContext = fragmentContext;
- // only enable all text mode if we aren't using embedded content mode.
- this.enableAllTextMode = embeddedContent == null &&
fragmentContext.getOptions().getOption(ExecConstants.JSON_READER_ALL_TEXT_MODE_VALIDATOR);
- this.enableNanInf =
fragmentContext.getOptions().getOption(ExecConstants.JSON_READER_NAN_INF_NUMBERS_VALIDATOR);
- this.enableEscapeAnyChar =
fragmentContext.getOptions().getOption(ExecConstants.JSON_READER_ESCAPE_ANY_CHAR_VALIDATOR);
- this.readNumbersAsDouble = embeddedContent == null &&
fragmentContext.getOptions().getOption(ExecConstants.JSON_READ_NUMBERS_AS_DOUBLE_VALIDATOR);
+
+ this.enableAllTextMode = allTextMode();
+ this.enableNanInf = nanInf();
+ this.enableEscapeAnyChar = escapeAnyChar();
+ this.readNumbersAsDouble = readNumbersAsDouble();
this.unionEnabled = embeddedContent == null &&
fragmentContext.getOptions().getBoolean(ExecConstants.ENABLE_UNION_TYPE_KEY);
- this.skipMalformedJSONRecords =
fragmentContext.getOptions().getOption(ExecConstants.JSON_SKIP_MALFORMED_RECORDS_VALIDATOR);
+ this.skipMalformedJSONRecords = skipMalformedJSONRecords();
this.printSkippedMalformedJSONRecordLineNumber =
fragmentContext.getOptions().getOption(ExecConstants.JSON_READER_PRINT_INVALID_RECORDS_LINE_NOS_FLAG_VALIDATOR);
setColumns(columns);
}
+ /**
+ * Returns the value of the all text mode. Values set in the format config
will override global values.
+ * @return The value of allTextMode
+ */
+ private boolean allTextMode() {
+ // only enable all text mode if we aren't using embedded content mode.
+ if (config.getAllTextMode() == null) {
+ return embeddedContent == null &&
fragmentContext.getOptions().getOption(ExecConstants.JSON_READER_ALL_TEXT_MODE_VALIDATOR);
Review comment:
Please ensure that we preserve the existing option scope precedence:
SYSTEM < FORMAT < SESSION < QUERY.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
> Add JSON Config Options to Format Config
> ----------------------------------------
>
> Key: DRILL-8167
> URL: https://issues.apache.org/jira/browse/DRILL-8167
> Project: Apache Drill
> Issue Type: Improvement
> Components: Storage - JSON
> Affects Versions: 1.20.0
> Reporter: Charles Givre
> Assignee: Charles Givre
> Priority: Major
> Fix For: Future
>
>
> Most all Drill format plugins allow the user to configure various options for
> that plugin as part of the format config. The one glaring exception is the
> JSON reader which has several configuration options which can only be set
> globally. This PR moves these to the format config so that users can set
> these options when they configure a storage plugin.
> This PR does not eliminate the global settings for JSON. It simply adds
> another place where a user can update the settings. If the settings in the
> config file are not defined (`null`) Drill will use the global settings.
--
This message was sent by Atlassian Jira
(v8.20.1#820001)
