[
https://issues.apache.org/jira/browse/DRILL-3423?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14649894#comment-14649894
]
Niels Basjes commented on DRILL-3423:
-------------------------------------
I took a quick stab at the addAsParseTarget method (didn't even run it or test
this) to show you how to use the casting options that the parser has built in.
Effectively I would move the mappings out into a property file and then do{code}
EnumSet<Casts> casts = parser.getCasts(path);
if (casts.contains(Casts.DOUBLE)) {
add(parser, path,
writer.rootAsMap().map(mapping.key()).float8(mapping.value()));
} else if (casts.contains(Casts.LONG)) {
add(parser, path,
writer.rootAsMap().map(mapping.key()).bigInt(mapping.value()));
} else {
add(parser, path,
writer.rootAsMap().map(mapping.key()).varChar(mapping.value()));
}
{code}
Here is the entire snippet I hacked at
{code} private void addParseTargetMapping(Map<String,Pair<String,String>>
mappings, String path, String map, String name){
mappings.put(path, new Pair<String,String>(map, name));
}
public void addAsParseTarget(Parser<ComplexWriterFacade> parser) {
try {
Map<String,Pair<String,String>> mappings = new HashMap<>();
// TODO: Move this to a property file
addParseTargetMapping(mappings, "NUMBER:connection.keepalivecount",
"client","keepalivecount");
addParseTargetMapping(mappings, "NUMBER:connection.client.logname"
,"request","logname");
addParseTargetMapping(mappings, "STRING:request.errorlogid"
,"request","errorlogid");
addParseTargetMapping(mappings, "HTTP.METHOD:request.method"
,"request","method");
addParseTargetMapping(mappings, "PORT:request.server.port.canonical"
,"server","canonical_port");
addParseTargetMapping(mappings,
"PORT:connection.server.port.canonical" ,"server","canonical_port");
addParseTargetMapping(mappings, "PORT:connection.client.port"
,"client","port");
addParseTargetMapping(mappings,
"NUMBER:connection.server.child.processid" ,"server","process_id");
addParseTargetMapping(mappings,
"NUMBER:connection.server.child.threadid" ,"server","thread_id");
addParseTargetMapping(mappings,
"STRING:connection.server.child.hexthreadid" ,"connection","hex_thread_id");
addParseTargetMapping(mappings,
"HTTP.QUERYSTRING:request.querystring" ,"","");
addParseTargetMapping(mappings, "HTTP.FIRSTLINE:request.firstline"
,"","");
addParseTargetMapping(mappings, "STRING:request.handler"
,"request","handler");
addParseTargetMapping(mappings, "STRING:request.status.original"
,"request","status_original");
addParseTargetMapping(mappings, "STRING:request.status.last"
,"request","status_last");
addParseTargetMapping(mappings, "TIME.STAMP:request.receive.time"
,"request","timestamp");
addParseTargetMapping(mappings,
"TIME.EPOCH:request.receive.time.begin.msec" ,"request","begin_msec");
addParseTargetMapping(mappings,
"TIME.EPOCH:request.receive.time.end.msec" ,"request","end_msec");
addParseTargetMapping(mappings,
"TIME.EPOCH.USEC:request.receive.time.begin.usec" ,"request","begin_usec");
addParseTargetMapping(mappings,
"TIME.EPOCH.USEC:request.receive.time.end.usec" ,"request","end_usec");
addParseTargetMapping(mappings,
"TIME.EPOCH:request.receive.time.begin.msec_frac" ,"request","begin_msec_frac");
addParseTargetMapping(mappings,
"TIME.EPOCH:request.receive.time.end.msec_frac" ,"request","end_msec_frac");
addParseTargetMapping(mappings,
"TIME.EPOCH.USEC_FRAC:request.receive.time.begin.usec_frac"
,"request","begin_usec_frac");
addParseTargetMapping(mappings,
"TIME.EPOCH.USEC_FRAC:request.receive.time.end.usec_frac"
,"request","end_usec_frac");
addParseTargetMapping(mappings,
"SECONDS:response.server.processing.time" ,"response","processing_time");
addParseTargetMapping(mappings, "STRING:connection.client.user"
,"client","user");
addParseTargetMapping(mappings, "URI:request.urlpath"
,"request","url");
addParseTargetMapping(mappings,
"STRING:connection.server.name.canonical" ,"server","canonical_name");
addParseTargetMapping(mappings, "STRING:connection.server.name"
,"server","name");
addParseTargetMapping(mappings,
"HTTP.CONNECTSTATUS:response.connection.status"
,"response","connection_status");
addParseTargetMapping(mappings, "BYTES:request.bytes"
,"request","bytes");
addParseTargetMapping(mappings, "BYTES:response.bytes"
,"response","bytes");
addParseTargetMapping(mappings, "HTTP.COOKIES:request.cookies"
,"request","cookies");
addParseTargetMapping(mappings, "HTTP.SETCOOKIES:response.cookies"
,"response","cookies");
addParseTargetMapping(mappings, "HTTP.USERAGENT:request.user-agent"
,"request","useragent");
addParseTargetMapping(mappings, "HTTP.URI:request.referer"
,"request","referer");
addParseTargetMapping(mappings, "HTTP.METHOD:method"
,"request","method");
addParseTargetMapping(mappings, "HTTP.URI:uri" ,"request","uri");
addParseTargetMapping(mappings, "HTTP.PROTOCOL:protocol"
,"request","protocol");
addParseTargetMapping(mappings,
"HTTP.PROTOCOL.VERSION:protocol.version" ,"request","protocol_version");
addParseTargetMapping(mappings,
"HTTP.METHOD:request.firstline.method" ,"request","method");
addParseTargetMapping(mappings, "HTTP.URI:request.firstline.uri"
,"request","uri");
addParseTargetMapping(mappings,
"HTTP.PROTOCOL:request.firstline.protocol" ,"request","protocol");
addParseTargetMapping(mappings,
"HTTP.PROTOCOL.VERSION:request.firstline.protocol.version"
,"request","protocol_version");
for (final String path : parser.getPossiblePaths()) {
EnumSet<Casts> casts = parser.getCasts(path);
Pair<String, String> mapping = mappings.get(path);
if (mapping == null) {
final String noPeriodPath = path.replace(".", "_");
if (casts.contains(Casts.DOUBLE)) {
add(parser, path, writer.rootAsMap().float8(noPeriodPath));
} else if (casts.contains(Casts.LONG)) {
add(parser, path, writer.rootAsMap().bigInt(noPeriodPath));
} else {
add(parser, path, writer.rootAsMap().varChar(noPeriodPath));
}
} else {
if (casts.contains(Casts.DOUBLE)) {
add(parser, path,
writer.rootAsMap().map(mapping.key()).float8(mapping.value()));
} else if (casts.contains(Casts.LONG)) {
add(parser, path,
writer.rootAsMap().map(mapping.key()).bigInt(mapping.value()));
} else {
add(parser, path,
writer.rootAsMap().map(mapping.key()).varChar(mapping.value()));
}
}
}
} catch (MissingDissectorsException | SecurityException |
NoSuchMethodException | InvalidDissectorException e) {
throw handleAndGenerate("Failure while setting up log mappings.", e);
}
}
}
}
{code}
> Add New HTTPD format plugin
> ---------------------------
>
> Key: DRILL-3423
> URL: https://issues.apache.org/jira/browse/DRILL-3423
> Project: Apache Drill
> Issue Type: New Feature
> Components: Storage - Other
> Reporter: Jacques Nadeau
> Assignee: Jacques Nadeau
> Fix For: 1.2.0
>
>
> Add an HTTPD logparser based format plugin. The author has been kind enough
> to move the logparser project to be released under the Apache License. Can
> find it here:
> <dependency>
> <groupId>nl.basjes.parse.httpdlog</groupId>
> <artifactId>httpdlog-parser</artifactId>
> <version>2.0</version>
> </dependency>
>
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
