Repository: asterixdb Updated Branches: refs/heads/master 7887e352b -> 31fb2e79f
[ASTERIXDB-2017][API] Access logs, log rotation - user model changes: no - storage format changes: no - interface changes: no Details: - Common Log Format (httpd) style logs for each servlet access. - Log rotation by default in asterix-server package, rather than console logging with no rotation Change-Id: I9a8d76bec308b2fdb20c33370fc9e58a154ba968 Reviewed-on: https://asterix-gerrit.ics.uci.edu/1915 Reviewed-by: Michael Blow <mb...@apache.org> Sonar-Qube: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Contrib: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/31fb2e79 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/31fb2e79 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/31fb2e79 Branch: refs/heads/master Commit: 31fb2e79fc00350cd2e60fc11ce87e3e1fa98617 Parents: 7887e35 Author: Ian Maxon <ima...@apache.org> Authored: Mon Aug 13 18:43:21 2018 -0700 Committer: Michael Blow <mb...@apache.org> Committed: Mon Aug 13 23:32:40 2018 -0700 ---------------------------------------------------------------------- .../asterix-app/src/main/resources/cc.conf | 1 + .../asterix-app/src/main/resources/cc2.conf | 1 + .../asterix-app/src/main/resources/cc3.conf | 1 + .../asterix-app/src/main/resources/cc4.conf | 1 + .../asterix-app/src/main/resources/log4j2.xml | 9 ++ .../asterix-app/src/test/resources/cc.conf | 1 + .../cluster_state_1/cluster_state_1.1.regexadm | 1 + .../cluster_state_1_full.1.regexadm | 1 + .../cluster_state_1_less.1.regexadm | 1 + .../src/main/opt/local/conf/cc.conf | 3 +- .../hyracks-control/hyracks-control-cc/pom.xml | 4 + .../org/apache/hyracks/control/cc/CCDriver.java | 10 ++ .../control/cc/CCLogConfigurationFactory.java | 89 ++++++++++++ .../common/controllers/ControllerConfig.java | 26 +++- .../hyracks-control/hyracks-control-nc/pom.xml | 4 + .../org/apache/hyracks/control/nc/NCDriver.java | 10 ++ .../control/nc/NCLogConfigurationFactory.java | 95 +++++++++++++ .../apache/hyracks/http/server/CLFLogger.java | 135 +++++++++++++++++++ .../apache/hyracks/http/server/HttpServer.java | 15 ++- .../http/server/HttpServerInitializer.java | 4 + 20 files changed, 404 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/31fb2e79/asterixdb/asterix-app/src/main/resources/cc.conf ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/resources/cc.conf b/asterixdb/asterix-app/src/main/resources/cc.conf index 0d9f54f..6971b2b 100644 --- a/asterixdb/asterix-app/src/main/resources/cc.conf +++ b/asterixdb/asterix-app/src/main/resources/cc.conf @@ -46,6 +46,7 @@ heartbeat.period=2000 heartbeat.max.misses=25 [common] +log.dir = logs/ log.level = INFO compiler.framesize=32KB compiler.sortmemory=320KB http://git-wip-us.apache.org/repos/asf/asterixdb/blob/31fb2e79/asterixdb/asterix-app/src/main/resources/cc2.conf ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/resources/cc2.conf b/asterixdb/asterix-app/src/main/resources/cc2.conf index ddf1438..65dbafc 100644 --- a/asterixdb/asterix-app/src/main/resources/cc2.conf +++ b/asterixdb/asterix-app/src/main/resources/cc2.conf @@ -46,6 +46,7 @@ heartbeat.period=2000 heartbeat.max.misses=25 [common] +log.dir = logs/ log.level = WARN compiler.framesize=32KB compiler.sortmemory=320KB http://git-wip-us.apache.org/repos/asf/asterixdb/blob/31fb2e79/asterixdb/asterix-app/src/main/resources/cc3.conf ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/resources/cc3.conf b/asterixdb/asterix-app/src/main/resources/cc3.conf index b819f24..20aa70d 100644 --- a/asterixdb/asterix-app/src/main/resources/cc3.conf +++ b/asterixdb/asterix-app/src/main/resources/cc3.conf @@ -46,6 +46,7 @@ heartbeat.period=2000 heartbeat.max.misses=25 [common] +log.dir = logs/ log.level = WARN compiler.framesize=32KB compiler.sortmemory=320KB http://git-wip-us.apache.org/repos/asf/asterixdb/blob/31fb2e79/asterixdb/asterix-app/src/main/resources/cc4.conf ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/resources/cc4.conf b/asterixdb/asterix-app/src/main/resources/cc4.conf index 3b7a993..5bdf8ea 100644 --- a/asterixdb/asterix-app/src/main/resources/cc4.conf +++ b/asterixdb/asterix-app/src/main/resources/cc4.conf @@ -44,6 +44,7 @@ heartbeat.period=2000 heartbeat.max.misses=25 [common] +log.dir = logs/ log.level = INFO compiler.framesize=32KB compiler.sortmemory=320KB http://git-wip-us.apache.org/repos/asf/asterixdb/blob/31fb2e79/asterixdb/asterix-app/src/main/resources/log4j2.xml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/resources/log4j2.xml b/asterixdb/asterix-app/src/main/resources/log4j2.xml index 1debf82..2ea8d4a 100644 --- a/asterixdb/asterix-app/src/main/resources/log4j2.xml +++ b/asterixdb/asterix-app/src/main/resources/log4j2.xml @@ -17,15 +17,24 @@ ! under the License. !--> <Configuration status="WARN"> + <CustomLevels> + <CustomLevel name="ACCESS" intLevel="550" /> + </CustomLevels> <Appenders> <Console name="Console" target="SYSTEM_OUT"> <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/> </Console> + <Console name="AccessLog" target="SYSTEM_OUT"> + <PatternLayout pattern="%m%n"/> + </Console> </Appenders> <Loggers> <Root level="WARN"> <AppenderRef ref="Console"/> </Root> <Logger name="org.apache.hyracks.control.nc.service" level="INFO"/> + <Logger name="org.apache.hyracks.http.server.CLFLogger" level="ACCESS" additivity="false"> + <AppenderRef ref="AccessLog"/> + </Logger> </Loggers> </Configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/31fb2e79/asterixdb/asterix-app/src/test/resources/cc.conf ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/cc.conf b/asterixdb/asterix-app/src/test/resources/cc.conf index fc95dd4..2694408 100644 --- a/asterixdb/asterix-app/src/test/resources/cc.conf +++ b/asterixdb/asterix-app/src/test/resources/cc.conf @@ -46,6 +46,7 @@ heartbeat.period=2000 heartbeat.max.misses=25 [common] +log.dir = logs/ log.level = INFO compiler.framesize=32KB compiler.sortmemory=320KB http://git-wip-us.apache.org/repos/asf/asterixdb/blob/31fb2e79/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm index ccb8c45..79db9ae 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm @@ -15,6 +15,7 @@ "compiler\.sortmemory" : 327680, "compiler\.textsearchmemory" : 163840, "default\.dir" : "target/io/dir/asterixdb", + "log\.dir" : "logs/", "log\.level" : "INFO", "max\.wait\.active\.cluster" : 60, "max.web.request.size" : 52428800, http://git-wip-us.apache.org/repos/asf/asterixdb/blob/31fb2e79/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm index 56fdf91..6e5547d 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm @@ -15,6 +15,7 @@ "compiler\.sortmemory" : 327680, "compiler\.textsearchmemory" : 163840, "default\.dir" : "target/io/dir/asterixdb", + "log\.dir" : "logs/", "log\.level" : "WARN", "max\.wait\.active\.cluster" : 60, "max.web.request.size" : 52428800, http://git-wip-us.apache.org/repos/asf/asterixdb/blob/31fb2e79/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm index 658f4b1..3237551 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm @@ -15,6 +15,7 @@ "compiler\.sortmemory" : 327680, "compiler\.textsearchmemory" : 163840, "default\.dir" : "target/io/dir/asterixdb", + "log\.dir" : "logs/", "log\.level" : "WARN", "max\.wait\.active\.cluster" : 60, "max.web.request.size" : 52428800, http://git-wip-us.apache.org/repos/asf/asterixdb/blob/31fb2e79/asterixdb/asterix-server/src/main/opt/local/conf/cc.conf ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-server/src/main/opt/local/conf/cc.conf b/asterixdb/asterix-server/src/main/opt/local/conf/cc.conf index 6fc0579..bd5f403 100644 --- a/asterixdb/asterix-server/src/main/opt/local/conf/cc.conf +++ b/asterixdb/asterix-server/src/main/opt/local/conf/cc.conf @@ -37,4 +37,5 @@ command=${NC_COMMAND} address = 127.0.0.1 [common] -log.level=INFO +log.dir = logs/ +log.level = INFO http://git-wip-us.apache.org/repos/asf/asterixdb/blob/31fb2e79/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/pom.xml ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/pom.xml b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/pom.xml index 6862582..86d6201 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/pom.xml @@ -97,5 +97,9 @@ <groupId>org.apache.logging.log4j</groupId> <artifactId>log4j-api</artifactId> </dependency> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-core</artifactId> + </dependency> </dependencies> </project> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/31fb2e79/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCDriver.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCDriver.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCDriver.java index a188594..c4ad139 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCDriver.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCDriver.java @@ -30,6 +30,10 @@ import org.apache.hyracks.control.common.controllers.CCConfig; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.config.Configuration; +import org.apache.logging.log4j.core.config.ConfigurationFactory; +import org.apache.logging.log4j.core.config.ConfigurationSource; import org.kohsuke.args4j.CmdLineException; public class CCDriver { @@ -44,6 +48,12 @@ public class CCDriver { ICCApplication application = getApplication(args); application.registerConfig(configManager); CCConfig ccConfig = new CCConfig(configManager); + LoggerContext ctx = (LoggerContext) LogManager.getContext(false); + Configuration cfg = ctx.getConfiguration(); + CCLogConfigurationFactory logCfgFactory = new CCLogConfigurationFactory(ccConfig); + ConfigurationFactory.setConfigurationFactory(logCfgFactory); + cfg.removeLogger("Console"); + ctx.start(logCfgFactory.getConfiguration(ctx, ConfigurationSource.NULL_SOURCE)); ClusterControllerService ccService = new ClusterControllerService(ccConfig, application); ccService.start(); while (true) { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/31fb2e79/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCLogConfigurationFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCLogConfigurationFactory.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCLogConfigurationFactory.java new file mode 100644 index 0000000..59ef913 --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCLogConfigurationFactory.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.control.cc; + +import org.apache.hyracks.control.common.controllers.CCConfig; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.config.Configuration; +import org.apache.logging.log4j.core.config.ConfigurationFactory; +import org.apache.logging.log4j.core.config.ConfigurationSource; +import org.apache.logging.log4j.core.config.builder.api.AppenderComponentBuilder; +import org.apache.logging.log4j.core.config.builder.api.ComponentBuilder; +import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilder; +import org.apache.logging.log4j.core.config.builder.api.LayoutComponentBuilder; +import org.apache.logging.log4j.core.config.builder.impl.BuiltConfiguration; + +import java.net.URI; + +public class CCLogConfigurationFactory extends ConfigurationFactory { + private CCConfig config; + + public CCLogConfigurationFactory(CCConfig config) { + this.config = config; + } + + public Configuration createConfiguration(ConfigurationBuilder<BuiltConfiguration> builder) { + String logDir = config.getLogDir(); + builder.setStatusLevel(Level.WARN); + builder.setConfigurationName("RollingBuilder"); + // create a rolling file appender + LayoutComponentBuilder defaultLayout = builder.newLayout("PatternLayout").addAttribute("pattern", + "%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"); + ComponentBuilder triggeringPolicy = builder.newComponent("Policies") + .addComponent(builder.newComponent("CronTriggeringPolicy").addAttribute("schedule", "0 0 0 * * ?")) + .addComponent(builder.newComponent("SizeBasedTriggeringPolicy").addAttribute("size", "50M")); + AppenderComponentBuilder defaultRoll = + builder.newAppender("default", "RollingFile").addAttribute("fileName", logDir + "cc.log") + .addAttribute("filePattern", logDir + "cc-%d{MM-dd-yy}.log.gz").add(defaultLayout) + .addComponent(triggeringPolicy); + builder.add(defaultRoll); + + // create the new logger + builder.add(builder.newRootLogger(Level.INFO).add(builder.newAppenderRef("default"))); + + LayoutComponentBuilder accessLayout = builder.newLayout("PatternLayout").addAttribute("pattern", "%m%n"); + AppenderComponentBuilder accessRoll = + builder.newAppender("access", "RollingFile").addAttribute("fileName", logDir + "access.log") + .addAttribute("filePattern", logDir + "access-%d{MM-dd-yy}.log.gz").add(accessLayout) + .addComponent(triggeringPolicy); + builder.add(accessRoll); + builder.add(builder.newLogger("org.apache.hyracks.http.server.CLFLogger", Level.forName("ACCESS", 550)) + .add(builder.newAppenderRef("access")).addAttribute("additivity", false)); + + return builder.build(); + } + + @Override + public Configuration getConfiguration(final LoggerContext loggerContext, final ConfigurationSource source) { + return getConfiguration(loggerContext, source.toString(), null); + } + + @Override + public Configuration getConfiguration(final LoggerContext loggerContext, final String name, + final URI configLocation) { + ConfigurationBuilder<BuiltConfiguration> builder = newConfigurationBuilder(); + return createConfiguration(builder); + } + + @Override + protected String[] getSupportedTypes() { + return new String[] { "*" }; + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/31fb2e79/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/ControllerConfig.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/ControllerConfig.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/ControllerConfig.java index 8ecd312..65c7ca5 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/ControllerConfig.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/ControllerConfig.java @@ -20,6 +20,7 @@ package org.apache.hyracks.control.common.controllers; import java.io.Serializable; import java.net.URL; +import java.util.function.Function; import org.apache.hyracks.api.config.IApplicationConfig; import org.apache.hyracks.api.config.IOption; @@ -35,21 +36,32 @@ public class ControllerConfig implements Serializable { private static final long serialVersionUID = 1L; public enum Option implements IOption { - CONFIG_FILE(OptionTypes.STRING, "Specify path to master configuration file", null), - CONFIG_FILE_URL(OptionTypes.URL, "Specify URL to master configuration file", null), + CONFIG_FILE(OptionTypes.STRING, (String) null, "Specify path to master configuration file"), + CONFIG_FILE_URL(OptionTypes.URL, (URL) null, "Specify URL to master configuration file"), DEFAULT_DIR( OptionTypes.STRING, "Directory where files are written to by default", - FileUtil.joinPath(System.getProperty(ConfigurationUtil.JAVA_IO_TMPDIR), "hyracks")),; + FileUtil.joinPath(System.getProperty(ConfigurationUtil.JAVA_IO_TMPDIR), "hyracks")), + LOG_DIR( + OptionTypes.STRING, + (Function<IApplicationConfig, String>) appConfig -> FileUtil + .joinPath(appConfig.getString(ControllerConfig.Option.DEFAULT_DIR), "logs/"), + "The directory where logs for this node are written"); private final IOptionType type; private final String description; - private String defaultValue; + private Object defaultValue; - Option(IOptionType type, String description, String defaultValue) { + <T> Option(IOptionType<T> type, T defaultValue, String description) { this.type = type; + this.defaultValue = defaultValue; this.description = description; + } + + <T> Option(IOptionType<T> type, Function<IApplicationConfig, T> defaultValue, String description) { + this.type = type; this.defaultValue = defaultValue; + this.description = description; } @Override @@ -106,4 +118,8 @@ public class ControllerConfig implements Serializable { public void setConfigFileUrl(URL configFileUrl) { configManager.set(ControllerConfig.Option.CONFIG_FILE_URL, configFileUrl); } + + public String getLogDir() { + return configManager.getAppConfig().getString(ControllerConfig.Option.LOG_DIR); + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/31fb2e79/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml index c962029..d99b5ff 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml @@ -88,6 +88,10 @@ <artifactId>log4j-api</artifactId> </dependency> <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-core</artifactId> + </dependency> + <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/31fb2e79/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCDriver.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCDriver.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCDriver.java index a03e0ce..fdd271c 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCDriver.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCDriver.java @@ -28,6 +28,10 @@ import org.apache.hyracks.control.common.controllers.NCConfig; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.config.Configuration; +import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.config.ConfigurationFactory; +import org.apache.logging.log4j.core.config.ConfigurationSource; import org.kohsuke.args4j.CmdLineException; @SuppressWarnings("InfiniteLoopStatement") @@ -44,6 +48,12 @@ public class NCDriver { INCApplication application = getApplication(args); application.registerConfig(configManager); NCConfig ncConfig = new NCConfig(nodeId, configManager); + LoggerContext ctx = (LoggerContext) LogManager.getContext(false); + Configuration cfg = ctx.getConfiguration(); + NCLogConfigurationFactory logCfgFactory = new NCLogConfigurationFactory(ncConfig); + ConfigurationFactory.setConfigurationFactory(logCfgFactory); + cfg.removeLogger("Console"); + ctx.start(logCfgFactory.getConfiguration(ctx, ConfigurationSource.NULL_SOURCE)); final NodeControllerService ncService = new NodeControllerService(ncConfig, application); ncService.start(); while (true) { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/31fb2e79/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCLogConfigurationFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCLogConfigurationFactory.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCLogConfigurationFactory.java new file mode 100644 index 0000000..990d6c9 --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCLogConfigurationFactory.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.control.nc; + +import org.apache.hyracks.control.common.controllers.NCConfig; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.config.Configuration; +import org.apache.logging.log4j.core.config.ConfigurationFactory; +import org.apache.logging.log4j.core.config.ConfigurationSource; +import org.apache.logging.log4j.core.config.builder.api.AppenderComponentBuilder; +import org.apache.logging.log4j.core.config.builder.api.ComponentBuilder; +import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilder; +import org.apache.logging.log4j.core.config.builder.api.LayoutComponentBuilder; +import org.apache.logging.log4j.core.config.builder.impl.BuiltConfiguration; + +import java.net.URI; + +public class NCLogConfigurationFactory extends ConfigurationFactory { + private NCConfig config; + + public NCLogConfigurationFactory(NCConfig config) { + this.config = config; + } + + public Configuration createConfiguration(ConfigurationBuilder<BuiltConfiguration> builder) { + String nodeId = config.getNodeId(); + String logDir = config.getLogDir(); + builder.setStatusLevel(Level.WARN); + builder.setConfigurationName("RollingBuilder"); + // create a rolling file appender + LayoutComponentBuilder defaultLayout = builder.newLayout("PatternLayout").addAttribute("pattern", + "%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"); + ComponentBuilder triggeringPolicy = builder.newComponent("Policies") + .addComponent(builder.newComponent("CronTriggeringPolicy").addAttribute("schedule", "0 0 0 * * ?")) + .addComponent(builder.newComponent("SizeBasedTriggeringPolicy").addAttribute("size", "50M")); + AppenderComponentBuilder defaultRoll = + builder.newAppender("default", "RollingFile").addAttribute("fileName", logDir + "nc-" + nodeId + ".log") + .addAttribute("filePattern", logDir + "nc-" + nodeId + "-%d{MM-dd-yy}.log.gz") + .add(defaultLayout).addComponent(triggeringPolicy); + builder.add(defaultRoll); + + // create the new logger + builder.add(builder.newRootLogger(Level.INFO).add(builder.newAppenderRef("default"))); + + LayoutComponentBuilder accessLayout = builder.newLayout("PatternLayout").addAttribute("pattern", "%m%n"); + AppenderComponentBuilder accessRoll = builder.newAppender("access", "RollingFile") + .addAttribute("fileName", logDir + "access-" + nodeId + ".log") + .addAttribute("filePattern", logDir + "access-" + nodeId + "-%d{MM-dd-yy}.log.gz").add(accessLayout) + .addComponent(triggeringPolicy); + builder.add(accessRoll); + builder.add(builder.newLogger("org.apache.hyracks.http.server.CLFLogger", Level.forName("ACCESS", 550)) + .add(builder.newAppenderRef("access")).addAttribute("additivity", false)); + + return builder.build(); + } + + @Override + public Configuration getConfiguration(final LoggerContext loggerContext, final ConfigurationSource source) { + return getConfiguration(loggerContext, source.toString(), null); + } + + @Override + public Configuration getConfiguration(final LoggerContext loggerContext, final String name, + final URI configLocation) { + ConfigurationBuilder<BuiltConfiguration> builder = newConfigurationBuilder(); + return createConfiguration(builder); + } + + @Override + protected String[] getSupportedTypes() { + return new String[] { "*" }; + } + + @Override + public String toString() { + return "NCLogConfiguration"; + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/31fb2e79/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/CLFLogger.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/CLFLogger.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/CLFLogger.java new file mode 100644 index 0000000..cfbcad8 --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/CLFLogger.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hyracks.http.server; + +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; + +import io.netty.channel.socket.nio.NioSocketChannel; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import io.netty.channel.ChannelDuplexHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelPromise; +import io.netty.handler.codec.http.DefaultFullHttpResponse; +import io.netty.handler.codec.http.DefaultHttpContent; +import io.netty.handler.codec.http.DefaultHttpResponse; +import io.netty.handler.codec.http.HttpContent; +import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpRequest; +import io.netty.handler.codec.http.HttpResponse; +import io.netty.handler.codec.http.LastHttpContent; + +//Based in part on LoggingHandler from Netty +public class CLFLogger extends ChannelDuplexHandler { + + private static final Logger accessLogger = LogManager.getLogger(); + private static final String ACCESS_LOG_LEVEL = "ACCESS"; + private static final DateTimeFormatter DATE_TIME_FORMATTER = + DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z").withZone(ZoneId.systemDefault()); + private StringBuilder logLineBuilder; + + private String clientIp; + private Instant requestTime; + private String reqLine; + private int statusCode; + private long respSize; + private String userAgentRef; + private boolean lastChunk = false; + + public CLFLogger() { + this.logLineBuilder = new StringBuilder(); + respSize = 0; + } + + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) { + if (msg instanceof HttpRequest) { + HttpRequest req = (HttpRequest) msg; + clientIp = ((NioSocketChannel) ctx.channel()).remoteAddress().getAddress().toString().substring(1); + requestTime = Instant.now(); + reqLine = req.method().toString() + " " + req.getUri() + " " + req.getProtocolVersion().toString(); + userAgentRef = headerValueOrDash("Referer", req) + " " + headerValueOrDash("User-Agent", req); + lastChunk = false; + } + ctx.fireChannelRead(msg); + } + + private String headerValueOrDash(String headerKey, HttpRequest req) { + String value = req.headers().get(headerKey); + if (value == null) { + value = "-"; + } else { + value = "\"" + value + "\""; + } + return value; + + } + + @Override + public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) { + if (msg instanceof DefaultHttpResponse) { + HttpResponse resp = (DefaultHttpResponse) msg; + statusCode = resp.status().code(); + if (msg instanceof DefaultFullHttpResponse) { + lastChunk = true; + respSize = resp.headers().getInt(HttpHeaderNames.CONTENT_LENGTH); + } + } else if (msg instanceof DefaultHttpContent) { + HttpContent content = (DefaultHttpContent) msg; + + respSize += content.content().readableBytes(); + } else if (msg instanceof LastHttpContent) { + lastChunk = true; + } + + ctx.write(msg, promise); + } + + @Override + public void flush(ChannelHandlerContext ctx) throws Exception { + if (lastChunk) { + printAndPrepare(); + lastChunk = false; + } + ctx.flush(); + } + + private void printAndPrepare() { + logLineBuilder.append(clientIp); + //identd value - not relevant here + logLineBuilder.append(" - "); + //no http auth or any auth either for that matter + logLineBuilder.append(" - ["); + logLineBuilder.append(DATE_TIME_FORMATTER.format(requestTime)); + logLineBuilder.append("] \""); + logLineBuilder.append(reqLine); + logLineBuilder.append("\""); + logLineBuilder.append(" ").append(statusCode); + logLineBuilder.append(" ").append(respSize); + logLineBuilder.append(" ").append(userAgentRef); + accessLogger.log(Level.forName(ACCESS_LOG_LEVEL, 550), logLineBuilder.toString()); + respSize = 0; + logLineBuilder.setLength(0); + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/31fb2e79/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java index 42b47fb..343faa4 100644 --- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java +++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java @@ -28,6 +28,9 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import io.netty.util.internal.logging.InternalLoggerFactory; +import io.netty.util.internal.logging.Log4J2LoggerFactory; +import io.netty.util.internal.logging.Log4JLoggerFactory; import org.apache.hyracks.http.api.IChannelClosedHandler; import org.apache.hyracks.http.api.IServlet; import org.apache.hyracks.util.MXHelper; @@ -49,6 +52,8 @@ import io.netty.channel.socket.nio.NioServerSocketChannel; import io.netty.handler.codec.http.FullHttpRequest; import io.netty.handler.logging.LogLevel; import io.netty.handler.logging.LoggingHandler; +import io.netty.util.internal.logging.InternalLoggerFactory; +import io.netty.util.internal.logging.Log4J2LoggerFactory; public class HttpServer { // Constants @@ -81,6 +86,10 @@ public class HttpServer { private Throwable cause; private HttpServerConfig config; + static { + InternalLoggerFactory.setDefaultFactory(Log4J2LoggerFactory.INSTANCE); + } + public HttpServer(EventLoopGroup bossGroup, EventLoopGroup workerGroup, int port, HttpServerConfig config) { this(bossGroup, workerGroup, port, config, null); } @@ -91,6 +100,7 @@ public class HttpServer { this.workerGroup = workerGroup; this.port = port; this.closedHandler = closeHandler; + InternalLoggerFactory.setDefaultFactory(Log4J2LoggerFactory.INSTANCE); this.config = config; ctx = new ConcurrentHashMap<>(); servlets = new ArrayList<>(); @@ -100,11 +110,12 @@ public class HttpServer { runnable -> new Thread(runnable, "HttpExecutor(port:" + port + ")-" + threadId.getAndIncrement())); long directMemoryBudget = numExecutorThreads * (long) HIGH_WRITE_BUFFER_WATER_MARK + numExecutorThreads * config.getMaxResponseChunkSize(); - LOGGER.log(Level.INFO, "The output direct memory budget for this server is " + directMemoryBudget + " bytes"); + LOGGER.log(Level.DEBUG, + "The output direct memory budget for this server " + "is " + directMemoryBudget + " bytes"); long inputBudgetEstimate = (long) config.getMaxRequestInitialLineLength() * (config.getRequestQueueSize() + numExecutorThreads); inputBudgetEstimate = inputBudgetEstimate * 2; - LOGGER.log(Level.INFO, + LOGGER.log(Level.DEBUG, "The \"estimated\" input direct memory budget for this server is " + inputBudgetEstimate + " bytes"); // Having multiple arenas, memory fragments, and local thread cached buffers // can cause the input memory usage to exceed estimate and custom buffer allocator must be used to avoid this http://git-wip-us.apache.org/repos/asf/asterixdb/blob/31fb2e79/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerInitializer.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerInitializer.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerInitializer.java index e10da64..eafee2d 100644 --- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerInitializer.java +++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerInitializer.java @@ -21,8 +21,11 @@ package org.apache.hyracks.http.server; import io.netty.channel.ChannelInitializer; import io.netty.channel.ChannelPipeline; import io.netty.channel.socket.SocketChannel; +import io.netty.handler.codec.http.HttpObjectAggregator; import io.netty.handler.codec.http.HttpRequestDecoder; import io.netty.handler.codec.http.HttpResponseEncoder; +import io.netty.handler.logging.LogLevel; +import org.apache.logging.log4j.Logger; public class HttpServerInitializer extends ChannelInitializer<SocketChannel> { @@ -49,6 +52,7 @@ public class HttpServerInitializer extends ChannelInitializer<SocketChannel> { p.addLast(new HttpRequestCapacityController(server)); p.addLast(new HttpRequestDecoder(maxRequestInitialLineLength, maxRequestHeaderSize, maxRequestChunkSize)); p.addLast(new HttpResponseEncoder()); + p.addLast(new CLFLogger()); p.addLast(new HttpRequestAggregator(maxRequestSize)); p.addLast(server.createHttpHandler(maxResponseChunkSize)); }
