This is an automated email from the ASF dual-hosted git repository.
marcuse pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/trunk by this push:
new 7d138e2 Add a new tool to dump audit logs
7d138e2 is described below
commit 7d138e20ea987d44fffbc47de4674b253b7431ff
Author: Vinay Chella <[email protected]>
AuthorDate: Mon Dec 17 23:42:14 2018 -0800
Add a new tool to dump audit logs
Patch by Vinay Chella; reviewed by marcuse for CASSANDRA-14885
---
CHANGES.txt | 1 +
doc/source/operating/audit_logging.rst | 27 ++-
.../org/apache/cassandra/audit/BinAuditLogger.java | 14 +-
.../org/apache/cassandra/tools/AuditLogViewer.java | 212 +++++++++++++++++++++
.../apache/cassandra/tools/AuditLogViewerTest.java | 84 ++++++++
tools/bin/auditlogviewer | 49 +++++
tools/bin/auditlogviewer.bat | 41 ++++
7 files changed, 423 insertions(+), 5 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index ec59fa6..a0c51f0 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
4.0
+ * Add a new tool to dump audit logs (CASSANDRA-14885)
* Fix generating javadoc with Java11 (CASSANDRA-14988)
* Only cancel conflicting compactions when starting anticompactions and sub
range compactions (CASSANDRA-14935)
* Use a stub IndexRegistry for non-daemon use cases (CASSANDRA-14938)
diff --git a/doc/source/operating/audit_logging.rst
b/doc/source/operating/audit_logging.rst
index 6cfd141..068209e 100644
--- a/doc/source/operating/audit_logging.rst
+++ b/doc/source/operating/audit_logging.rst
@@ -143,19 +143,44 @@ NodeTool command to reload AuditLog filters
``enableauditlog``: NodeTool enableauditlog command can be used to reload
auditlog filters when called with default or previous ``loggername`` and
updated filters
E.g.,
+
::
nodetool enableauditlog --loggername <Default/ existing loggerName>
--included-keyspaces <New Filter values>
+View the contents of AuditLog Files
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+``auditlogviewer`` is the new tool introduced to help view the contents of
binlog file in human readable text format.
+
+::
+ auditlogviewer <path1> [<path2>...<pathN>] [options]
+
+Options
+""""""""
+``-f,--follow``
+ Upon reacahing the end of the log continue indefinitely
+ waiting for more records
+``-r,--roll_cycle``
+ How often to roll the log file was rolled. May be
+ necessary for Chronicle to correctly parse file
names. (MINUTELY, HOURLY,
+ DAILY). Default HOURLY.
+``-h,--help``
+ display this help message
+For example, to dump the contents of audit log files on the console
+
+::
+
+ auditlogviewer /logs/cassandra/audit
Sample output
-^^^^^^^^^^^^^^^^
+"""""""""""""
+
::
LogMessage:
user:anonymous|host:localhost/X.X.X.X|source:/X.X.X.X|port:60878|timestamp:1521158923615|type:USE_KS|category:DDL|ks:dev1|operation:USE
"dev1"
diff --git a/src/java/org/apache/cassandra/audit/BinAuditLogger.java
b/src/java/org/apache/cassandra/audit/BinAuditLogger.java
index bd3a158..23b9977 100644
--- a/src/java/org/apache/cassandra/audit/BinAuditLogger.java
+++ b/src/java/org/apache/cassandra/audit/BinAuditLogger.java
@@ -19,6 +19,7 @@ package org.apache.cassandra.audit;
import java.nio.file.Paths;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.primitives.Ints;
import net.openhft.chronicle.wire.WireOut;
@@ -29,6 +30,10 @@ import org.apache.cassandra.utils.concurrent.WeightedQueue;
public class BinAuditLogger extends BinLogAuditLogger implements IAuditLogger
{
+ public static final String TYPE = "type";
+ public static final String AUDITLOG_TYPE = "AuditLog";
+ public static final String AUDITLOG_MESSAGE = "message";
+
public BinAuditLogger()
{
// due to the way that IAuditLogger instance are created in
AuditLogManager, via reflection, we can't assume
@@ -56,11 +61,12 @@ public class BinAuditLogger extends BinLogAuditLogger
implements IAuditLogger
super.logRecord(new Message(auditLogEntry.getLogString()), binLog);
}
- static class Message extends BinLog.ReleaseableWriteMarshallable
implements WeightedQueue.Weighable
+ @VisibleForTesting
+ public static class Message extends BinLog.ReleaseableWriteMarshallable
implements WeightedQueue.Weighable
{
private final String message;
- Message(String message)
+ public Message(String message)
{
this.message = message;
}
@@ -68,8 +74,8 @@ public class BinAuditLogger extends BinLogAuditLogger
implements IAuditLogger
@Override
public void writeMarshallable(WireOut wire)
{
- wire.write("type").text("AuditLog");
- wire.write("message").text(message);
+ wire.write(TYPE).text(AUDITLOG_TYPE);
+ wire.write(AUDITLOG_MESSAGE).text(message);
}
@Override
diff --git a/src/java/org/apache/cassandra/tools/AuditLogViewer.java
b/src/java/org/apache/cassandra/tools/AuditLogViewer.java
new file mode 100644
index 0000000..01ea7b3
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/AuditLogViewer.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.tools;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.List;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+
+import net.openhft.chronicle.core.io.IORuntimeException;
+import net.openhft.chronicle.queue.ChronicleQueueBuilder;
+import net.openhft.chronicle.queue.ExcerptTailer;
+import net.openhft.chronicle.queue.RollCycles;
+import net.openhft.chronicle.queue.impl.single.SingleChronicleQueue;
+import net.openhft.chronicle.threads.Pauser;
+import net.openhft.chronicle.wire.ReadMarshallable;
+import net.openhft.chronicle.wire.WireIn;
+import org.apache.cassandra.audit.BinAuditLogger;
+
+/**
+ * Tool to view the contenst of AuditLog files in human readable format.
Default implementation for AuditLog files
+ * logs audit messages in {@link org.apache.cassandra.utils.binlog.BinLog}
format, this tool prints the contens of
+ * binary audit log files in text format.
+ */
+public class AuditLogViewer
+{
+ private static final String TOOL_NAME = "auditlogviewer";
+ private static final String ROLL_CYCLE = "roll_cycle";
+ private static final String FOLLOW = "follow";
+ private static final String HELP_OPTION = "help";
+
+ public static void main(String[] args)
+ {
+ AuditLogViewerOptions options = AuditLogViewerOptions.parseArgs(args);
+
+ try
+ {
+ dump(options.pathList, options.rollCycle, options.follow,
System.out::print);
+ }
+ catch (Exception e)
+ {
+ System.err.println(e.getMessage());
+ System.exit(1);
+ }
+ }
+
+ static void dump(List<String> pathList, String rollCycle, boolean follow,
Consumer<String> displayFun)
+ {
+ //Backoff strategy for spinning on the queue, not aggressive at all as
this doesn't need to be low latency
+ Pauser pauser = Pauser.millis(100);
+ List<ExcerptTailer> tailers = pathList.stream()
+ .distinct()
+ .map(path ->
ChronicleQueueBuilder.single(new
File(path)).readOnly(true).rollCycle(RollCycles.valueOf(rollCycle)).build())
+
.map(SingleChronicleQueue::createTailer)
+ .collect(Collectors.toList());
+ boolean hadWork = true;
+ while (hadWork)
+ {
+ hadWork = false;
+ for (ExcerptTailer tailer : tailers)
+ {
+ while (tailer.readDocument(new DisplayRecord(displayFun)))
+ {
+ hadWork = true;
+ }
+ }
+
+ if (follow)
+ {
+ if (!hadWork)
+ {
+ //Chronicle queue doesn't support blocking so use this
backoff strategy
+ pauser.pause();
+ }
+ //Don't terminate the loop even if there wasn't work
+ hadWork = true;
+ }
+ }
+ }
+
+ private static class DisplayRecord implements ReadMarshallable
+ {
+ private final Consumer<String> displayFun;
+
+ DisplayRecord(Consumer<String> displayFun)
+ {
+ this.displayFun = displayFun;
+ }
+
+ public void readMarshallable(WireIn wireIn) throws IORuntimeException
+ {
+ StringBuilder sb = new StringBuilder();
+
+ String type = wireIn.read(BinAuditLogger.TYPE).text();
+ sb.append("Type: ")
+ .append(type)
+ .append(System.lineSeparator());
+
+ if (null != type && type.equals(BinAuditLogger.AUDITLOG_TYPE))
+ {
+ sb.append("LogMessage:
").append(wireIn.read(BinAuditLogger.AUDITLOG_MESSAGE).text()).append(System.lineSeparator());
+ }
+
+ displayFun.accept(sb.toString());
+ }
+ }
+
+ private static class AuditLogViewerOptions
+ {
+ private final List<String> pathList;
+ private String rollCycle = "HOURLY";
+ private boolean follow;
+
+ private AuditLogViewerOptions(String[] pathList)
+ {
+ this.pathList = Arrays.asList(pathList);
+ }
+
+ static AuditLogViewerOptions parseArgs(String cmdArgs[])
+ {
+ CommandLineParser parser = new GnuParser();
+ Options options = getCmdLineOptions();
+ try
+ {
+ CommandLine cmd = parser.parse(options, cmdArgs, false);
+
+ if (cmd.hasOption(HELP_OPTION))
+ {
+ printUsage(options);
+ System.exit(0);
+ }
+
+ String[] args = cmd.getArgs();
+ if (args.length <= 0)
+ {
+ System.err.println("Audit log files directory path is a
required argument.");
+ printUsage(options);
+ System.exit(1);
+ }
+
+ AuditLogViewerOptions opts = new AuditLogViewerOptions(args);
+
+ opts.follow = cmd.hasOption(FOLLOW);
+
+ if (cmd.hasOption(ROLL_CYCLE))
+ {
+ opts.rollCycle = cmd.getOptionValue(ROLL_CYCLE);
+ }
+
+ return opts;
+ }
+ catch (ParseException e)
+ {
+ errorMsg(e.getMessage(), options);
+ return null;
+ }
+ }
+
+ static void errorMsg(String msg, Options options)
+ {
+ System.err.println(msg);
+ printUsage(options);
+ System.exit(1);
+ }
+
+ static Options getCmdLineOptions()
+ {
+ Options options = new Options();
+
+ options.addOption(new Option("r", ROLL_CYCLE, false, "How often to
roll the log file was rolled. May be necessary for Chronicle to correctly parse
file names. (MINUTELY, HOURLY, DAILY). Default HOURLY."));
+ options.addOption(new Option("f", FOLLOW, false, "Upon reacahing
the end of the log continue indefinitely waiting for more records"));
+ options.addOption(new Option("h", HELP_OPTION, false, "display
this help message"));
+
+ return options;
+ }
+
+ static void printUsage(Options options)
+ {
+ String usage = String.format("%s <path1> [<path2>...<pathN>]
[options]", TOOL_NAME);
+ StringBuilder header = new StringBuilder();
+ header.append("--\n");
+ header.append("View the audit log contents in human readable
format");
+ header.append("\n--\n");
+ header.append("Options are:");
+ new HelpFormatter().printHelp(usage, header.toString(), options,
"");
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/tools/AuditLogViewerTest.java
b/test/unit/org/apache/cassandra/tools/AuditLogViewerTest.java
new file mode 100644
index 0000000..078f899
--- /dev/null
+++ b/test/unit/org/apache/cassandra/tools/AuditLogViewerTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.commons.io.FileUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import net.openhft.chronicle.queue.ChronicleQueue;
+import net.openhft.chronicle.queue.ChronicleQueueBuilder;
+import net.openhft.chronicle.queue.ExcerptAppender;
+import net.openhft.chronicle.queue.RollCycles;
+import org.apache.cassandra.audit.BinAuditLogger;
+
+public class AuditLogViewerTest
+{
+ private Path path;
+
+ @Before
+ public void setUp() throws IOException
+ {
+ path = Files.createTempDirectory("foo");
+ }
+
+ @After
+ public void tearDown() throws IOException
+ {
+ if (path.toFile().exists() && path.toFile().isDirectory())
+ {
+ //Deletes directory and all of it's contents
+ FileUtils.deleteDirectory(path.toFile());
+ }
+ }
+
+ @Test
+ public void testDisplayRecord()
+ {
+ List<String> records = new ArrayList<>();
+ records.add("Test foo bar 1");
+ records.add("Test foo bar 2");
+
+ try (ChronicleQueue queue =
ChronicleQueueBuilder.single(path.toFile()).rollCycle(RollCycles.TEST_SECONDLY).build())
+ {
+ ExcerptAppender appender = queue.acquireAppender();
+
+ //Write bunch of records
+ records.forEach(s -> appender.writeDocument(new
BinAuditLogger.Message(s)));
+
+ //Read those written records
+ List<String> actualRecords = new ArrayList<>();
+ AuditLogViewer.dump(ImmutableList.of(path.toString()),
RollCycles.TEST_SECONDLY.toString(), false, actualRecords::add);
+
+ for (int i = 0; i < records.size(); i++)
+ {
+
Assert.assertTrue(actualRecords.get(i).contains(records.get(i)));
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/tools/bin/auditlogviewer b/tools/bin/auditlogviewer
new file mode 100755
index 0000000..a6a7375
--- /dev/null
+++ b/tools/bin/auditlogviewer
@@ -0,0 +1,49 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ "x$CASSANDRA_INCLUDE" = "x" ]; then
+ # Locations (in order) to use when searching for an include file.
+ for include in "`dirname "$0"`/cassandra.in.sh" \
+ "$HOME/.cassandra.in.sh" \
+ /usr/share/cassandra/cassandra.in.sh \
+ /usr/local/share/cassandra/cassandra.in.sh \
+ /opt/cassandra/cassandra.in.sh; do
+ if [ -r "$include" ]; then
+ . "$include"
+ break
+ fi
+ done
+elif [ -r "$CASSANDRA_INCLUDE" ]; then
+ . "$CASSANDRA_INCLUDE"
+fi
+
+if [ -z "$CLASSPATH" ]; then
+ echo "You must set the CLASSPATH var" >&2
+ exit 1
+fi
+
+if [ "x$MAX_HEAP_SIZE" = "x" ]; then
+ MAX_HEAP_SIZE="256M"
+fi
+
+"$JAVA" $JAVA_AGENT -ea -cp "$CLASSPATH" $JVM_OPTS -Xmx$MAX_HEAP_SIZE \
+ -Dcassandra.storagedir="$cassandra_storagedir" \
+ -Dlogback.configurationFile=logback-tools.xml \
+ org.apache.cassandra.tools.AuditLogViewer "$@"
+
+# vi:ai sw=4 ts=4 tw=0 et
diff --git a/tools/bin/auditlogviewer.bat b/tools/bin/auditlogviewer.bat
new file mode 100644
index 0000000..3b6bd81
--- /dev/null
+++ b/tools/bin/auditlogviewer.bat
@@ -0,0 +1,41 @@
+@REM
+@REM Licensed to the Apache Software Foundation (ASF) under one or more
+@REM contributor license agreements. See the NOTICE file distributed with
+@REM this work for additional information regarding copyright ownership.
+@REM The ASF licenses this file to You under the Apache License, Version 2.0
+@REM (the "License"); you may not use this file except in compliance with
+@REM the License. You may obtain a copy of the License at
+@REM
+@REM http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing, software
+@REM distributed under the License is distributed on an "AS IS" BASIS,
+@REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@REM See the License for the specific language governing permissions and
+@REM limitations under the License.
+
+@echo off
+if "%OS%" == "Windows_NT" setlocal
+
+pushd "%~dp0"
+call cassandra.in.bat
+
+if NOT DEFINED CASSANDRA_MAIN set
CASSANDRA_MAIN=org.apache.cassandra.tools.AuditLogViewer
+if NOT DEFINED JAVA_HOME goto :err
+
+REM ***** JAVA options *****
+set JAVA_OPTS=^
+ -Dlogback.configurationFile=logback-tools.xml
+
+set TOOLS_PARAMS=
+
+"%JAVA_HOME%\bin\java" %JAVA_OPTS% %CASSANDRA_PARAMS% -cp
%CASSANDRA_CLASSPATH% "%CASSANDRA_MAIN%" %*
+goto finally
+
+:err
+echo JAVA_HOME environment variable must be set!
+pause
+
+:finally
+
+ENDLOCAL
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
