QChris has uploaded a new change for review.
https://gerrit.wikimedia.org/r/185341
Change subject: Add script to compare count of events between EventLogging logs
and database
......................................................................
Add script to compare count of events between EventLogging logs and database
By allowing to compare the volume of logs to the volume of events in
the database, this script assists both in detecting impact / duration
of issues, and also in identifying bottlenecks.
Change-Id: I3ed9f59159c22114977304cabeefdaf976e413b0
---
A server/tools/compare_eventlogging_log_to_db.sh
1 file changed, 172 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/EventLogging
refs/changes/41/185341/1
diff --git a/server/tools/compare_eventlogging_log_to_db.sh
b/server/tools/compare_eventlogging_log_to_db.sh
new file mode 100755
index 0000000..57ff28f
--- /dev/null
+++ b/server/tools/compare_eventlogging_log_to_db.sh
@@ -0,0 +1,172 @@
+#!/bin/bash
+
+set -e
+set -o pipefail
+
+JQ=/usr/bin/jq
+ZCAT=/bin/zcat
+
+KIND="all"
+SCHEMA="SaveTiming"
+DATE="today"
+DIFF_FORMAT="--unified"
+
+error() {
+ echo "Error" "$@" >&2
+ exit 1
+}
+
+print_help() {
+ cat <<EOF
+compare_eventlogging_log_to_db.sh [ OPTIONS ]
+
+Compares counts of events stored in the log files to those from the
+plain-file logs.
+
+Events are grouped by timestamp (seconds since epoc / 100).
+The counts for the groups are then diffed (LHS is log file, RHS is db).
+
+This script is built to run on stat1003.
+
+OPTIONS:
+
+--date DATE -- check for DATE instead of "today"
+--kind KIND -- check the kind KIND instead of "all"
+ KIND has to be one of:
+ all (validated)
+ client (raw)
+ server (raw)
+--schema SCHEMA -- check schema SCHEMA instead of "SaveTiming"
+--side-by-side -- diff in side-by-side format
+--unified -- diff in unified format (default)
+EOF
+}
+
+while [ $# -gt 0 ]
+do
+ PARAM="$1"
+ shift
+ case "$PARAM" in
+ "--help" | "-h" | "-?" )
+ print_help
+ exit 1
+ ;;
+ "--kind" )
+ [[ $# -gt 0 ]] || error "$PARAM requires a further parameter"
+ KIND="$1"
+ if [ "$KIND" != "server" -a "$KIND" != "client" -a "$KIND" !=
"all" ]
+ then
+ error "Unrecognized kind '$KIND'"
+ fi
+ shift
+ ;;
+ "--date" )
+ [[ $# -gt 0 ]] || error "$PARAM requires a further parameter"
+ DATE="$1"
+ shift
+ ;;
+ "--schema" )
+ [[ $# -gt 0 ]] || error "$PARAM requires a further parameter"
+ SCHEMA="$1"
+ shift
+ ;;
+ "--side-by-side" | "--unified" )
+ DIFF_FORMAT="$PARAM"
+ ;;
+ * )
+ error "Unknown parameter '$PARAM'"
+ esac
+done
+
+DATE="$(date -d "$DATE" +'%Y-%m-%d')"
+
+EVENTLOGGING_ARCHIVE_DIR_ABS="/a/eventlogging/archive/"
+
+ALL_EVENTS_FILE_ABS="${EVENTLOGGING_ARCHIVE_DIR_ABS}/all-events.log-$(date -d
"$DATE" +'%Y%m%d').gz"
+SERVER_SIDE_EVENTS_FILE_ABS="${EVENTLOGGING_ARCHIVE_DIR_ABS}/server-side-events.log-$(date
-d "$DATE" +'%Y%m%d').gz"
+CLIENT_SIDE_EVENTS_FILE_ABS="${EVENTLOGGING_ARCHIVE_DIR_ABS}/client-side-events.log-$(date
-d "$DATE" +'%Y%m%d').gz"
+
+FIRST_TIMESTAMP="$DATE 06:40:00+0000 - 1 day"
+LAST_TIMESTAMP="$DATE 06:15:00+0000"
+
+run_mysql() {
+ mysql \
+ --defaults-extra-file=/etc/mysql/conf.d/research-client.cnf
--host=dbstore1002.eqiad.wmnet --database=log \
+ --skip-column-names \
+ --execute "$@"
+}
+
+make_comparable() {
+ sed -e 's/^[[:space:]]\+//' -e 's/[[:space:]]\+/ /g' -e 's/$/00/'
+}
+
+post_process_timestamps_from_files() {
+ awk '{if ('"$(date -d "$FIRST_TIMESTAMP" +'%s')"' <= $1 && $1 <= '"$(date
-d "$LAST_TIMESTAMP" +'%s')"') print}' \
+ | sed -e 's/..$//' \
+ | sort \
+ | uniq -c \
+ | make_comparable
+}
+
+cat_client_file() {
+ zcat "$CLIENT_SIDE_EVENTS_FILE_ABS" \
+ | grep "%22schema%22%3A%22$SCHEMA%22%2C" \
+ | grep "%22revision%22%3A$REVISION%2C" \
+ | grep
'\\t\(20[0-9][0-9]-[0-9][0-9]-[0-9][0-9]T[0-9][0-9]:[0-9][0-9]:[0-9][0-9]\)\\t'
\
+ | sed -e
's/^.*\\t\(20[0-9][0-9]-[0-9][0-9]-[0-9][0-9]T[0-9][0-9]:[0-9][0-9]:[0-9][0-9]\)\\t.*$/\1/'
\
+ | date -f - +%s \
+ | post_process_timestamps_from_files
+}
+
+cat_server_file() {
+ zcat "$SERVER_SIDE_EVENTS_FILE_ABS" \
+ | grep '\\"schema\\":\\"'"$SCHEMA"'\\",' \
+ | grep '\\"revision\\":'"$REVISION"',' \
+ | sed -e 's/^.*\\"timestamp\\":\([0-9]\+\)\(,.*\)\?$/\1/' \
+ | post_process_timestamps_from_files
+}
+
+cat_all_file() {
+ zcat "$ALL_EVENTS_FILE_ABS" \
+ | grep "$SCHEMA" \
+ | jq --compact-output "\
+ if ( \
+ .[\"timestamp\"] >= $(date -d "$FIRST_TIMESTAMP" +'%s') \
+ and .[\"timestamp\"] <= $(date -d "$LAST_TIMESTAMP" +'%s')
\
+ and .[\"schema\"] == \"$SCHEMA\" \
+ and .[\"revision\"] == $REVISION \
+ ) then \
+ .[\"timestamp\"] \
+ else \
+ empty \
+ end" \
+ | post_process_timestamps_from_files
+}
+
+cat_db() {
+ run_mysql "\
+ SELECT \
+ COUNT(*), UNIX_TIMESTAMP(STR_TO_DATE(timestamp,'%Y%m%d%H%i%s'))
DIV 100 ts \
+ FROM ${SCHEMA}_${REVISION} \
+ WHERE \
+ timestamp BETWEEN $(date -d "$FIRST_TIMESTAMP"
+'%Y%m%d%H%M%S') AND \
+ $(date -d "$LAST_TIMESTAMP" +'%Y%m%d%H%M%S') \
+ GROUP BY ts \
+ ORDER BY ts;\
+ " \
+ | make_comparable
+}
+
+set_revision() {
+ REVISION="$(run_mysql "\
+ SHOW TABLES LIKE '${SCHEMA}_%'
+ " \
+ | cut -f 2 -d "_" \
+ | grep '^[0-9]\+$' \
+ | sort -n \
+ | tail -n 1 \
+ )"
+}
+
+set_revision
+diff -Na "$DIFF_FORMAT" <(cat_${KIND}_file) <(cat_db)
--
To view, visit https://gerrit.wikimedia.org/r/185341
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I3ed9f59159c22114977304cabeefdaf976e413b0
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/EventLogging
Gerrit-Branch: master
Gerrit-Owner: QChris <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
