Springle has submitted this change and it was merged.
Change subject: Eventlogging backfilling consistency checks.
......................................................................
Eventlogging backfilling consistency checks.
Change-Id: Ice9e1f2a9485153024625218742a97b783ea728f
---
A dbtools/eventlogging_resync.sh
M dbtools/eventlogging_sync.sh
2 files changed, 58 insertions(+), 10 deletions(-)
Approvals:
Springle: Looks good to me, approved
jenkins-bot: Verified
diff --git a/dbtools/eventlogging_resync.sh b/dbtools/eventlogging_resync.sh
new file mode 100755
index 0000000..223aa83
--- /dev/null
+++ b/dbtools/eventlogging_resync.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Event Logging tables hold time-series data which is not updated, but may be
purged.
+# Every table includes a uuid and a timestamp.
+#
+# ./eventlogging_resync.sh dbstore1002 20150201000000
+
+db='log'
+ls="regexp '^[A-Z0-9].*[0-9]+$'"
+mhost='m2-master'
+shost="$1"
+since="$2"
+
+slave="mysql -h $shost --compress --skip-column-names"
+master="mysql -h $mhost --compress --skip-column-names"
+dump="mysqldump -h $mhost --skip-opt --single-transaction --quick
--skip-triggers"
+dumpdata="$dump --no-create-info --order-by-primary --insert-ignore
--extended-insert --compress --hex-blob"
+querytables="select table_name from information_schema.tables where
table_schema = '$db' and table_name"
+
+set -e
+
+for table in $($master $db -e "$querytables $ls order by rand()"); do
+
+ echo -n $table
+
+ ts1="$since"
+ ts2=$($master $db -e "select max(timestamp) from \`$table\`")
+
+ if [ ! "$ts2" = "NULL" ]; then
+
+ echo " $ts1 to $ts2"
+
+ while [ $($slave -e "select if('$ts1'<'$ts2',1,0)") -eq 1 ]; do
+ tsn=$($slave -e "select date_format('$ts1' + interval 1 hour,
'%Y%m%d%H%i%s')")
+
+ scount=$($slave $db -e "select count(*) from \`$table\` where
timestamp >= '$ts1' and timestamp < '$tsn'")
+ mcount=$($master $db -e "select count(*) from \`$table\` where
timestamp >= '$ts1' and timestamp < '$tsn'")
+
+ if [ $mcount -ne $scount ]; then
+ echo "$table $ts1:$tsn $mcount $scount"
+ $dumpdata --insert-ignore --where="timestamp >= '$ts1' and
timestamp < '$tsn'" $db "$table" | $slave $db
+ fi
+ ts1="$tsn"
+ done
+ else
+ echo " empty"
+ fi
+done
diff --git a/dbtools/eventlogging_sync.sh b/dbtools/eventlogging_sync.sh
index 16eaacd..b91a14c 100755
--- a/dbtools/eventlogging_sync.sh
+++ b/dbtools/eventlogging_sync.sh
@@ -38,18 +38,18 @@
$dump --no-data $db $table | $slave $db
fi
- id=$($master $db -e "select min(id) from \`$table\`")
+# id=$($master $db -e "select min(id) from \`$table\`")
+#
+# if [ ! $id = "NULL" ]; then
+# echo -n ", purge < $id"
+# $slave $db -e "delete from \`$table\` where id < $id order by id
limit 100000"
+# fi
- if [ ! $id = "NULL" ]; then
- echo -n ", purge < $id"
- $slave $db -e "delete from \`$table\` where id < $id order by id limit
100000"
- fi
+ ts=$($slave $db -e "select max(timestamp) from \`$table\`")
- id=$($slave $db -e "select max(id) from \`$table\`")
-
- if [ ! $id = "NULL" ]; then
- echo -n ", load > $id"
- $dumpdata --where="id > $id" $db $table | $slave $db
+ if [ ! $ts = "NULL" ]; then
+ echo -n ", load > $ts"
+ $dumpdata --replace --where="timestamp >= $ts" $db $table | $slave $db
else
echo -n ", import"
$dumpdata $db $table | $slave $db
--
To view, visit https://gerrit.wikimedia.org/r/190772
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ice9e1f2a9485153024625218742a97b783ea728f
Gerrit-PatchSet: 1
Gerrit-Project: operations/software
Gerrit-Branch: master
Gerrit-Owner: Springle <[email protected]>
Gerrit-Reviewer: Springle <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
