DrTrigon has uploaded a new change for review.
https://gerrit.wikimedia.org/r/114749
Change subject: improvement; sge stdout/-err logs and lighttpd server access
log maintained automatically
......................................................................
improvement; sge stdout/-err logs and lighttpd server access log maintained
automatically
Change-Id: Icc19daa079a80d5fbd4f29bf718c2a9682c6a748
---
M crontab
M fabfile.py
2 files changed, 31 insertions(+), 22 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/pywikibot/bots/drtrigonbot
refs/changes/49/114749/1
diff --git a/crontab b/crontab
index bd802f0..4c17d06 100644
--- a/crontab
+++ b/crontab
@@ -78,35 +78,35 @@
PATH=/usr/local/bin:/usr/bin:/bin
# pywikibot-core: script_wui.py
-#06 0 * * * jstart -l h_vmem=512M -quiet -N script_wui python
$HOME/pywikibot-core/pwb.py script_wui.py -log
-06 0 * * * jstart -l h_vmem=512M -quiet -N script_wui python
$HOME/pywikibot-core/pwb.py script_wui.py -log >/dev/null
+#06 0 * * * jstart -l h_vmem=512M -quiet -o $HOME/public_html/logs/sge -j y -N
script_wui python $HOME/pywikibot-core/pwb.py script_wui.py -log
+06 0 * * * jstart -l h_vmem=512M -quiet -o $HOME/public_html/logs/sge -j y -N
script_wui python $HOME/pywikibot-core/pwb.py script_wui.py -log >/dev/null
# pywikibot-compat: sum_disc.py
-36 2 * * * jsub -l h_vmem=1024M -quiet -N sum_disc python
$HOME/pywikibot-compat/pwb.py -sum_disc
-06 2 */14 * * jsub -l h_vmem=256M -quiet -N compbot python
$HOME/pywikibot-compat/pwb.py -sum_disc -compress_history:[]
+36 2 * * * jsub -l h_vmem=1024M -quiet -o $HOME/public_html/logs/sge -j y -N
sum_disc python $HOME/pywikibot-compat/pwb.py -sum_disc
+06 2 */14 * * jsub -l h_vmem=256M -quiet -o $HOME/public_html/logs/sge -j y -N
compbot python $HOME/pywikibot-compat/pwb.py -sum_disc -compress_history:[]
# pywikibot-compat: catimages.py
-06 20 * * * jsub -l h_vmem=2048M -quiet -N catimages python
$HOME/pywikibot-compat/pwb.py -catimages
+06 20 * * * jsub -l h_vmem=2048M -quiet -o $HOME/public_html/logs/sge -j y -N
catimages python $HOME/pywikibot-compat/pwb.py -catimages
# pywikibot-compat: subster.py, subster_irc.py [SIMULATE]
-36 6 * * * jsub -l h_vmem=1024M -quiet -N subster python
$HOME/pywikibot-compat/pwb.py -subster -simulate
-#06 * * * * jstart -l h_vmem=512M -quiet -N ircbot python
$HOME/pywikibot-compat/pwb.py -subster_irc -simulate
-#06 0 * * * jsub -l h_vmem=512M -quiet -N subster_meta python
$HOME/pywikibot-compat/pwb.py -subster -family:meta -lang: -simulate
-#36 0 * * * jsub -l h_vmem=512M -quiet -N subster_frr python
$HOME/pywikibot-compat/pwb.py -subster -lang:frr -simulate
-#06 1 * * * jsub -l h_vmem=512M -quiet -N subster_en python
$HOME/pywikibot-compat/pwb.py -subster -lang:en -simulate
-36 1 * * * jsub -l h_vmem=512M -quiet -N subster_nl python
$HOME/pywikibot-compat/pwb.py -subster -lang:nl -simulate
-#06 * * * * jsub -l h_vmem=512M -quiet -N subster_ar python
$HOME/pywikibot-compat/pwb.py -subster -lang:ar -simulate
-#06 3 * * * jsub -l h_vmem=512M -quiet -N subster_data python
$HOME/pywikibot-compat/pwb.py -subster -family:wikidata -lang:wikidata -simulate
+36 6 * * * jsub -l h_vmem=1024M -quiet -o $HOME/public_html/logs/sge -j y -N
subster python $HOME/pywikibot-compat/pwb.py -subster -simulate
+#06 * * * * jstart -l h_vmem=512M -quiet -o $HOME/public_html/logs/sge -j y -N
ircbot python $HOME/pywikibot-compat/pwb.py -subster_irc -simulate
+#06 0 * * * jsub -l h_vmem=512M -quiet -o $HOME/public_html/logs/sge -j y -N
subster_meta python $HOME/pywikibot-compat/pwb.py -subster -family:meta -lang:
-simulate
+#36 0 * * * jsub -l h_vmem=512M -quiet -o $HOME/public_html/logs/sge -j y -N
subster_frr python $HOME/pywikibot-compat/pwb.py -subster -lang:frr -simulate
+#06 1 * * * jsub -l h_vmem=512M -quiet -o $HOME/public_html/logs/sge -j y -N
subster_en python $HOME/pywikibot-compat/pwb.py -subster -lang:en -simulate
+36 1 * * * jsub -l h_vmem=512M -quiet -o $HOME/public_html/logs/sge -j y -N
subster_nl python $HOME/pywikibot-compat/pwb.py -subster -lang:nl -simulate
+#06 * * * * jsub -l h_vmem=512M -quiet -o $HOME/public_html/logs/sge -j y -N
subster_ar python $HOME/pywikibot-compat/pwb.py -subster -lang:ar -simulate
+#06 3 * * * jsub -l h_vmem=512M -quiet -o $HOME/public_html/logs/sge -j y -N
subster_data python $HOME/pywikibot-compat/pwb.py -subster -family:wikidata
-lang:wikidata -simulate
## pywikibot-compat: subster.py, subster_irc.py
-#36 6 * * * jsub -l h_vmem=1024M -quiet -N subster python
$HOME/pywikibot-compat/pwb.py -subster
-#06 * * * * jstart -l h_vmem=512M -quiet -N ircbot python
$HOME/pywikibot-compat/pwb.py -subster_irc
-06 0 * * * jsub -l h_vmem=512M -quiet -N subster_meta python
$HOME/pywikibot-compat/pwb.py -subster -family:meta -lang:
-36 0 * * * jsub -l h_vmem=512M -quiet -N subster_frr python
$HOME/pywikibot-compat/pwb.py -subster -lang:frr
-06 1 * * * jsub -l h_vmem=512M -quiet -N subster_en python
$HOME/pywikibot-compat/pwb.py -subster -lang:en
-#36 1 * * * jsub -l h_vmem=512M -quiet -N subster_nl python
$HOME/pywikibot-compat/pwb.py -subster -lang:nl
-06 * * * * jsub -l h_vmem=512M -quiet -N subster_ar python
$HOME/pywikibot-compat/pwb.py -subster -lang:ar
-06 3 * * * jsub -l h_vmem=512M -quiet -N subster_data python
$HOME/pywikibot-compat/pwb.py -subster -family:wikidata -lang:wikidata
+#36 6 * * * jsub -l h_vmem=1024M -quiet -o $HOME/public_html/logs/sge -j y -N
subster python $HOME/pywikibot-compat/pwb.py -subster
+#06 * * * * jstart -l h_vmem=512M -quiet -o $HOME/public_html/logs/sge -j y -N
ircbot python $HOME/pywikibot-compat/pwb.py -subster_irc
+06 0 * * * jsub -l h_vmem=512M -quiet -o $HOME/public_html/logs/sge -j y -N
subster_meta python $HOME/pywikibot-compat/pwb.py -subster -family:meta -lang:
+36 0 * * * jsub -l h_vmem=512M -quiet -o $HOME/public_html/logs/sge -j y -N
subster_frr python $HOME/pywikibot-compat/pwb.py -subster -lang:frr
+06 1 * * * jsub -l h_vmem=512M -quiet -o $HOME/public_html/logs/sge -j y -N
subster_en python $HOME/pywikibot-compat/pwb.py -subster -lang:en
+#36 1 * * * jsub -l h_vmem=512M -quiet -o $HOME/public_html/logs/sge -j y -N
subster_nl python $HOME/pywikibot-compat/pwb.py -subster -lang:nl
+06 * * * * jsub -l h_vmem=512M -quiet -o $HOME/public_html/logs/sge -j y -N
subster_ar python $HOME/pywikibot-compat/pwb.py -subster -lang:ar
+06 3 * * * jsub -l h_vmem=512M -quiet -o $HOME/public_html/logs/sge -j y -N
subster_data python $HOME/pywikibot-compat/pwb.py -subster -family:wikidata
-lang:wikidata
# maintenance: archive_logs
-06 23 1 * * jsub -l h_vmem=256M -quiet -N maint_archive fab -f
$HOME/fabfile.py -H localhost archive_logs
+06 23 1 * * jsub -l h_vmem=256M -quiet -o $HOME/public_html/logs/sge -j y -N
maint_archive fab -f $HOME/fabfile.py -H localhost archive_logs
diff --git a/fabfile.py b/fabfile.py
index 33cc119..0b0e32e 100644
--- a/fabfile.py
+++ b/fabfile.py
@@ -137,6 +137,7 @@
local('mkdir public_html/logs') # contains symlinks
local('echo AddType text/plain .log > public_html/logs/.htaccess')
local('mkdir public_html/logs/archive') # target for log archive
+ local('mkdir public_html/logs/sge') # sge stdout/-err files
else: # toolserver
local('mkdir public_html/doc') # contains symlinks
local('mkdir public_html/DrTrigonBot') # contains symlinks
@@ -371,6 +372,9 @@
def archive_logs():
""" A.A) Archive all log files on the server (all A.# steps) """
+ # how does this scheme/system interact with backup?
+ # https://wikitech.wikimedia.org/wiki/Nova_Resource:Tools/Help#Backups
+
#
http://serverlinux.blogspot.ch/2006/08/simple-rotation-backup-with-tar.html
# fecha has a formated date
import time
@@ -384,6 +388,11 @@
local('find ~/public_html/logs/archive/ -mtime +200 -exec rm {} \;')
local('find -L ~/public_html/logs/*/*.log.* -mtime +40 -exec rm {} \;')
+ # "Rotate" (delete old) sge stdout/-err files (they are NOT archived)
+ local('find ~/public_html/logs/sge/ -mtime +40 -exec rm {} \;')
+ # Remove all but the last 1000 lines from web access log (NOT archived)
+ local('tail -n -1000 ~/access.log > ~/access.log')
+
if (__name__ == '__main__') and ('sys' in locals()):
if sys.argv[3] in locals():
--
To view, visit https://gerrit.wikimedia.org/r/114749
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Icc19daa079a80d5fbd4f29bf718c2a9682c6a748
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/bots/drtrigonbot
Gerrit-Branch: master
Gerrit-Owner: DrTrigon <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
