Hello,
Seems like the purging is working. What is the 'periodic' in the trigger
portion of the file mean? Everytime we restart robinhood do we have to wait 4h
according to my config for the trigger to start? I know we can do it manually.
Thanks.
_____________________________________________
From: Matthew BETTINGER
Sent: Tuesday, April 17, 2018 1:43 PM
To: robinhood-supp...@lists.sf.net
Subject: Robinhood 3.1 not purging files
Hello still battling robinhood upgrade from 2.5 to 3.1 here.. We would like
to remove all files older than 14days from /lustre4/Users while preserving
symbolic links and empty directories. Hope someone can shed some light as to
why no purging is appearing to happen.
Here is my robinhood configuration file
[root@robinhood robinhood.d]# cat lustre4.conf
%include "includes/tmpfs.inc"
General {
fs_path = "/lustre4";
fs_type = lustre;
}
Log {
log_file = "/var/log/robinhood/lustre4.log";
report_file = "/var/log/robinhood/lustre4_actions.log";
alert_file = "/var/log/robinhood/lustre4_alerts.log";
}
ListManager {
MySQL {
server = localhost;
db = robinhood_lustre4;
user = robinhood;
password_file = /etc/robinhood.d/.dbpassword;
engine = innodb;
}
}
EntryProcessor {
nb_threads = 32;
STAGE_GET_FID_threads_max = 24;
STAGE_GET_INFO_DB_threads_max = 24;
STAGE_GET_INFO_FS_threads_max = 24;
STAGE_PRE_APPLY_threads_max = 24;
max_pending_operations = 100000;
max_batch_size = 1;
}
FS_Scan {
scan_interval = 1d;
nb_threads_scan = 12;
scan_op_timeout = 10h;
exit_on_timeout = TRUE;
spooler_check_interval = 1min;
}
# Lustre 2.x only
ChangeLog {
MDT {
mdt_name = "MDT0000";
reader_id = "cl1";
}
}
################################################################
# File classes. Define the scope of entries to be scanned #
################################################################
fileclass _scratch_ {
definition { tree == "/lustre4/Users" }
}
fileclass _projects_ {
definition { tree == "/lustre4/Projects" }
}
fileclass _shifter_ {
definition { tree == "/lustre4/shifter" }
}
fileclass _totem_ {
definition { tree == "/lustre4/TOTEM" }
}
################################################################
# File classes. Define the file criteria to be acted upon #
################################################################
fileclass _emptydirs_ {
definition { type == directory and dircount == 0 }
}
fileclass _hugefiles_ {
definition { type == file and size > 1TB }
}
fileclass _bigfiles_ {
definition { type == file and size >500GB and size < 1TB }
}
###############################################################
# Policies. These are actions that we apply to File classes #
###############################################################
# ----------Policy for /lustre4/scratch
# 1. Purge files not accessed in 14 days
# 2. Ignore Symbolic links
# 3. Ignore Empty directories
cleanup_rules {
ignore { owner == "root" or type == "symlink" }
ignore_fileclass = _projects_;
ignore_fileclass = _shifter_;
ignore_fileclass = _totem_;
ignore_fileclass = _emptydirs_;
rule clean_f {
target_fileclass = _scratch_;
condition { last_access > 14d }
}
}
###############################################################
# Trigger cleanup policy every 4 hours #
###############################################################
cleanup_trigger {
trigger_on = periodic;
check_interval = 4h;
}
Robinhood is running as a service after the database has been populated using a
scan once command
root 9362 0.0 0.0 112660 972 pts/0 S+ 13:32 0:00 grep
--color=auto robinhood
mysql 18325 3.4 3.9 4251584 642156 ? Sl Apr16 43:03
/usr/libexec/mysqld --basedir=/usr --datadir=/robinhood/mysql
--plugin-dir=/usr/lib64/mysql/plugin --log-error=/var/log/mariadb/mariadb.log
--open-files-limit=2048 --pid-file=/var/run/mariadb/mariadb.pid
--socket=/var/lib/mysql/mysql.sock
root 22060 0.0 0.1 2881100 20832 ? Ssl 08:14 0:09
/usr/sbin/robinhood --readlog --run=all
rbh-report shows some old files under the target /lustre4/Users which I thought
should be purged but it does not appear that the purge feature is happening.
rbh-report --oldest-files
Using config file '/etc/robinhood.d/lustre4.conf'.
rank, path, user, group,
last_access, last_mod, size, spc_used, stripe_cnt,
stripe_size, pool, stripes
1, /lustre4/scratch/l0363734/scratch/FULLDIR/rm_me, root, root,
2004/02/29 18:21:42, 2004/02/29 18:21:42, 0, 0, 8,
4.00 MB, , ost#15: 1230062, ost#4: 1226465, ost#23: 1226251,
ost#28: 1228816, ost#14: 1230075, ost#5: 1226684, ost#16: 1231164, ost#25:
1234553
2, /lustre4/scratch/j0432886/mpi-tile-io/COPYRIGHT, j0432886, rtstaff,
2015/02/09 15:26:31, 2015/02/09 15:26:31, 1.64 KB, 4.00 KB, 8,
4.00 MB, , ost#29: 416908, ost#9: 407942, ost#0: 415080, ost#22:
413832, ost#30: 417196, ost#11: 416678, ost#3: 414088, ost#17: 413644
3, /lustre4/scratch/j0432886/mpi-tile-io/Makefile.linux, j0432886,
rtstaff, 2015/02/09 15:26:31, 2015/02/09 15:26:31, 181, 4.00 KB,
8, 4.00 MB, , ost#10: 416712, ost#1: 414859, ost#21:
418605, ost#26: 417278, ost#13: 416552, ost#7: 415400, ost#18: 414047, ost#29:
416909
[root@robinhood robinhood]# cat lustre4.log
2018/04/17 13:38:53 [9940/1] CheckFS | '/lustre4' matches mount point
'/lustre4', type=lustre,
fs=172.40.2.60@o2ib4:172.40.2.61@o2ib4:/lustre4<mailto:fs=172.40.2.60@o2ib4:172.40.2.61@o2ib4:/lustre4>
2018/04/17 13:38:53 [9940/1] cleanup | Notice: no 'default' policy rule is
defined. Unmatched entries will be ignored.
2018/04/17 13:38:53 [9940/1] Main | Daemon started (running modules:
log_reader,policy_run(cleanup))
2018/04/17 13:38:53 [9940/2] cleanup | Checking policy rules for all
2018/04/17 13:38:53 [9940/2] cleanup | Building policy list - last full FS
Scan: 2018/04/16 16:51:22
2018/04/17 13:38:53 [9940/2] cleanup | Starting policy run
2018/04/17 13:38:56 [9940/2] cleanup | Policy run summary: time=01s;
target=all; 0 successful actions (0.00/sec); volume: 0 (0/sec); 0 entries
skipped; 0 errors.
[root@robinhood robinhood]#
cat /etc/sysconfig/robinhood
# directory of configuration files for this service
#RH_CONF_DIR=/etc/robinhood.d/
# current directory where the daemon is started
#RH_RUN_DIR=/
# path to robinhood command
#DAEMON=/usr/sbin/robinhood
# options for starting the daemon
# eg. --scan --purge (leave empty for default actions).
RBH_OPT="--readlog --run=all"
[root@robinhood robinhood]#
cat ../includes/tmpfs.inc
# used to be rbh 2.5 "purge" policy in TMPFS mode
define_policy cleanup {
scope { type != directory }
status_manager = none;
default_action = common.unlink;
default_lru_sort_attr = last_access;
}
Thanks for any insight. Would like to get this going since I had to reload our
old robinhood server and the FS is growing pretty crazy at the moment ;-)
Matt
------------------------------------------------------------------------------
Check out the vibrant tech community on one of the world's most
engaging tech sites, Slashdot.org! http://sdm.link/slashdot
_______________________________________________
robinhood-support mailing list
robinhood-support@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/robinhood-support
