Hello Doug,
At first sight, your configuration looks good.
Could you run "rbh-report -e" on an entry that should be eligible to
rmdir and is not processed by the policy?
Thanks
Thomas
On 03/28/18 12:14, Hayman, Doug wrote:
Hello,
I have been using Robinhood 3.1-1 to cleanup old files from our
scratch file systems but when adding a rmdir policy to remove empty
directories it is not working. Scanning the file system with a fresh
database populates the target fileclass as expected and the database
entry-info for the directories seems to fit my policy deletion rules.
However, when running the policy, it completes successfully but
without removing any directories. This is how I am running the policy:
[xctrh00:~]$ sudo robinhood -f test --run=rmdir --once
When running the policy with log verbosity level DEBUG or FULL the
directories within the target fileclass are not in the output.
I have appended the configuration file below - any help with this
would be appreciated.
Many thanks
Doug
[xctrh00:~]$ cat /etc/robinhood.d/test.conf
General {
fs_path = "/work";
fs_type = lustre;
}
#### policy definitions ####
# include template policy definitions for legacy TMPFS flavor
%include "includes/tmpfs.inc"
# include template policy definitions for removing directories
%include "includes/rmdir.inc"
#### fileclass definitions ####
FileClass working_empty_dir {
definition { type == "directory" and dircount == 0 and tree ==
"/work/.lustre_home/research/working/d0*/*/jtmp/*" }
}
FileClass scratch_empty_dir {
definition { type == "directory" and dircount == 0 and tree ==
"/work/.lustre_multi/research/scratch/d0*/*/*" }
}
FileClass research_home {
definition { type == "file" and tree ==
"/work/.lustre_home/research/home/d0*/*" }
}
FileClass research_working {
definition { type == "file" and tree ==
"/work/.lustre_home/research/working/d0*/*" }
}
FileClass research_data {
definition { type == "file" and tree ==
"/work/.lustre_multi/research/data/d0*/*" }
}
FileClass research_scratch {
definition { type == "file" and tree ==
"/work/.lustre_multi/research/scratch/d0*/*" }
}
FileClass largedir {
definition { type == "directory" and dircount > 10000 }
}
FileClass hpc_admins {
definition { group == "crayadm" or group == "super_unix" }
}
#### Deleting old unused files ####
cleanup_rules {
# don't purge entries matching the following conditions
ignore { owner == "root" or type == "symlink" or type == "fifo" or
type == "socket" }
# don't purge files from the following fileclasses
ignore_fileclass = hpc_admins;
ignore_fileclass = research_home;
ignore_fileclass = research_data;
#ignore_fileclass = working_empty_dir;
ignore_fileclass = scratch_empty_dir;
# cleanup policy for files older than 30 days
rule clean_files {
target_fileclass = research_working;
target_fileclass = research_scratch;
condition { last_access > 30d }
}
}
# trigger cleanup policy every 24 hours
cleanup_trigger {
trigger_on = periodic;
check_interval = 24h;
}
############# rmdir rules ############
# specifies no default sort order should be applied to the policy
#rmdir_parameters {
# lru_sort_attr = none;
#}
# trigger rmdir policy every 24 hours
rmdir_trigger {
trigger_on = periodic;
check_interval = 24h;
}
rmdir_rules {
# don't remove directories matching the following conditions
ignore { owner == "root" or depth < 5 }
# don't remove directories from the following fileclasses
ignore_fileclass = hpc_admins;
ignore_fileclass = research_home;
#ignore_fileclass = research_working;
ignore_fileclass = research_data;
ignore_fileclass = research_scratch;
ignore_fileclass = scratch_empty_dir;
# remove empty directories after 30 days
rule rmdir_empty {
target_fileclass = working_empty_dir;
action = common.rmdir;
condition { last_mod > 30d }
}
}
#### end of policy rules #####
Log {
debug_level = EVENT;
log_file = "/var/log/robinhood/work_fs.log";
report_file = "/var/log/robinhood/reports.log";
alert_file = "/var/log/robinhood/alerts.log";
stats_interval = 5min;
log_hostname = TRUE;
}
ListManager {
commit_behavior = autocommit;
MySQL {
server = localhost;
db = robinhood_work;
user = robinhood;
password_file = /etc/robinhood.d/.dbpassword;
engine = InnoDB;
}
}
EntryProcessor {
nb_threads = 32;
STAGE_GET_FID_threads_max = 24;
STAGE_GET_INFO_DB_threads_max = 24;
STAGE_GET_INFO_FS_threads_max = 24;
STAGE_PRE_APPLY_threads_max = 24;
STAGE_DB_APPLY_threads_max = 24;
max_pending_operations = 100000;
max_batch_size = 1;
}
FS_Scan {
scan_interval = 1d;
nb_threads_scan = 8;
scan_op_timeout = 1h;
exit_on_timeout = TRUE;
spooler_check_interval = 1min;
Ignore {
# ignore large directories
tree == "/work/home/crayadm/lfsck-1M"
or
tree ==
"/work/.lustre_home/research/home/d03/xxxx/xxxx-2017/836597/data"
}
}
Changelog {
MDT {
mdt_name = "MDT0000";
reader_id = "cl2";
}
MDT {
mdt_name = "MDT0001";
reader_id = "cl1";
}
MDT {
mdt_name = "MDT0002";
reader_id = "cl1";
}
batch_ack_count = 1024;
force_polling = ON;
polling_interval = 1s;
queue_max_size = 1000;
queue_max_age = 5s;
queue_check_interval = 1s;
}
[xctrh00:~]$ cat /etc/robinhood.d/includes/rmdir.inc
# This rmdir policy applies to all directories (empty or non-empty)
# This allow applying policies to both empty and non empty directories
# in a single policy run.
# - Explicitely add conditions "dircount == 0" in targeted fileclass
# to apply action to empty directories only.
# - Explicitely set action = cmd("rm -fr {fullpath}") in the policy rules
# to recursively remove non-empty directories.
define_policy rmdir {
scope {type == directory}
status_manager = none;
default_action = common.rmdir;
default_lru_sort_attr = last_mod;
}
------------------------------------------------------------------------------
Check out the vibrant tech community on one of the world's most
engaging tech sites, Slashdot.org! http://sdm.link/slashdot
_______________________________________________
robinhood-support mailing list
robinhood-support@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/robinhood-support
------------------------------------------------------------------------------
Check out the vibrant tech community on one of the world's most
engaging tech sites, Slashdot.org! http://sdm.link/slashdot
_______________________________________________
robinhood-support mailing list
robinhood-support@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/robinhood-support
