Dear all
I would like to know how fast could reach the robinhood changelog read
speed. In a "normal" situation (when the waiting changelog records are
under 50000) our system perform well and the changelog speed change
between 2000 to 10000 record/s. Unfortunately sometime the lustre start
creating a huge amount of changelog (ca. 5000 record/s) and in this case
the robinhood read speed decrease to 1000/1500 record/s which cause an
increment of waiting changelog record to ca. 200M in 12h.
I tried different tuning but are not enough and the maximum I got is
1800 record/s. I'm not sure but I'm thinking the problem is on mysql
configuration (see below). Following some details concerning our server;
16 x CPU E5-2650 v2 @ 2.60GHz (in hyper-threading)
132 GB DDR3 memory
250 SSD disk
# cat /etc/my.cnf
[mysqld]
large-pages=true
datadir=/var/lib/mysql
socket=/var/lib/mysql/mysql.sock
user=mysql
symbolic-links=0
max_connections= 128
innodb_flush_log_at_trx_commit = 2
innodb_buffer_pool_size= 64G
innodb_max_dirty_pages_pct= 15
innodb_thread_concurrency= 0
innodb_log_file_size= 20M
innodb_log_buffer_size= 50M
innodb_data_file_path= ibdata1:1G:autoextend
innodb_lock_wait_timeout=120
innodb_additional_mem_pool_size = 16M
innodb_file_per_table = 1
innodb_flush_method=O_DIRECT
innodb_write_io_threads = 32
innodb_read_io_threads = 32
innodb_io_capacity=50000
innodb_log_files_in_group = 4
table-open-cache= 2000
sort-buffer-size= 32M
read-buffer-size= 16M
read-rnd-buffer-size= 4M
thread-cache-size= 128
query-cache-size= 40M
query-cache-limit= 1M
tmp-table-size= 16M
[mysqld_safe]
log-error=/var/log/mysqld.log
pid-file=/var/run/mysqld/mysqld.pid
# cat rbh_daily.conf
##########################################
# Robinhood configuration file template #
##########################################
# Global configuration
General
{
fs_path = "/scratch/daint" ;
lock_file = "/var/locks/robinhood.lock" ;
stay_in_fs = TRUE ;
check_mounted = TRUE ;
}
# Log configuration
Log
{
debug_level = EVENT ;
log_file = "/var/log/robinhood.daint.log" ;
report_file = "/var/log/robinhood_reports.daint.log" ;
alert_file = "/var/log/robinhood_alerts.daint.log" ;
alert_mail = "[email protected],[email protected]" ;
stats_interval = 1min ;
batch_alert_max = 5000 ;
alert_show_attrs = FALSE ;
log_procname = TRUE;
log_hostname = TRUE;
log_module = TRUE;
}
# List Manager configuration
ListManager
{
# Method for committing information to database.
commit_behavior = autocommit ;
connect_retry_interval_min = 1 ;
connect_retry_interval_max = 30 ;
# disable the following options if you are not interested in
# user or group stats (to speed up scan)
user_acct = enabled ;
group_acct = enabled ;
MySQL
{
server = "localhost" ;
db = "rbhdaint" ;
user = "robinhood" ;
password_file = "/etc/robinhood.d/.dbpassword" ;
engine = InnoDB ;
}
}
# Policies configuration
db_update_policy
{
md_update = on_event_periodic(1sec,1min) ;
path_update = on_event ;
}
# Entry Processor configuration
EntryProcessor
{
Alert Too_many_entries_in_directory
{
type == directory
and
dircount > 900000
}
Alert Large_file
{
type == file
and
size > 200GB
}
nb_threads = 8 ;
max_pending_operations = 100000 ;
max_batch_size = 1;
match_classes = TRUE;
detect_fake_mtime = FALSE;
}
# FS Scan configuration
FS_Scan
{
min_scan_interval = 12h ;
max_scan_interval = 1d ;
nb_threads_scan = 16 ;
scan_retry_delay = 1h ;
scan_op_timeout = 1h ;
exit_on_timeout = TRUE ;
spooler_check_interval = 1min ;
nb_prealloc_tasks = 256 ;
Ignore
{
type == directory
and
( name == ".snapdir" or name == ".snapshot" )
}
}
# ChangeLog Reader configuration
# Parameters for processing MDT changelogs :
ChangeLog
{
MDT
{
mdt_name = "MDT0000" ;
reader_id = "cl1" ;
}
batch_ack_count = 1024 ;
force_polling = ON ;
polling_interval = 1s ;
queue_max_size = 1000 ;
queue_max_age = 5s ;
queue_check_interval = 1s ;
}
Purge_Policies
{
policy default
{
condition { last_access > 30d }
}
}
Purge_Parameters
{
nb_threads_purge = 8 ;
post_purge_df_latency = 1min ;
}
Purge_Trigger
{
trigger_on = global_usage ;
high_watermark_pct = 59% ;
low_watermark_pct = 40% ;
check_interval = 24h ;
alert_high = TRUE;
notify_hw = TRUE;
alert_lw = TRUE;
}
rmdir_policy {
age_rm_empty_dirs = 30d ;
}
rmdir_parameters {
runtime_interval = 12h ;
nb_threads_rmdir = 8 ;
}
--
----------------------------------------------------------------------
Carmelo Ponti System Engineer
CSCS Swiss Center for Scientific Computing
Via Trevano 131 Email: [email protected]
CH-6900 Lugano http://www.cscs.ch
Phone: +41 91 610 82 15/Fax: +41 91 610 82 82
----------------------------------------------------------------------
------------------------------------------------------------------------------
One dashboard for servers and applications across Physical-Virtual-Cloud
Widest out-of-the-box monitoring support with 50+ applications
Performance metrics, stats and reports that give you Actionable Insights
Deep dive visibility with transaction tracing using APM Insight.
http://ad.doubleclick.net/ddm/clk/290420510;117567292;y
_______________________________________________
robinhood-support mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/robinhood-support
