[
https://issues.apache.org/jira/browse/CASSANDRA-11594?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15422580#comment-15422580
]
n0rad commented on CASSANDRA-11594:
-----------------------------------
Sorry It's actually 2 keyspaces :
{code}
CREATE KEYSPACE email_logs_query WITH replication = {'class':
'NetworkTopologyStrategy', 'DC1': '1', 'DC2': '3'} AND durable_writes = true;
CREATE TYPE email_logs_query.click_user_type (
created_at timeuuid,
link_category text
);
CREATE TABLE email_logs_query.emails (
email_id text,
user_id blob,
clicks list<frozen<click_user_type>>,
created_at timeuuid,
email text,
lang text,
opens list<timeuuid>,
sender text,
site_id int,
type text,
PRIMARY KEY (email_id, user_id)
) WITH CLUSTERING ORDER BY (user_id ASC)
AND bloom_filter_fp_chance = 0.1
AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
AND comment = ''
AND compaction = {'class':
'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy',
'max_threshold': '32', 'min_threshold': '4'}
AND compression = {'chunk_length_in_kb': '64', 'class':
'org.apache.cassandra.io.compress.LZ4Compressor'}
AND crc_check_chance = 1.0
AND dclocal_read_repair_chance = 0.1
AND default_time_to_live = 0
AND gc_grace_seconds = 864000
AND max_index_interval = 2048
AND memtable_flush_period_in_ms = 0
AND min_index_interval = 128
AND read_repair_chance = 0.0
AND speculative_retry = '99PERCENTILE';
CREATE MATERIALIZED VIEW email_logs_query.emails_by_user_id AS
SELECT *
FROM email_logs_query.emails
WHERE user_id IS NOT NULL AND created_at IS NOT NULL AND email_id IS NOT
NULL
PRIMARY KEY (user_id, created_at, email_id)
WITH CLUSTERING ORDER BY (created_at DESC, email_id ASC)
AND bloom_filter_fp_chance = 0.01
AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
AND comment = ''
AND compaction = {'class':
'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy',
'max_threshold': '32', 'min_threshold': '4'}
AND compression = {'chunk_length_in_kb': '64', 'class':
'org.apache.cassandra.io.compress.LZ4Compressor'}
AND crc_check_chance = 1.0
AND dclocal_read_repair_chance = 0.1
AND default_time_to_live = 0
AND gc_grace_seconds = 864000
AND max_index_interval = 2048
AND memtable_flush_period_in_ms = 0
AND min_index_interval = 128
AND read_repair_chance = 0.0
AND speculative_retry = '99PERCENTILE';
{code}
{code}
CREATE KEYSPACE email_logs_command WITH replication = {'class':
'NetworkTopologyStrategy', 'DC1': '1', 'DC2': '3'} AND durable_writes = true;
CREATE TABLE email_logs_command.events (
bucket bigint,
date timeuuid,
event_type text,
version int,
event text,
PRIMARY KEY (bucket, date, event_type, version)
) WITH CLUSTERING ORDER BY (date ASC, event_type ASC, version ASC)
AND bloom_filter_fp_chance = 0.01
AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
AND comment = ''
AND compaction = {'class':
'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy',
'max_threshold': '32', 'min_threshold': '4'}
AND compression = {'chunk_length_in_kb': '64', 'class':
'org.apache.cassandra.io.compress.LZ4Compressor'}
AND crc_check_chance = 1.0
AND dclocal_read_repair_chance = 0.1
AND default_time_to_live = 0
AND gc_grace_seconds = 864000
AND max_index_interval = 2048
AND memtable_flush_period_in_ms = 0
AND min_index_interval = 128
AND read_repair_chance = 0.0
AND speculative_retry = '99PERCENTILE';
{code}
> Too many open files on directories
> ----------------------------------
>
> Key: CASSANDRA-11594
> URL: https://issues.apache.org/jira/browse/CASSANDRA-11594
> Project: Cassandra
> Issue Type: Bug
> Components: Core
> Reporter: n0rad
> Priority: Critical
> Attachments: openfiles.zip, screenshot.png
>
>
> I have a 6 nodes cluster in prod in 3 racks.
> each node :
> - 4Gb commitlogs on 343 files
> - 275Gb data on 504 files
> On saturday, 1 node in each rack crash with with too many open files (seems
> to be the similar node in each rack).
> {code}
> lsof -n -p $PID give me 66899 out of 65826 max
> {code}
> it contains 64527 open directories (2371 uniq)
> a part of the list :
> {code}
> java 19076 root 2140r DIR 8,17 143360 4386718705
> /opt/stage2/pod-cassandra-aci-cassandra/rootfs/data/keyspaces/email_logs_query/emails-2d4abd00e9ea11e591199d740e07bd95
> java 19076 root 2141r DIR 8,17 143360 4386718705
> /opt/stage2/pod-cassandra-aci-cassandra/rootfs/data/keyspaces/email_logs_query/emails-2d4abd00e9ea11e591199d740e07bd95
> java 19076 root 2142r DIR 8,17 143360 4386718705
> /opt/stage2/pod-cassandra-aci-cassandra/rootfs/data/keyspaces/email_logs_query/emails-2d4abd00e9ea11e591199d740e07bd95
> java 19076 root 2143r DIR 8,17 143360 4386718705
> /opt/stage2/pod-cassandra-aci-cassandra/rootfs/data/keyspaces/email_logs_query/emails-2d4abd00e9ea11e591199d740e07bd95
> java 19076 root 2144r DIR 8,17 143360 4386718705
> /opt/stage2/pod-cassandra-aci-cassandra/rootfs/data/keyspaces/email_logs_query/emails-2d4abd00e9ea11e591199d740e07bd95
> java 19076 root 2145r DIR 8,17 143360 4386718705
> /opt/stage2/pod-cassandra-aci-cassandra/rootfs/data/keyspaces/email_logs_query/emails-2d4abd00e9ea11e591199d740e07bd95
> java 19076 root 2146r DIR 8,17 143360 4386718705
> /opt/stage2/pod-cassandra-aci-cassandra/rootfs/data/keyspaces/email_logs_query/emails-2d4abd00e9ea11e591199d740e07bd95
> java 19076 root 2147r DIR 8,17 143360 4386718705
> /opt/stage2/pod-cassandra-aci-cassandra/rootfs/data/keyspaces/email_logs_query/emails-2d4abd00e9ea11e591199d740e07bd95
> java 19076 root 2148r DIR 8,17 143360 4386718705
> /opt/stage2/pod-cassandra-aci-cassandra/rootfs/data/keyspaces/email_logs_query/emails-2d4abd00e9ea11e591199d740e07bd95
> java 19076 root 2149r DIR 8,17 143360 4386718705
> /opt/stage2/pod-cassandra-aci-cassandra/rootfs/data/keyspaces/email_logs_query/emails-2d4abd00e9ea11e591199d740e07bd95
> java 19076 root 2150r DIR 8,17 143360 4386718705
> /opt/stage2/pod-cassandra-aci-cassandra/rootfs/data/keyspaces/email_logs_query/emails-2d4abd00e9ea11e591199d740e07bd95
> java 19076 root 2151r DIR 8,17 143360 4386718705
> /opt/stage2/pod-cassandra-aci-cassandra/rootfs/data/keyspaces/email_logs_query/emails-2d4abd00e9ea11e591199d740e07bd95
> java 19076 root 2152r DIR 8,17 143360 4386718705
> /opt/stage2/pod-cassandra-aci-cassandra/rootfs/data/keyspaces/email_logs_query/emails-2d4abd00e9ea11e591199d740e07bd95
> java 19076 root 2153r DIR 8,17 143360 4386718705
> /opt/stage2/pod-cassandra-aci-cassandra/rootfs/data/keyspaces/email_logs_query/emails-2d4abd00e9ea11e591199d740e07bd95
> java 19076 root 2154r DIR 8,17 143360 4386718705
> /opt/stage2/pod-cassandra-aci-cassandra/rootfs/data/keyspaces/email_logs_query/emails-2d4abd00e9ea11e591199d740e07bd95
> java 19076 root 2155r DIR 8,17 143360 4386718705
> /opt/stage2/pod-cassandra-aci-cassandra/rootfs/data/keyspaces/email_logs_query/emails-2d4abd00e9ea11e591199d740e07bd95
> {code}
> The 3 others nodes crashes 4 hours later
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
