[
https://issues.apache.org/jira/browse/CASSANDRA-12125?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15372623#comment-15372623
]
Relish Chackochan commented on CASSANDRA-12125:
-----------------------------------------------
Thank you for the replay.
Please find the Table structure and system.log entries attached.
Table Structure:
CREATE TABLE proddb.log_curve (
log_curve_uid text,
mnemonic text,
alternateintex text,
array_curve text,
class_index text,
class_proddb text,
column_index text,
curve_description text,
data_type text,
datasource text,
densdata text,
isindexmnemonic text,
lower_mnemonic text,
mapped_mnemonic_name text,
max_index text,
maxdatetimeindex text,
min_index text,
mindatetimeindex text,
mnem_alias text,
mnem_alias_namingsystem text,
null_value text,
sensor_offset text,
trace_state text,
traceorgin text,
uid text,
unit text,
welldatum text,
welldatum_uidref text,
PRIMARY KEY (log_curve_uid, mnemonic)
) WITH CLUSTERING ORDER BY (mnemonic ASC)
AND bloom_filter_fp_chance = 0.1
AND caching = '{"keys":"ALL", "rows_per_partition":"NONE"}'
AND comment = ''
AND compaction = {'class':
'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
AND compression = {'sstable_compression':
'org.apache.cassandra.io.compress.LZ4Compressor'}
AND dclocal_read_repair_chance = 0.1
AND default_time_to_live = 0
AND gc_grace_seconds = 172800
AND max_index_interval = 2048
AND memtable_flush_period_in_ms = 0
AND min_index_interval = 128
AND read_repair_chance = 0.0
AND speculative_retry = '99.0PERCENTILE';
CREATE INDEX mnem_alias_idx ON proddb.log_curve (mnem_alias);
CREATE INDEX class_index_idx ON proddb.log_curve (class_index);
CREATE INDEX traceorgin_idx ON proddb.log_curve (traceorgin);
CREATE INDEX mindatetimeindex_idx ON proddb.log_curve (mindatetimeindex);
CREATE INDEX userindex ON proddb.log_curve (isindexmnemonic);
CREATE INDEX curve_description_idx ON proddb.log_curve (curve_description);
CREATE INDEX lower_mnemonic_idx ON proddb.log_curve (lower_mnemonic);
CREATE INDEX datasource_idx ON proddb.log_curve (datasource);
CREATE INDEX unit_idx ON proddb.log_curve (unit);
CREATE INDEX mnem_alias_namingsystem_idx ON proddb.log_curve
(mnem_alias_namingsystem);
CREATE INDEX min_index_idx ON proddb.log_curve (min_index);
CREATE INDEX trace_state_idx ON proddb.log_curve (trace_state);
CREATE INDEX null_value_idx ON proddb.log_curve (null_value);
CREATE INDEX data_types_idx ON proddb.log_curve (data_type);
CREATE INDEX class_proddb_idx ON proddb.log_curve (class_proddb);
CREATE INDEX alternateintex_idx ON proddb.log_curve (alternateintex);
CREATE INDEX uid3_idx ON proddb.log_curve (uid);
CREATE INDEX maxdatetimeindex_idx ON proddb.log_curve (maxdatetimeindex);
CREATE INDEX welldatum_idx ON proddb.log_curve (welldatum);
CREATE INDEX max_index_idx ON proddb.log_curve (max_index);
system.log :
INFO [StreamReceiveTask:40] 2016-07-01 08:30:57,710
SecondaryIndexManager.java:164 - Submitting index build of [log_curve.uid3_idx,
log_curve.welldatum_idx, log_curve.curve_description_idx,
log_curve.min_index_idx, log_curve.class_proddb_idx, log_curve.traceorgin_idx,
log_curve.mindatetimeindex_idx, log_curve.unit_idx,
log_curve.lower_mnemonic_idx, log_curve.mnem_alias_namingsystem_idx,
log_curve.userindex, log_curve.mnem_alias_idx, log_curve.max_index_idx,
log_curve.trace_state_idx, log_curve.alternateintex_idx,
log_curve.class_index_idx, log_curve.null_value_idx, log_curve.datasource_idx,
log_curve.data_types_idx, log_curve.maxdatetimeindex_idx] for data in
BigTableReader(path='/opt/cassandra/data/proddb/log_curve-0a5092a0a4fa11e5872fc1ce0a46dc27/la-19388-big-Data.db')
INFO [StreamReceiveTask:40] 2016-07-01 08:30:59,928
SecondaryIndexManager.java:173 - Index build of [log_curve.uid3_idx,
log_curve.welldatum_idx, log_curve.curve_description_idx,
log_curve.min_index_idx, log_curve.class_proddb_idx, log_curve.traceorgin_idx,
log_curve.mindatetimeindex_idx, log_curve.unit_idx,
log_curve.lower_mnemonic_idx, log_curve.mnem_alias_namingsystem_idx,
log_curve.userindex, log_curve.mnem_alias_idx, log_curve.max_index_idx,
log_curve.trace_state_idx, log_curve.alternateintex_idx,
log_curve.class_index_idx, log_curve.null_value_idx, log_curve.datasource_idx,
log_curve.data_types_idx, log_curve.maxdatetimeindex_idx] complete
INFO [StreamReceiveTask:40] 2016-07-01 08:30:59,929
StreamResultFuture.java:182 - [Stream #076b4590-3f90-11e6-83b5-4542ce30ffc0]
Session with /192.168.1.135 is complete
INFO [StreamReceiveTask:40] 2016-07-01 08:30:59,929
StreamResultFuture.java:214 - [Stream #076b4590-3f90-11e6-83b5-4542ce30ffc0]
All sessions completed
INFO [StreamReceiveTask:40] 2016-07-01 08:30:59,929
StreamingRepairTask.java:85 - [repair #bb34f481-3f8c-11e6-a4b1-b1bbbbe8b805]
streaming task succeed, returning response to /192.168.1.132
INFO [STREAM-IN-/192.168.1.135] 2016-07-01 08:31:02,213
StreamResultFuture.java:168 - [Stream #084048d0-3f90-11e6-83b5-4542ce30ffc0
ID#0] Prepare completed. Receiving 3 files(26377 bytes), sending 2 files(26851
bytes)
INFO [StreamReceiveTask:40] 2016-07-01 08:31:02,675
SecondaryIndexManager.java:164 - Submitting index build of [log_curve.uid3_idx,
log_curve.welldatum_idx, log_curve.curve_description_idx,
log_curve.min_index_idx, log_curve.class_proddb_idx, log_curve.traceorgin_idx,
log_curve.mindatetimeindex_idx, log_curve.unit_idx,
log_curve.lower_mnemonic_idx, log_curve.mnem_alias_namingsystem_idx,
log_curve.userindex, log_curve.mnem_alias_idx, log_curve.max_index_idx,
log_curve.trace_state_idx, log_curve.alternateintex_idx,
log_curve.class_index_idx, log_curve.null_value_idx, log_curve.datasource_idx,
log_curve.data_types_idx, log_curve.maxdatetimeindex_idx] for data in
BigTableReader(path='/opt/cassandra/data/proddb/log_curve-0a5092a0a4fa11e5872fc1ce0a46dc27/la-19391-big-Data.db'),
BigTableReader(path='/opt/cassandra/data/proddb/log_curve-0a5092a0a4fa11e5872fc1ce0a46dc27/la-19392-big-Data.db'),
BigTableReader(path='/opt/cassandra/data/proddb/log_curve-0a5092a0a4fa11e5872fc1ce0a46dc27/la-19393-big-Data.db')
INFO [StreamReceiveTask:40] 2016-07-01 08:31:03,626
SecondaryIndexManager.java:173 - Index build of [log_curve.uid3_idx,
log_curve.welldatum_idx, log_curve.curve_description_idx,
log_curve.min_index_idx, log_curve.class_proddb_idx, log_curve.traceorgin_idx,
log_curve.mindatetimeindex_idx, log_curve.unit_idx,
log_curve.lower_mnemonic_idx, log_curve.mnem_alias_namingsystem_idx,
log_curve.userindex, log_curve.mnem_alias_idx, log_curve.max_index_idx,
log_curve.trace_state_idx, log_curve.alternateintex_idx,
log_curve.class_index_idx, log_curve.null_value_idx, log_curve.datasource_idx,
log_curve.data_types_idx, log_curve.maxdatetimeindex_idx] complete
INFO [STREAM-IN-/192.168.1.134] 2016-07-01 08:31:04,979
StreamResultFuture.java:182 - [Stream #03bdcdf0-3f90-11e6-83b5-4542ce30ffc0]
Session with /192.168.1.134 is complete
INFO [STREAM-IN-/192.168.1.134] 2016-07-01 08:31:04,979
StreamResultFuture.java:214 - [Stream #03bdcdf0-3f90-11e6-83b5-4542ce30ffc0]
All sessions completed
INFO [STREAM-IN-/192.168.1.134] 2016-07-01 08:31:04,979
StreamingRepairTask.java:85 - [repair #bb34a662-3f8c-11e6-a4b1-b1bbbbe8b805]
streaming task succeed, returning response to /192.168.1.132
INFO [STREAM-IN-/192.168.1.135] 2016-07-01 08:31:12,346
StreamResultFuture.java:182 - [Stream #084048d0-3f90-11e6-83b5-4542ce30ffc0]
Session with /192.168.1.135 is complete
INFO [STREAM-IN-/192.168.1.135] 2016-07-01 08:31:12,346
StreamResultFuture.java:214 - [Stream #084048d0-3f90-11e6-83b5-4542ce30ffc0]
All sessions completed
INFO [STREAM-IN-/192.168.1.135] 2016-07-01 08:31:12,346
StreamingRepairTask.java:85 - [repair #bb34f483-3f8c-11e6-a4b1-b1bbbbe8b805]
streaming task succeed, returning response to /192.168.1.132
INFO [CompactionExecutor:9] 2016-07-01 08:31:12,358 CompactionManager.java:491
- Starting anticompaction for proddb.log_curve on 0/2 sstables
INFO [CompactionExecutor:9] 2016-07-01 08:31:12,358 CompactionManager.java:549
- Completed anticompaction successfully
ERROR [MemtableFlushWriter:9] 2016-07-01 08:32:22,186 CassandraDaemon.java:185
- Exception in thread Thread[MemtableFlushWriter:9,5,main]
java.lang.RuntimeException: Last written key DecoratedKey(1467379751.0000,
313436373337393735312e30303030) >= current key
DecoratedKey(^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@, 313436373337393833312e38383730)
writing into
/opt/cassandra/data/proddb/log_curve-0a5092a0a4fa11e5872fc1ce0a46dc27/.maxdatetimeindex_idx/tmp-la-14120-big-Data.db
at
org.apache.cassandra.io.sstable.format.big.BigTableWriter.beforeAppend(BigTableWriter.java:108)
~[apache-cassandra-2.2.5.jar:2.2.5]
at
org.apache.cassandra.io.sstable.format.big.BigTableWriter.append(BigTableWriter.java:162)
~[apache-cassandra-2.2.5.jar:2.2.5]
at
org.apache.cassandra.db.Memtable$FlushRunnable.writeSortedContents(Memtable.java:401)
~[apache-cassandra-2.2.5.jar:2.2.5]
at
org.apache.cassandra.db.Memtable$FlushRunnable.runMayThrow(Memtable.java:361)
~[apache-cassandra-2.2.5.jar:2.2.5]
at
org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28)
~[apache-cassandra-2.2.5.jar:2.2.5]
at
com.google.common.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:297)
~[guava-16.0.jar:na]
at
org.apache.cassandra.db.ColumnFamilyStore$Flush.run(ColumnFamilyStore.java:1139)
~[apache-cassandra-2.2.5.jar:2.2.5]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
~[na:1.8.0_77]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
~[na:1.8.0_77]
at java.lang.Thread.run(Thread.java:745) ~[na:1.8.0_77]
INFO [IndexSummaryManager:1] 2016-07-01 08:32:46,554
IndexSummaryRedistribution.java:74 - Redistributing index summaries
INFO [IndexSummaryManager:1] 2016-07-01 09:32:46,579
IndexSummaryRedistribution.java:74 - Redistributing index summaries
INFO [IndexSummaryManager:1] 2016-07-01 10:32:46,594
IndexSummaryRedistribution.java:74 - Redistributing index summaries
> ERROR [MemtableFlushWriter:4] 2016-07-01 06:20:41,137
> CassandraDaemon.java:185 - Exception in thread
> Thread[MemtableFlushWriter:4,5,main] java.lang.RuntimeException: Last
> written key DecoratedKey(XXXX.XX, XXXXX) >= current key DecoratedKey
> ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
>
> Key: CASSANDRA-12125
> URL: https://issues.apache.org/jira/browse/CASSANDRA-12125
> Project: Cassandra
> Issue Type: Bug
> Components: Core
> Environment: RHEL-6.5 64-bit Apache Cassandra 2.2.5v
> Reporter: Relish Chackochan
> Fix For: 2.2.x
>
>
> We are running on RHEL-6.5 64-bit with Apache Cassandra 2.2.5v on 4 node
> cluster and getting the following error on multiple node while running the
> repair job and when getting the error repair job is hang.
> Can some one help to identify the issue.
> {code}
> ERROR [MemtableFlushWriter:4] 2016-07-01 06:20:41,137
> CassandraDaemon.java:185 - Exception in thread
> Thread[MemtableFlushWriter:4,5,main]
> java.lang.RuntimeException: Last written key DecoratedKey(1467371986.8870,
> 313436373337313938362e38383730) >= current key DecoratedKey(,
> 313436373337323030312e38383730) writing into
> /opt/cassandra/data/proddb/log_data1-0a5092a0a4fa11e5872fc1ce0a46dc27/.maxdatetimeindex_idx/tmp-la-470-big-Data.db
> {code}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
