[
https://issues.apache.org/jira/browse/TRAFODION-2195?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15450920#comment-15450920
]
QiaoYanke commented on TRAFODION-2195:
--------------------------------------
I read the data file and insert into trafodion through trafodion jdbc driver
using upsert using load, below is the code.
String event_file = "/root/event";
String line;
BufferedReader reader = new BufferedReader(new FileReader(event_file));
StringBuffer sb = new StringBuffer("upsert using load into
trafodion.seabase.event values ");
//upsert using load into trafodion.seabase.table1 values (1,'a'),
(2,'b');
int i = 1;
// int num = 139999;
int ln = 0;
int n = 0;
while ((line = reader.readLine()) != null) {
//date,‘sid’,uid,vid,e_time,e_pid,e_cat,e_act,e_lab,e_val,e_brf
ln++;
if (ln < nn) continue;
if (i == 1000) {
i = 0;
sb.deleteCharAt(sb.length() -1);
ps = conn.prepareStatement(sb.toString());
sb = new StringBuffer("upsert using load into
trafodion.seabase.event values ");
ps.executeBatch();
System.out.println(1000 * ++n);
// ps.close();
}
String[] lines = line.split(",", -1);
String values = "(timestamp '"+ lines[0] +" 00:00:00','09576b2a','"
+ lines[1] + "'," + lines[2] +"," + lines[3] + "," + lines[4] +"," + lines[5]
+",'" +
lines[6] +"','" + lines[7] +"','" + lines[8] +"'," +
lines[9] +"," + lines[10] + "),";
i++;
sb.append(values);
}
sb.deleteCharAt(sb.length() - 1);
ps = conn.prepareStatement(sb.toString());
ps.executeBatch();
ps.close();
conn.close();
> create Trafodion table with hbase options DATA_BLOCK_ENCODING and
> COMPRESSION together won't get all results when executing queries
> -------------------------------------------------------------------------------------------------------------------------------------
>
> Key: TRAFODION-2195
> URL: https://issues.apache.org/jira/browse/TRAFODION-2195
> Project: Apache Trafodion
> Issue Type: Bug
> Reporter: QiaoYanke
>
> I create a table using the follow ddl, and I insert into the table 1
> million lines, and the 'select count(*) from event' will get the right
> result. But when I execute 'select * from event', I only get several hundred
> lines.
> After I create the table just using 'HBASE_OPTIONS(
> COMPRESSION='snappy',
> MEMSTORE_FLUSH_SIZE = '1073741824') ' without data_block_encoding, I will get
> the data I insert .
> here is a link make help me just using compression.
> http://hadoop-hbase.blogspot.com/2016/02/hbase-compression-vs-blockencoding_17.html
> CREATE TABLE EVENT (
> v_date timestamp(6) NOT NULL,
> sid varchar(16 BYTES) NOT NULL DEFAULT '',
> uid varchar(20 BYTES) NOT NULL DEFAULT '',
> vid int unsigned NOT NULL ,
> idx tinyint NOT NULL,
> created_at int unsigned NOT NULL ,
> p_id bigint NOT NULL ,
> category varchar(255) CHARACTER SET UTF8 NOT NULL,
> e_action varchar(255) CHARACTER SET UTF8 NOT NULL,
> label varchar(255) CHARACTER SET UTF8 NOT NULL,
> e_value int NOT NULL,
> is_bounced tinyint NOT NULL DEFAULT 0,
> primary key (sid,v_date desc,uid,vid,idx)
> )
> salt using 4 partitions on (sid)
> division by (date_trunc('day', v_date))
> HBASE_OPTIONS( DATA_BLOCK_ENCODING = 'FAST_DIFF',
> COMPRESSION='snappy',
> MEMSTORE_FLUSH_SIZE = '1073741824');
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
