[
https://issues.apache.org/jira/browse/TRAFODION-2617?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16127708#comment-16127708
]
ASF GitHub Bot commented on TRAFODION-2617:
-------------------------------------------
Github user sbroeder commented on a diff in the pull request:
https://github.com/apache/incubator-trafodion/pull/1206#discussion_r133239797
--- Diff:
core/sqf/src/seatrans/hbase-trx/src/main/java/org/apache/hadoop/hbase/coprocessor/transactional/TrxRegionEndpoint.java.tmpl
---
@@ -7335,6 +7338,245 @@ TrxTransactionState
isTransactionOnCommittedList(final long transactionId)
public void setClosing(boolean value) {
closing.set(value);
}
+
+ // The following are methods for the Trafodion SQL coprocessors.
+
+ // compares two qualifiers as unsigned, lexicographically ordered byte
strings
+
+ static private boolean isQualifierLessThanOrEqual(Cell nextKv, Cell
currKv) {
+ int currLength = currKv.getQualifierLength();
+ int currOffset = currKv.getQualifierOffset();
+ byte [] currQual = currKv.getQualifierArray();
+ int nextLength = nextKv.getQualifierLength();
+ int nextOffset = nextKv.getQualifierOffset();
+ byte [] nextQual = nextKv.getQualifierArray();
+
+ int minLength = nextLength;
+ if (currLength < nextLength)
+ minLength = currLength;
+
+ for (int i = 0; i < minLength; i++) {
+ // ugh... have to do some gymnastics to make this an
+ // unsigned comparison
+ int nextQualI = nextQual[i+nextOffset];
+ if (nextQualI < 0)
+ nextQualI = nextQualI + 256;
+ int currQualI = currQual[i+currOffset];
+ if (currQualI < 0)
+ currQualI = currQualI + 256;
+
+ if (nextQualI < currQualI)
+ return true;
+ else if (nextQualI > currQualI)
+ return false;
+ // else equal, move on to next byte
+ }
+
+ // the first minLength bytes are the same; the shorter array
+ // is regarded as less
+
+ boolean rc = (nextLength <= currLength);
+
+ return rc;
+ }
+
+
+ // debugging function
+
+ private static String bytesToHex(byte[] in) {
+ final StringBuilder builder = new StringBuilder();
+ for(byte b : in) {
+ builder.append(String.format("%02x", b));
+ }
+ return builder.toString();
+ }
+
+ // Returns data needed to estimate the row count in the table.
+ // Entry counts and total size in bytes are extracted from HFiles.
+ // For non-aligned tables (numCols > 1), sampling is done in order
+ // to estimate how many entries make up a row on average.
+
+ @Override
+ public void trafEstimateRowCount(RpcController controller,
+ TrafEstimateRowCountRequest request,
+ RpcCallback<TrafEstimateRowCountResponse>
done) {
+
+ TrafEstimateRowCountResponse response = null;
+ Throwable t = null;
+
+ int numCols = request.getNumCols();
+
+ // To estimate incidence of nulls, read the first 500 rows worth
+ // of KeyValues.
+ final int ROWS_TO_SAMPLE = 500;
+ int putKVsSampled = 0;
+ int nonPutKVsSampled = 0;
+ int missingKVsCount = 0;
+ int sampleRowCount = 0;
+ long totalEntries = 0; // KeyValues in all HFiles for table
+ long totalSizeBytes = 0; // Size of all HFiles for table
+ long estimatedTotalPuts = 0;
+ boolean more = true;
+ long estimatedRowCount = 0;
+
+ // Access the file system to go directly to the table's HFiles.
+ // Create a reader for the file to access the KV entry count and
+ // size in bytes stored in the trailer block.
+
+ // For aligned format tables, the number of rows equals the
+ // number of KeyValue entries. For non-aligned format, it's
+ // more complicated. There is a KeyValue entry for each
+ // column value, except the KeyValue may be missing because
+ // the column has a null value or because the column has a
+ // default value that has not been materialized.
+
+ // For non-aligned format tables, we sample some rows and
+ // count how many entries there are per row, so our caller
+ // can estimate the average number of missing values per row.
+ // Once our caller has that estimate, it can estimate the
+ // number of rows.
+
+ // We only do the sampling for non-aligned tables (numCols > 1),
+ // and we only do it on the first HFile of the first Region.
+ // The first Region is detected by having a null start key.
+
+ CacheConfig cacheConf = new CacheConfig(config);
+ byte[] startKey = regionInfo.getStartKey();
+
+ // Get the list of store files in column family '#1'. There might
+ // not be any. For example, a new Trafodion table might be entirely
+ // in memstore with nothing written out yet. Or we may be accessing
+ // a native HBase table which lacks the '#1' colum family.
+ List<String> storeFileList = null;
+ try {
+ byte[] familyName = "#1".getBytes();
+ byte[][] familyNames = { familyName };
+ storeFileList = m_Region.getStoreFileList(familyNames);
+ }
+ catch (IllegalArgumentException iae) {
+ // this gets thrown when the column family doesn't exist;
+ // we'll just use an empty list instead
+ storeFileList = new ArrayList<String>();
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Trafodion estimate row count sees " +
storeFileList.size() + " files.");
+ for (String sfn : storeFileList) {
+ LOG.debug("*** " + sfn);
+ }
+ if (startKey == null)
+ LOG.debug("startKey is null.");
+ else
+ LOG.debug("startKey.length is " + startKey.length + ", startKey is
hex " + bytesToHex(startKey));
+ }
+
+ try {
+ FileSystem fileSystem = FileSystem.get(config);
+
+ for (String storeFileName : storeFileList) {
+ Path path = new Path(storeFileName);
+ // Make sure the file name conforms to HFile name pattern.
+ if (!StoreFileInfo.isHFile(path)) {
+ if (LOG.isDebugEnabled()) LOG.debug("Trafodion estimate row
count file name " + path + " is not an HFile.");
+ continue;
+ }
+ HFile.Reader reader = HFile.createReader(fileSystem, path,
cacheConf, config);
+ try {
+ totalEntries += reader.getEntries();
+ totalSizeBytes += reader.length();
+ if (ROWS_TO_SAMPLE > 0 &&
+ numCols > 1 && // only need to do this for non-aligned
format
+ (startKey == null || startKey.length == 0) && // first
region only
+ totalEntries == reader.getEntries()) { // first file only
+
+ // Trafodion column qualifiers are ordinal numbers, but are
represented
+ // as varying length unsigned little-endian integers in
lexicographical
+ // order. So, for example, in a table with 260 columns, the
column
+ // qualifiers (if present) will be read in this order:
+ // 1 (x'01'), 257 (x'0101'), 2 (x'02'), 258 (x'0201'), 3
(x'03'),
+ // 259 (x'0301'), 4 (x'04'), 260 (x'0401'), 5 (x'05'), 6
(x'06'),
+ // 7 (x'07'), ...
+ // We have crossed the boundary to the next row if and only if
the
+ // next qualifier read is less than or equal to the previous,
+ // compared unsigned, lexicographically.
+
+ HFileScanner scanner = reader.getScanner(false, false, false);
+ scanner.seekTo(); //position at beginning of first data block
+
+ // the next line should succeed, as we know the HFile is
non-empty
+ Cell currKv = scanner.getKeyValue();
+ while ((more) && (currKv.getTypeByte() !=
KeyValue.Type.Put.getCode())) {
+ nonPutKVsSampled++;
+ more = scanner.next();
+ currKv = scanner.getKeyValue();
+ }
+ if (more) {
+ // now we have the first KeyValue in the HFile
+
+ int putKVsThisRow = 1;
+ putKVsSampled++;
+ sampleRowCount++; // we have at least one row
+ more = scanner.next();
+
+ while ((more) && (sampleRowCount <= ROWS_TO_SAMPLE)) {
+ Cell nextKv = scanner.getKeyValue();
+ if (nextKv.getTypeByte() == KeyValue.Type.Put.getCode()) {
+ if (isQualifierLessThanOrEqual(nextKv,currKv)) {
+ // we have crossed a row boundary
+ sampleRowCount++;
+ missingKVsCount += (numCols - putKVsThisRow);
+ putKVsThisRow = 1;
+ } else {
+ putKVsThisRow++;
+ }
+ currKv = nextKv;
+ putKVsSampled++;
+ } else {
+ nonPutKVsSampled++; // don't count these toward the
number
+ }
+ more = scanner.next();
+ }
+ }
+
+ if (sampleRowCount > ROWS_TO_SAMPLE) {
+ // we read one KeyValue beyond the ROWS_TO_SAMPLE-eth row, so
+ // adjust counts for that
+ putKVsSampled--;
+ sampleRowCount--;
+ }
+ } // code for first file
+ } catch (IOException exp1) {
+ if (LOG.isErrorEnabled()) LOG.error("Trafodion estimate row
count encountered exception ", exp1);
+ t = exp1;
+ } finally {
+ reader.close(false);
--- End diff --
Need to close the scanner?
> Error 9252 during update statistics of an encrypted Trafodion table
> -------------------------------------------------------------------
>
> Key: TRAFODION-2617
> URL: https://issues.apache.org/jira/browse/TRAFODION-2617
> Project: Apache Trafodion
> Issue Type: Bug
> Components: sql-cmp
> Affects Versions: 2.1-incubating
> Environment: Any, HBase encryption is enabled for the table.
> Reporter: Hans Zeller
> Assignee: David Wayne Birdsall
>
> Anu tried an update statistics command for a table that is using HBase
> encryption. That failed with the following stack trace, as printed
> >>update statistics for table t on every column sample;
> ..
> *** ERROR[9252] Unable to get row count estimate: Error code 68, detail 4.
> Exception info (if any):
> Instead of showing the exception info printed to stdout, I'm showing the
> contents of the ulog file:
> UPDATE STATISTICS
> =====================================================================
> [Wed 17 May 2017 10:38:30 PM UTC] update statistics for table t on every
> column sample;
> [Wed 17 May 2017 10:38:30 PM UTC] :BEGIN UpdateStats()
> [Wed 17 May 2017 10:38:30 PM UTC] :| BEGIN Setup CQDs prior to parsing
> [Wed 17 May 2017 10:38:30 PM UTC] CONTROL QUERY DEFAULT QUERY_CACHE '0'
> [Wed 17 May 2017 10:38:30 PM UTC] CONTROL QUERY DEFAULT CACHE_HISTOGRAMS 'OFF'
> [Wed 17 May 2017 10:38:30 PM UTC] CONTROL QUERY DEFAULT
> USTAT_MODIFY_DEFAULT_UEC '0.05'
> [Wed 17 May 2017 10:38:30 PM UTC] CONTROL QUERY DEFAULT OUTPUT_DATE_FORMAT
> 'ANSI'
> [Wed 17 May 2017 10:38:30 PM UTC] CONTROL QUERY DEFAULT
> HIST_MISSING_STATS_WARNING_LEVEL '0'
> [Wed 17 May 2017 10:38:30 PM UTC] CONTROL QUERY DEFAULT
> USTAT_AUTOMATION_INTERVAL '0'
> [Wed 17 May 2017 10:38:30 PM UTC] CONTROL QUERY DEFAULT
> MV_ALLOW_SELECT_SYSTEM_ADDED_COLUMNS 'ON'
> [Wed 17 May 2017 10:38:30 PM UTC] CONTROL QUERY DEFAULT
> HIST_ON_DEMAND_STATS_SIZE '0'
> [Wed 17 May 2017 10:38:30 PM UTC] CONTROL QUERY DEFAULT ISOLATION_LEVEL 'READ
> COMMITTED'
> [Wed 17 May 2017 10:38:30 PM UTC] CONTROL QUERY DEFAULT
> ALLOW_DML_ON_NONAUDITED_TABLE 'ON'
> [Wed 17 May 2017 10:38:30 PM UTC] CONTROL QUERY DEFAULT
> MV_ALLOW_SELECT_SYSTEM_ADDED_COLUMNS 'ON'
> [Wed 17 May 2017 10:38:30 PM UTC] CONTROL QUERY DEFAULT
> ALLOW_NULLABLE_UNIQUE_KEY_CONSTRAINT 'OFF'
> [Wed 17 May 2017 10:38:30 PM UTC] CONTROL QUERY DEFAULT
> CAT_ERROR_ON_NOTNULL_STOREBY 'ON'
> [Wed 17 May 2017 10:38:30 PM UTC] CONTROL QUERY DEFAULT
> WMS_CHILD_QUERY_MONITORING 'OFF'
> [Wed 17 May 2017 10:38:30 PM UTC] CONTROL QUERY DEFAULT WMS_QUERY_MONITORING
> 'OFF'
> [Wed 17 May 2017 10:38:30 PM UTC] CONTROL QUERY DEFAULT
> TRAF_TINYINT_RETURN_VALUES 'ON'
> [Wed 17 May 2017 10:38:30 PM UTC] CONTROL QUERY DEFAULT TRAF_BOOLEAN_IO 'ON'
> [Wed 17 May 2017 10:38:30 PM UTC] CONTROL QUERY DEFAULT
> TRAF_LARGEINT_UNSIGNED_IO 'ON'
> [Wed 17 May 2017 10:38:31 PM UTC] CONTROL QUERY DEFAULT
> TRAF_ALLOW_RESERVED_COLNAMES 'ON'
> [Wed 17 May 2017 10:38:31 PM UTC] CONTROL QUERY DEFAULT TRAF_BLOB_AS_VARCHAR
> 'OFF'
> [Wed 17 May 2017 10:38:31 PM UTC] CONTROL QUERY DEFAULT TRAF_CLOB_AS_VARCHAR
> 'OFF'
> [Wed 17 May 2017 10:38:31 PM UTC] :| END Setup CQDs prior to parsing
> elapsed time (00:00:00.420)
> [Wed 17 May 2017 10:38:31 PM UTC] :| BEGIN Parse statement
> [Wed 17 May 2017 10:38:31 PM UTC] call HSHbaseTableDef::objExists
> [Wed 17 May 2017 10:38:31 PM UTC] naTbl_->objectUid() is 6001738912217799228
> [Wed 17 May 2017 10:38:31 PM UTC] CONTROL QUERY DEFAULT
> DISPLAY_DIVISION_BY_COLUMNS RESET
> [Wed 17 May 2017 10:38:31 PM UTC]
> CHECK SCHEMA VERSION FOR TABLE: XXXXXXXXXXXX
> [Wed 17 May 2017 10:38:31 PM UTC]
> UpdateStats: TABLE: XXXXXXXXXXXX; SCHEMA VERSION: 2600; AUTOMATION INTERVAL: 0
> [Wed 17 May 2017 10:38:31 PM UTC] KEY:
> (_SALT_,PATH_ID,NAME_ID)
> [Wed 17 May 2017 10:38:31 PM UTC] GroupExists: argument: colSet
> [Wed 17 May 2017 10:38:31 PM UTC] colSet[0]: :_SALT_: 12
> [Wed 17 May 2017 10:38:31 PM UTC] colSet[1]: :PATH_ID: 0
> [Wed 17 May 2017 10:38:31 PM UTC] colSet[2]: :NAME_ID: 1
> [Wed 17 May 2017 10:38:31 PM UTC] KEY: (_SALT_,PATH_ID)
> [Wed 17 May 2017 10:38:31 PM UTC] GroupExists: argument: colSet
> [Wed 17 May 2017 10:38:31 PM UTC] colSet[0]: :_SALT_: 12
> [Wed 17 May 2017 10:38:31 PM UTC] colSet[1]: :PATH_ID: 0
> [Wed 17 May 2017 10:38:31 PM UTC] GroupExists: mgroup->colSet
> [Wed 17 May 2017 10:38:31 PM UTC] colSet[0]: :_SALT_: 12
> [Wed 17 May 2017 10:38:31 PM UTC] colSet[1]: :PATH_ID: 0
> [Wed 17 May 2017 10:38:31 PM UTC] colSet[2]: :NAME_ID: 1
> [Wed 17 May 2017 10:38:31 PM UTC] :| END Parse statement elapsed time
> (00:00:00.930)
> [Wed 17 May 2017 10:38:31 PM UTC]
> USTAT_CQDS_ALLOWED_FOR_SPAWNED_COMPILERS size of (0) is not acceptable
> [Wed 17 May 2017 10:38:31 PM UTC] :| BEGIN Initialize environment
> [Wed 17 May 2017 10:38:31 PM UTC] Creating histogram tables for schema
> TRAFODION.XXXXXXX on demand.
> [Wed 17 May 2017 10:38:31 PM UTC] :| | BEGIN Create histogram tables
> [Wed 17 May 2017 10:38:31 PM UTC] BEGIN WORK
> [Wed 17 May 2017 10:38:32 PM UTC] BEGINWORK(Create histogram tables.)
> [Wed 17 May 2017 10:38:32 PM UTC] Transaction started: 2017-05-17
> 22:38:32.007401
> [Wed 17 May 2017 10:38:33 PM UTC] :| | END Create histogram tables
> elapsed time (00:00:01.090)
> [Wed 17 May 2017 10:38:33 PM UTC] COMMIT WORK
> [Wed 17 May 2017 10:38:33 PM UTC] COMMITWORK()
> [Wed 17 May 2017 10:38:33 PM UTC] Transaction committed: 2017-05-17
> 22:38:33.099332
> [Wed 17 May 2017 10:38:33 PM UTC] :| | BEGIN getRowCount()
> [Wed 17 May 2017 10:38:33 PM UTC] :| | END getRowCount() elapsed time
> (00:00:00.065)
> [Wed 17 May 2017 10:38:33 PM UTC] currentRowCountIsEstimate_=1 from
> getRowCount()
> [Wed 17 May 2017 10:38:33 PM UTC] errorCode=68, breadCrumb=4
> [Wed 17 May 2017 10:38:33 PM UTC] JNI exception info:
> [Wed 17 May 2017 10:38:33 PM UTC]
> org.apache.hadoop.hbase.io.hfile.CorruptHFileException: Problem reading HFile
> Trailer from file
> hdfs://ip-172-31-65-71.ec2.internal:8020/apps/hbase/data/data/default/XXXXXXXXXX/00c6a0e9c39b98bd04f188647bd50253/#1/033b3f07b7c84725b5bc9e7aaf75eb54
> org.apache.hadoop.hbase.io.hfile.HFile.pickReaderVersion(HFile.java:481)
> org.apache.hadoop.hbase.io.hfile.HFile.createReader(HFile.java:524)
> org.trafodion.sql.HBaseClient.estimateRowCountBody(HBaseClient.java:1302)
> org.trafodion.sql.HBaseClient.estimateRowCount(HBaseClient.java:1207) Caused
> by
> java.lang.RuntimeException: java.lang.RuntimeException:
> java.io.FileNotFoundException: /etc/hbase/conf/hbase.jks (Permission denied)
> org.apache.hadoop.hbase.io.crypto.Encryption.getKeyProvider(Encryption.java:560)
> org.apache.hadoop.hbase.io.crypto.Encryption.getSecretKeyForSubject(Encryption.java:427)
> org.apache.hadoop.hbase.io.crypto.Encryption.decryptWithSubjectKey(Encryption.java:474)
> org.apache.hadoop.hbase.security.EncryptionUtil.getUnwrapKey(EncryptionUtil.java:129)
> org.apache.hadoop.hbase.security.EncryptionUtil.unwrapKey(EncryptionUtil.java:122)
> org.apache.hadoop.hbase.io.hfile.HFileReaderV3.createHFileContext(HFileReaderV3.java:107)
> org.apache.hadoop.hbase.io.hfile.HFileReaderV2.<init>(HFileReaderV2.java:130)
> org.apache.hadoop.hbase.io.hfile.HFileReaderV3.<init>(HFileReaderV3.java:77)
> org.apache.hadoop.hbase.io.hfile.HFile.pickReaderVersion(HFile.java:471)
> org.apache.hadoop.hbase.io.hfile.HFile.createReader(HFile.java:524)
> org.trafodion.sql.HBaseClient.estimateRowCountBody(HBaseClient.java:1302)
> org.trafodion.sql.HBaseClient.estimateRowCount(HBaseClient.java:1207) Caused
> by
> java.lang.RuntimeException: java.io.FileNotFoundException:
> /etc/hbase/conf/hbase.jks (Permission denied)
> org.apache.hadoop.hbase.io.crypto.KeyStoreKeyProvider.init(KeyStoreKeyProvider.java:153)
> org.apache.hadoop.hbase.io.crypto.Encryption.getKeyProvider(Encryption.java:553)
> org.apache.hadoop.hbase.io.crypto.Encryption.getSecretKeyForSubject(Encryption.java:427)
> org.apache.hadoop.hbase.io.crypto.Encryption.decryptWithSubjectKey(Encryption.java:474)
> org.apache.hadoop.hbase.security.EncryptionUtil.getUnwrapKey(EncryptionUtil.java:129)
> org.apache.hadoop.hbase.security.EncryptionUtil.unwrapKey(EncryptionUtil.java:122)
> org.apache.hadoop.hbase.io.hfile.HFileReaderV3.createHFileContext(HFileReaderV3.java:107)
> org.apache.hadoop.hbase.io.hfile.HFileReaderV2.<init>(HFileReaderV2.java:130)
> org.apache.hadoop.hbase.io.hfile.HFileReaderV3.<init>(HFileReaderV3.java:77)
> org.apache.hadoop.hbase.io.hfile.HFile.pickReaderVersion(HFile.java:471)
> org.apache.hadoop.hbase.io.hfile.HFile.createReader(HFile.java:524)
> org.trafodion.sql.HBaseClient.estimateRowCountBody(HBaseClient.java:1302)
> org.trafodion.sql.HBaseClient.estimateRowCount(HBaseClient.java:1207) Caused
> by
> java.io.FileNotFoundException: /etc/hbase/conf/hbase.jks (Permission denied)
> java.io.FileInputStream.open0(Native Method)
> java.io.FileInputStream.open(FileInputStream.java:195)
> java.io.FileInputStream.<init>(FileInputStream.java:138)
> org.apache.hadoop.hbase.io.crypto.KeyStoreKeyProvider.load(KeyStoreKeyProvider.java:124)
> org.apache.hadoop.hbase.io.crypto.KeyStoreKeyProvider.init(KeyStoreKeyProvider.java:147)
> org.apache.hadoop.hbase.io.crypto.Encryption.getKeyProvider(Encryption.java:553)
> org.apache.hadoop.hbase.io.crypto.Encryption.getSecretKeyForSubject(Encryption.java:427)
> org.apache.hadoop.hbase.io.crypto.Encryption.decryptWithSubjectKey(Encryption.java:474)
> org.apache.hadoop.hbase.security.EncryptionUtil.getUnwrapKey(EncryptionUtil.java:129)
> org.apache.hadoop.hbase.security.EncryptionUtil.unwrapKey(EncryptionUtil.java:122)
> org.apache.hadoop.hbase.io.hfile.HFileReaderV3.createHFileContext(HFileReaderV3.java:107)
> org.apache.hadoop.hbase.io.hfile.HFileReaderV2.<init>(HFileReaderV2.java:130)
> org.apache.hadoop.hbase.io.hfile.HFileReaderV3.<init>(HFileReaderV3.java:77)
> org.apache.hadoop.hbase.io.hfile.HFile.pickReaderVersion(HFile.java:471)
> org.apache.hadoop.hbase.io.hfile.HFile.createReader(HFile.java:524)
> org.trafodion.sql.HBaseClient.estimateRowCountBody(HBaseClient.java:1302)
> org.trafodion.sql.HBaseClient.estimateRowCount(HBaseClient.java:1207)
> [Wed 17 May 2017 10:38:33 PM UTC] :| END Initialize environment elapsed
> time (00:00:01.169)
> [Wed 17 May 2017 10:38:33 PM UTC] *** ERROR[-1] in hs_update:445
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
