[
https://issues.apache.org/jira/browse/TRAFODION-2617?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16127707#comment-16127707
]
ASF GitHub Bot commented on TRAFODION-2617:
-------------------------------------------
Github user sbroeder commented on a diff in the pull request:
https://github.com/apache/incubator-trafodion/pull/1206#discussion_r133238501
--- Diff:
core/sqf/src/seatrans/hbase-trx/src/main/java/org/apache/hadoop/hbase/coprocessor/transactional/TrxRegionEndpoint.java.tmpl
---
@@ -7335,6 +7338,245 @@ TrxTransactionState
isTransactionOnCommittedList(final long transactionId)
public void setClosing(boolean value) {
closing.set(value);
}
+
+ // The following are methods for the Trafodion SQL coprocessors.
+
+ // compares two qualifiers as unsigned, lexicographically ordered byte
strings
+
+ static private boolean isQualifierLessThanOrEqual(Cell nextKv, Cell
currKv) {
+ int currLength = currKv.getQualifierLength();
+ int currOffset = currKv.getQualifierOffset();
+ byte [] currQual = currKv.getQualifierArray();
+ int nextLength = nextKv.getQualifierLength();
+ int nextOffset = nextKv.getQualifierOffset();
+ byte [] nextQual = nextKv.getQualifierArray();
+
+ int minLength = nextLength;
+ if (currLength < nextLength)
+ minLength = currLength;
+
+ for (int i = 0; i < minLength; i++) {
+ // ugh... have to do some gymnastics to make this an
+ // unsigned comparison
+ int nextQualI = nextQual[i+nextOffset];
+ if (nextQualI < 0)
+ nextQualI = nextQualI + 256;
+ int currQualI = currQual[i+currOffset];
+ if (currQualI < 0)
+ currQualI = currQualI + 256;
+
+ if (nextQualI < currQualI)
+ return true;
+ else if (nextQualI > currQualI)
+ return false;
+ // else equal, move on to next byte
+ }
+
+ // the first minLength bytes are the same; the shorter array
+ // is regarded as less
+
+ boolean rc = (nextLength <= currLength);
+
+ return rc;
+ }
+
+
+ // debugging function
+
+ private static String bytesToHex(byte[] in) {
+ final StringBuilder builder = new StringBuilder();
+ for(byte b : in) {
+ builder.append(String.format("%02x", b));
+ }
+ return builder.toString();
+ }
+
+ // Returns data needed to estimate the row count in the table.
+ // Entry counts and total size in bytes are extracted from HFiles.
+ // For non-aligned tables (numCols > 1), sampling is done in order
+ // to estimate how many entries make up a row on average.
+
+ @Override
+ public void trafEstimateRowCount(RpcController controller,
+ TrafEstimateRowCountRequest request,
+ RpcCallback<TrafEstimateRowCountResponse>
done) {
+
+ TrafEstimateRowCountResponse response = null;
+ Throwable t = null;
+
+ int numCols = request.getNumCols();
+
+ // To estimate incidence of nulls, read the first 500 rows worth
+ // of KeyValues.
+ final int ROWS_TO_SAMPLE = 500;
+ int putKVsSampled = 0;
+ int nonPutKVsSampled = 0;
+ int missingKVsCount = 0;
+ int sampleRowCount = 0;
+ long totalEntries = 0; // KeyValues in all HFiles for table
+ long totalSizeBytes = 0; // Size of all HFiles for table
+ long estimatedTotalPuts = 0;
+ boolean more = true;
+ long estimatedRowCount = 0;
+
+ // Access the file system to go directly to the table's HFiles.
+ // Create a reader for the file to access the KV entry count and
+ // size in bytes stored in the trailer block.
+
+ // For aligned format tables, the number of rows equals the
+ // number of KeyValue entries. For non-aligned format, it's
+ // more complicated. There is a KeyValue entry for each
+ // column value, except the KeyValue may be missing because
+ // the column has a null value or because the column has a
+ // default value that has not been materialized.
+
+ // For non-aligned format tables, we sample some rows and
+ // count how many entries there are per row, so our caller
+ // can estimate the average number of missing values per row.
+ // Once our caller has that estimate, it can estimate the
+ // number of rows.
+
+ // We only do the sampling for non-aligned tables (numCols > 1),
+ // and we only do it on the first HFile of the first Region.
+ // The first Region is detected by having a null start key.
+
+ CacheConfig cacheConf = new CacheConfig(config);
+ byte[] startKey = regionInfo.getStartKey();
+
+ // Get the list of store files in column family '#1'. There might
+ // not be any. For example, a new Trafodion table might be entirely
+ // in memstore with nothing written out yet. Or we may be accessing
+ // a native HBase table which lacks the '#1' colum family.
+ List<String> storeFileList = null;
+ try {
+ byte[] familyName = "#1".getBytes();
+ byte[][] familyNames = { familyName };
+ storeFileList = m_Region.getStoreFileList(familyNames);
+ }
+ catch (IllegalArgumentException iae) {
+ // this gets thrown when the column family doesn't exist;
+ // we'll just use an empty list instead
+ storeFileList = new ArrayList<String>();
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Trafodion estimate row count sees " +
storeFileList.size() + " files.");
+ for (String sfn : storeFileList) {
+ LOG.debug("*** " + sfn);
+ }
+ if (startKey == null)
+ LOG.debug("startKey is null.");
+ else
+ LOG.debug("startKey.length is " + startKey.length + ", startKey is
hex " + bytesToHex(startKey));
--- End diff --
If start key is not empty, should we return early? Would eliminate extra
processing and testing below for regions we don't need, for example, they all
get the fileSystem and loop through the storeFileList.
> Error 9252 during update statistics of an encrypted Trafodion table
> -------------------------------------------------------------------
>
> Key: TRAFODION-2617
> URL: https://issues.apache.org/jira/browse/TRAFODION-2617
> Project: Apache Trafodion
> Issue Type: Bug
> Components: sql-cmp
> Affects Versions: 2.1-incubating
> Environment: Any, HBase encryption is enabled for the table.
> Reporter: Hans Zeller
> Assignee: David Wayne Birdsall
>
> Anu tried an update statistics command for a table that is using HBase
> encryption. That failed with the following stack trace, as printed
> >>update statistics for table t on every column sample;
> ..
> *** ERROR[9252] Unable to get row count estimate: Error code 68, detail 4.
> Exception info (if any):
> Instead of showing the exception info printed to stdout, I'm showing the
> contents of the ulog file:
> UPDATE STATISTICS
> =====================================================================
> [Wed 17 May 2017 10:38:30 PM UTC] update statistics for table t on every
> column sample;
> [Wed 17 May 2017 10:38:30 PM UTC] :BEGIN UpdateStats()
> [Wed 17 May 2017 10:38:30 PM UTC] :| BEGIN Setup CQDs prior to parsing
> [Wed 17 May 2017 10:38:30 PM UTC] CONTROL QUERY DEFAULT QUERY_CACHE '0'
> [Wed 17 May 2017 10:38:30 PM UTC] CONTROL QUERY DEFAULT CACHE_HISTOGRAMS 'OFF'
> [Wed 17 May 2017 10:38:30 PM UTC] CONTROL QUERY DEFAULT
> USTAT_MODIFY_DEFAULT_UEC '0.05'
> [Wed 17 May 2017 10:38:30 PM UTC] CONTROL QUERY DEFAULT OUTPUT_DATE_FORMAT
> 'ANSI'
> [Wed 17 May 2017 10:38:30 PM UTC] CONTROL QUERY DEFAULT
> HIST_MISSING_STATS_WARNING_LEVEL '0'
> [Wed 17 May 2017 10:38:30 PM UTC] CONTROL QUERY DEFAULT
> USTAT_AUTOMATION_INTERVAL '0'
> [Wed 17 May 2017 10:38:30 PM UTC] CONTROL QUERY DEFAULT
> MV_ALLOW_SELECT_SYSTEM_ADDED_COLUMNS 'ON'
> [Wed 17 May 2017 10:38:30 PM UTC] CONTROL QUERY DEFAULT
> HIST_ON_DEMAND_STATS_SIZE '0'
> [Wed 17 May 2017 10:38:30 PM UTC] CONTROL QUERY DEFAULT ISOLATION_LEVEL 'READ
> COMMITTED'
> [Wed 17 May 2017 10:38:30 PM UTC] CONTROL QUERY DEFAULT
> ALLOW_DML_ON_NONAUDITED_TABLE 'ON'
> [Wed 17 May 2017 10:38:30 PM UTC] CONTROL QUERY DEFAULT
> MV_ALLOW_SELECT_SYSTEM_ADDED_COLUMNS 'ON'
> [Wed 17 May 2017 10:38:30 PM UTC] CONTROL QUERY DEFAULT
> ALLOW_NULLABLE_UNIQUE_KEY_CONSTRAINT 'OFF'
> [Wed 17 May 2017 10:38:30 PM UTC] CONTROL QUERY DEFAULT
> CAT_ERROR_ON_NOTNULL_STOREBY 'ON'
> [Wed 17 May 2017 10:38:30 PM UTC] CONTROL QUERY DEFAULT
> WMS_CHILD_QUERY_MONITORING 'OFF'
> [Wed 17 May 2017 10:38:30 PM UTC] CONTROL QUERY DEFAULT WMS_QUERY_MONITORING
> 'OFF'
> [Wed 17 May 2017 10:38:30 PM UTC] CONTROL QUERY DEFAULT
> TRAF_TINYINT_RETURN_VALUES 'ON'
> [Wed 17 May 2017 10:38:30 PM UTC] CONTROL QUERY DEFAULT TRAF_BOOLEAN_IO 'ON'
> [Wed 17 May 2017 10:38:30 PM UTC] CONTROL QUERY DEFAULT
> TRAF_LARGEINT_UNSIGNED_IO 'ON'
> [Wed 17 May 2017 10:38:31 PM UTC] CONTROL QUERY DEFAULT
> TRAF_ALLOW_RESERVED_COLNAMES 'ON'
> [Wed 17 May 2017 10:38:31 PM UTC] CONTROL QUERY DEFAULT TRAF_BLOB_AS_VARCHAR
> 'OFF'
> [Wed 17 May 2017 10:38:31 PM UTC] CONTROL QUERY DEFAULT TRAF_CLOB_AS_VARCHAR
> 'OFF'
> [Wed 17 May 2017 10:38:31 PM UTC] :| END Setup CQDs prior to parsing
> elapsed time (00:00:00.420)
> [Wed 17 May 2017 10:38:31 PM UTC] :| BEGIN Parse statement
> [Wed 17 May 2017 10:38:31 PM UTC] call HSHbaseTableDef::objExists
> [Wed 17 May 2017 10:38:31 PM UTC] naTbl_->objectUid() is 6001738912217799228
> [Wed 17 May 2017 10:38:31 PM UTC] CONTROL QUERY DEFAULT
> DISPLAY_DIVISION_BY_COLUMNS RESET
> [Wed 17 May 2017 10:38:31 PM UTC]
> CHECK SCHEMA VERSION FOR TABLE: XXXXXXXXXXXX
> [Wed 17 May 2017 10:38:31 PM UTC]
> UpdateStats: TABLE: XXXXXXXXXXXX; SCHEMA VERSION: 2600; AUTOMATION INTERVAL: 0
> [Wed 17 May 2017 10:38:31 PM UTC] KEY:
> (_SALT_,PATH_ID,NAME_ID)
> [Wed 17 May 2017 10:38:31 PM UTC] GroupExists: argument: colSet
> [Wed 17 May 2017 10:38:31 PM UTC] colSet[0]: :_SALT_: 12
> [Wed 17 May 2017 10:38:31 PM UTC] colSet[1]: :PATH_ID: 0
> [Wed 17 May 2017 10:38:31 PM UTC] colSet[2]: :NAME_ID: 1
> [Wed 17 May 2017 10:38:31 PM UTC] KEY: (_SALT_,PATH_ID)
> [Wed 17 May 2017 10:38:31 PM UTC] GroupExists: argument: colSet
> [Wed 17 May 2017 10:38:31 PM UTC] colSet[0]: :_SALT_: 12
> [Wed 17 May 2017 10:38:31 PM UTC] colSet[1]: :PATH_ID: 0
> [Wed 17 May 2017 10:38:31 PM UTC] GroupExists: mgroup->colSet
> [Wed 17 May 2017 10:38:31 PM UTC] colSet[0]: :_SALT_: 12
> [Wed 17 May 2017 10:38:31 PM UTC] colSet[1]: :PATH_ID: 0
> [Wed 17 May 2017 10:38:31 PM UTC] colSet[2]: :NAME_ID: 1
> [Wed 17 May 2017 10:38:31 PM UTC] :| END Parse statement elapsed time
> (00:00:00.930)
> [Wed 17 May 2017 10:38:31 PM UTC]
> USTAT_CQDS_ALLOWED_FOR_SPAWNED_COMPILERS size of (0) is not acceptable
> [Wed 17 May 2017 10:38:31 PM UTC] :| BEGIN Initialize environment
> [Wed 17 May 2017 10:38:31 PM UTC] Creating histogram tables for schema
> TRAFODION.XXXXXXX on demand.
> [Wed 17 May 2017 10:38:31 PM UTC] :| | BEGIN Create histogram tables
> [Wed 17 May 2017 10:38:31 PM UTC] BEGIN WORK
> [Wed 17 May 2017 10:38:32 PM UTC] BEGINWORK(Create histogram tables.)
> [Wed 17 May 2017 10:38:32 PM UTC] Transaction started: 2017-05-17
> 22:38:32.007401
> [Wed 17 May 2017 10:38:33 PM UTC] :| | END Create histogram tables
> elapsed time (00:00:01.090)
> [Wed 17 May 2017 10:38:33 PM UTC] COMMIT WORK
> [Wed 17 May 2017 10:38:33 PM UTC] COMMITWORK()
> [Wed 17 May 2017 10:38:33 PM UTC] Transaction committed: 2017-05-17
> 22:38:33.099332
> [Wed 17 May 2017 10:38:33 PM UTC] :| | BEGIN getRowCount()
> [Wed 17 May 2017 10:38:33 PM UTC] :| | END getRowCount() elapsed time
> (00:00:00.065)
> [Wed 17 May 2017 10:38:33 PM UTC] currentRowCountIsEstimate_=1 from
> getRowCount()
> [Wed 17 May 2017 10:38:33 PM UTC] errorCode=68, breadCrumb=4
> [Wed 17 May 2017 10:38:33 PM UTC] JNI exception info:
> [Wed 17 May 2017 10:38:33 PM UTC]
> org.apache.hadoop.hbase.io.hfile.CorruptHFileException: Problem reading HFile
> Trailer from file
> hdfs://ip-172-31-65-71.ec2.internal:8020/apps/hbase/data/data/default/XXXXXXXXXX/00c6a0e9c39b98bd04f188647bd50253/#1/033b3f07b7c84725b5bc9e7aaf75eb54
> org.apache.hadoop.hbase.io.hfile.HFile.pickReaderVersion(HFile.java:481)
> org.apache.hadoop.hbase.io.hfile.HFile.createReader(HFile.java:524)
> org.trafodion.sql.HBaseClient.estimateRowCountBody(HBaseClient.java:1302)
> org.trafodion.sql.HBaseClient.estimateRowCount(HBaseClient.java:1207) Caused
> by
> java.lang.RuntimeException: java.lang.RuntimeException:
> java.io.FileNotFoundException: /etc/hbase/conf/hbase.jks (Permission denied)
> org.apache.hadoop.hbase.io.crypto.Encryption.getKeyProvider(Encryption.java:560)
> org.apache.hadoop.hbase.io.crypto.Encryption.getSecretKeyForSubject(Encryption.java:427)
> org.apache.hadoop.hbase.io.crypto.Encryption.decryptWithSubjectKey(Encryption.java:474)
> org.apache.hadoop.hbase.security.EncryptionUtil.getUnwrapKey(EncryptionUtil.java:129)
> org.apache.hadoop.hbase.security.EncryptionUtil.unwrapKey(EncryptionUtil.java:122)
> org.apache.hadoop.hbase.io.hfile.HFileReaderV3.createHFileContext(HFileReaderV3.java:107)
> org.apache.hadoop.hbase.io.hfile.HFileReaderV2.<init>(HFileReaderV2.java:130)
> org.apache.hadoop.hbase.io.hfile.HFileReaderV3.<init>(HFileReaderV3.java:77)
> org.apache.hadoop.hbase.io.hfile.HFile.pickReaderVersion(HFile.java:471)
> org.apache.hadoop.hbase.io.hfile.HFile.createReader(HFile.java:524)
> org.trafodion.sql.HBaseClient.estimateRowCountBody(HBaseClient.java:1302)
> org.trafodion.sql.HBaseClient.estimateRowCount(HBaseClient.java:1207) Caused
> by
> java.lang.RuntimeException: java.io.FileNotFoundException:
> /etc/hbase/conf/hbase.jks (Permission denied)
> org.apache.hadoop.hbase.io.crypto.KeyStoreKeyProvider.init(KeyStoreKeyProvider.java:153)
> org.apache.hadoop.hbase.io.crypto.Encryption.getKeyProvider(Encryption.java:553)
> org.apache.hadoop.hbase.io.crypto.Encryption.getSecretKeyForSubject(Encryption.java:427)
> org.apache.hadoop.hbase.io.crypto.Encryption.decryptWithSubjectKey(Encryption.java:474)
> org.apache.hadoop.hbase.security.EncryptionUtil.getUnwrapKey(EncryptionUtil.java:129)
> org.apache.hadoop.hbase.security.EncryptionUtil.unwrapKey(EncryptionUtil.java:122)
> org.apache.hadoop.hbase.io.hfile.HFileReaderV3.createHFileContext(HFileReaderV3.java:107)
> org.apache.hadoop.hbase.io.hfile.HFileReaderV2.<init>(HFileReaderV2.java:130)
> org.apache.hadoop.hbase.io.hfile.HFileReaderV3.<init>(HFileReaderV3.java:77)
> org.apache.hadoop.hbase.io.hfile.HFile.pickReaderVersion(HFile.java:471)
> org.apache.hadoop.hbase.io.hfile.HFile.createReader(HFile.java:524)
> org.trafodion.sql.HBaseClient.estimateRowCountBody(HBaseClient.java:1302)
> org.trafodion.sql.HBaseClient.estimateRowCount(HBaseClient.java:1207) Caused
> by
> java.io.FileNotFoundException: /etc/hbase/conf/hbase.jks (Permission denied)
> java.io.FileInputStream.open0(Native Method)
> java.io.FileInputStream.open(FileInputStream.java:195)
> java.io.FileInputStream.<init>(FileInputStream.java:138)
> org.apache.hadoop.hbase.io.crypto.KeyStoreKeyProvider.load(KeyStoreKeyProvider.java:124)
> org.apache.hadoop.hbase.io.crypto.KeyStoreKeyProvider.init(KeyStoreKeyProvider.java:147)
> org.apache.hadoop.hbase.io.crypto.Encryption.getKeyProvider(Encryption.java:553)
> org.apache.hadoop.hbase.io.crypto.Encryption.getSecretKeyForSubject(Encryption.java:427)
> org.apache.hadoop.hbase.io.crypto.Encryption.decryptWithSubjectKey(Encryption.java:474)
> org.apache.hadoop.hbase.security.EncryptionUtil.getUnwrapKey(EncryptionUtil.java:129)
> org.apache.hadoop.hbase.security.EncryptionUtil.unwrapKey(EncryptionUtil.java:122)
> org.apache.hadoop.hbase.io.hfile.HFileReaderV3.createHFileContext(HFileReaderV3.java:107)
> org.apache.hadoop.hbase.io.hfile.HFileReaderV2.<init>(HFileReaderV2.java:130)
> org.apache.hadoop.hbase.io.hfile.HFileReaderV3.<init>(HFileReaderV3.java:77)
> org.apache.hadoop.hbase.io.hfile.HFile.pickReaderVersion(HFile.java:471)
> org.apache.hadoop.hbase.io.hfile.HFile.createReader(HFile.java:524)
> org.trafodion.sql.HBaseClient.estimateRowCountBody(HBaseClient.java:1302)
> org.trafodion.sql.HBaseClient.estimateRowCount(HBaseClient.java:1207)
> [Wed 17 May 2017 10:38:33 PM UTC] :| END Initialize environment elapsed
> time (00:00:01.169)
> [Wed 17 May 2017 10:38:33 PM UTC] *** ERROR[-1] in hs_update:445
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
