This is an automated email from the ASF dual-hosted git repository.
tkhurana pushed a commit to branch PHOENIX-7170
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/PHOENIX-7170 by this push:
new 20577821f1 Support masking for Conditional TTL (#2002)
20577821f1 is described below
commit 20577821f18dc92c0c9f63d98db8d329019781d7
Author: tkhurana <[email protected]>
AuthorDate: Mon Nov 18 19:47:45 2024 -0800
Support masking for Conditional TTL (#2002)
* Support masking for Conditional TTL
* Fix re-write logic
* Renamed files
* Add comment to explain the query re-write for condition ttl masking
* In-List tests and literals util
* Partial index test
* Ignoring condition TTL filter for raw scans
* Checkstyle fixes
---
.../org/apache/phoenix/compile/QueryCompiler.java | 51 ++-
.../phoenix/compile/ServerBuildIndexCompiler.java | 7 +
.../phoenix/schema/ConditionTTLExpression.java | 23 +-
.../org/apache/phoenix/schema/DelegateTable.java | 5 +
.../java/org/apache/phoenix/schema/PTable.java | 5 +
.../java/org/apache/phoenix/schema/PTableImpl.java | 5 +
.../PhoenixServerBuildIndexInputFormat.java | 7 +
.../apache/phoenix/end2end/TTLAsPhoenixTTLIT.java | 12 +-
.../PhoenixServerBuildIndexInputFormatIT.java | 25 +-
.../phoenix/schema/ConditionTTLExpressionIT.java | 195 +++++++++
.../phoenix/schema/ConditionTTLExpressionTest.java | 446 +++++++++++++++++++++
.../schema/ConditionalTTLExpressionDDLTest.java | 221 ----------
.../java/org/apache/phoenix/util/TestUtil.java | 12 +
13 files changed, 772 insertions(+), 242 deletions(-)
diff --git
a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/QueryCompiler.java
b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/QueryCompiler.java
index 2d15206789..45cb435cff 100644
---
a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/QueryCompiler.java
+++
b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/QueryCompiler.java
@@ -23,18 +23,13 @@ import static
org.apache.phoenix.query.QueryServicesOptions.DEFAULT_WILDCARD_QUE
import java.sql.SQLException;
import java.sql.SQLFeatureNotSupportedException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.phoenix.expression.function.PhoenixRowTimestampFunction;
-import org.apache.phoenix.parse.HintNode;
-import org.apache.phoenix.parse.NamedTableNode;
-import org.apache.phoenix.parse.TerminalParseNode;
-import org.apache.phoenix.schema.PTableType;
-import org.apache.phoenix.schema.SortOrder;
-import org.apache.phoenix.thirdparty.com.google.common.base.Optional;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.phoenix.compile.GroupByCompiler.GroupBy;
@@ -60,6 +55,7 @@ import org.apache.phoenix.execute.UnionPlan;
import org.apache.phoenix.expression.Expression;
import org.apache.phoenix.expression.LiteralExpression;
import org.apache.phoenix.expression.RowValueConstructorExpression;
+import org.apache.phoenix.expression.function.PhoenixRowTimestampFunction;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.iterate.ParallelIteratorFactory;
import org.apache.phoenix.jdbc.PhoenixConnection;
@@ -68,8 +64,10 @@ import org.apache.phoenix.join.HashJoinInfo;
import org.apache.phoenix.optimize.Cost;
import org.apache.phoenix.parse.AliasedNode;
import org.apache.phoenix.parse.EqualParseNode;
+import org.apache.phoenix.parse.HintNode;
import org.apache.phoenix.parse.HintNode.Hint;
import org.apache.phoenix.parse.JoinTableNode.JoinType;
+import org.apache.phoenix.parse.NamedTableNode;
import org.apache.phoenix.parse.OrderByNode;
import org.apache.phoenix.parse.ParseNode;
import org.apache.phoenix.parse.ParseNodeFactory;
@@ -77,27 +75,30 @@ import org.apache.phoenix.parse.SQLParser;
import org.apache.phoenix.parse.SelectStatement;
import org.apache.phoenix.parse.SubqueryParseNode;
import org.apache.phoenix.parse.TableNode;
+import org.apache.phoenix.parse.TerminalParseNode;
import org.apache.phoenix.query.ConnectionQueryServices;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.AmbiguousColumnException;
import org.apache.phoenix.schema.ColumnNotFoundException;
+import org.apache.phoenix.schema.ConditionTTLExpression;
import org.apache.phoenix.schema.PDatum;
import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTableType;
import
org.apache.phoenix.schema.RowValueConstructorOffsetNotCoercibleException;
+import org.apache.phoenix.schema.SortOrder;
import org.apache.phoenix.schema.TableNotFoundException;
import org.apache.phoenix.schema.TableRef;
+import org.apache.phoenix.thirdparty.com.google.common.base.Optional;
+import org.apache.phoenix.thirdparty.com.google.common.collect.Lists;
+import org.apache.phoenix.thirdparty.com.google.common.collect.Sets;
import org.apache.phoenix.util.CDCUtil;
import org.apache.phoenix.util.EnvironmentEdgeManager;
+import org.apache.phoenix.util.MetaDataUtil;
import org.apache.phoenix.util.ParseNodeUtil;
import org.apache.phoenix.util.ParseNodeUtil.RewriteResult;
import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.ScanUtil;
-import org.apache.phoenix.util.MetaDataUtil;
-import org.apache.hadoop.conf.Configuration;
-
-import org.apache.phoenix.thirdparty.com.google.common.collect.Lists;
-import org.apache.phoenix.thirdparty.com.google.common.collect.Sets;
/**
@@ -787,6 +788,32 @@ public class QueryCompiler {
TableRef tableRef = context.getCurrentTable();
PTable table = tableRef.getTable();
+ if (table.hasConditionTTL()) { // TODO CDC index
+ ConditionTTLExpression condTTLExpr = (ConditionTTLExpression)
table.getTTL();
+ // For non-index tables we have to re-write the WHERE clause by
ANDing the condition
+ // TTL expression. We can do it since the condition TTL expression
always evaluates to
+ // a BOOLEAN. For index tables we don't need to re-write since we
first re-write
+ // the query for data table and later the optimizer will re-write
for index tables.
+ // The only time we need to re-write the WHERE clause for index
tables is when the
+ // query is run directly against the index tables in which case
the dataPlans
+ // will be empty
+ boolean rewrite = table.getType() != PTableType.INDEX ||
dataPlans.isEmpty();
+ if (rewrite) {
+ PhoenixConnection conn = this.statement.getConnection();
+ // Takes care of translating data table column references
+ // to index table column references
+ ParseNode ttlCondition = condTTLExpr.parseExpression(conn,
table);
+ ParseNode negateTTL = NODE_FACTORY.not(ttlCondition);
+ ParseNode where = select.getWhere();
+ if (where == null) {
+ where = negateTTL;
+ } else {
+ where = NODE_FACTORY.and(Arrays.asList(where, negateTTL));
+ }
+ select = NODE_FACTORY.select(select, where);
+ }
+ }
+
if (table.getType() == PTableType.CDC) {
List<AliasedNode> selectNodes = select.getSelect();
// For CDC queries, if a single wildcard projection is used,
automatically insert
diff --git
a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/ServerBuildIndexCompiler.java
b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/ServerBuildIndexCompiler.java
index bd8dcd480c..9b6515e35b 100644
---
a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/ServerBuildIndexCompiler.java
+++
b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/ServerBuildIndexCompiler.java
@@ -108,6 +108,13 @@ public class ServerBuildIndexCompiler {
Scan scan = plan.getContext().getScan();
ImmutableBytesWritable ptr = new ImmutableBytesWritable();
dataTable = tableRef.getTable();
+ if (dataTable.hasConditionTTL()) {
+ // For raw scans like Index rebuild don't use the condition
ttl filter
+ // because the filters don't handle delete markers. The only
downside is
+ // you will build some extra expired rows but those will be
masked and purged
+ // when compaction runs
+ scan.setFilter(null);
+ }
if (IndexUtil.isGlobalIndex(index) &&
dataTable.isTransactional()) {
throw new IllegalArgumentException(
"ServerBuildIndexCompiler does not support global
indexes on transactional tables");
diff --git
a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/ConditionTTLExpression.java
b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/ConditionTTLExpression.java
index 828010a034..e303f223c8 100644
---
a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/ConditionTTLExpression.java
+++
b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/ConditionTTLExpression.java
@@ -24,6 +24,7 @@ import static
org.apache.phoenix.schema.PTable.QualifierEncodingScheme.NON_ENCOD
import java.sql.SQLException;
import java.util.Collections;
import java.util.List;
+import java.util.Map;
import java.util.Objects;
import org.apache.hadoop.hbase.Cell;
@@ -31,6 +32,7 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.compile.ColumnResolver;
import org.apache.phoenix.compile.ExpressionCompiler;
import org.apache.phoenix.compile.FromCompiler;
+import org.apache.phoenix.compile.IndexStatementRewriter;
import org.apache.phoenix.compile.StatementContext;
import org.apache.phoenix.exception.SQLExceptionCode;
import org.apache.phoenix.exception.SQLExceptionInfo;
@@ -47,7 +49,6 @@ import org.apache.phoenix.parse.SQLParser;
import org.apache.phoenix.parse.TableName;
import org.apache.phoenix.schema.types.PBoolean;
import org.apache.phoenix.schema.types.PDataType;
-import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.SchemaUtil;
public class ConditionTTLExpression extends TTLExpression {
@@ -121,6 +122,26 @@ public class ConditionTTLExpression extends TTLExpression {
return null;
}
+ public ParseNode parseExpression(PhoenixConnection connection,
+ PTable table) throws SQLException {
+ ParseNode ttlCondition = SQLParser.parseCondition(this.ttlExpr);
+ return table.getType() != PTableType.INDEX ? ttlCondition
+ : rewriteForIndex(connection, table, ttlCondition);
+ }
+
+ private ParseNode rewriteForIndex(PhoenixConnection connection,
+ PTable index,
+ ParseNode ttlCondition) throws
SQLException {
+ for (Map.Entry<PTableKey, Long> entry :
index.getAncestorLastDDLTimestampMap().entrySet()) {
+ PTableKey parentKey = entry.getKey();
+ PTable parent = connection.getTable(parentKey);
+ ColumnResolver parentResolver = FromCompiler.getResolver(new
TableRef(parent));
+ return IndexStatementRewriter.translate(ttlCondition,
parentResolver);
+ }
+ // TODO: Fix exception
+ throw new SQLException("Parent not found");
+ }
+
/**
* Validates that all the columns used in the conditional TTL expression
are present in the table
* or its parent table in case of view
diff --git
a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/DelegateTable.java
b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/DelegateTable.java
index 9e1f2e81a1..398e2d33d0 100644
---
a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/DelegateTable.java
+++
b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/DelegateTable.java
@@ -385,6 +385,11 @@ public class DelegateTable implements PTable {
return delegate.getTTL();
}
+ @Override
+ public boolean hasConditionTTL() {
+ return delegate.hasConditionTTL();
+ }
+
@Override
public Long getLastDDLTimestamp() {
return delegate.getLastDDLTimestamp();
diff --git
a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTable.java
b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTable.java
index 867c47d4a2..db1bc2610c 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTable.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTable.java
@@ -937,6 +937,11 @@ public interface PTable extends PMetaDataEntity {
*/
TTLExpression getTTL();
+ /**
+ * @return Returns true if table has condition TTL set
+ */
+ boolean hasConditionTTL();
+
/**
* @return the last timestamp at which this entity had its data shape
created or modified (e
* .g, create entity, adding or dropping a column. Not affected by
changing table properties
diff --git
a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTableImpl.java
b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTableImpl.java
index 7bd6ecedce..abcfb02e1d 100644
---
a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTableImpl.java
+++
b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/PTableImpl.java
@@ -2469,6 +2469,11 @@ public class PTableImpl implements PTable {
return ttl;
}
+ @Override
+ public boolean hasConditionTTL() {
+ return ttl instanceof ConditionTTLExpression;
+ }
+
@Override public boolean hasViewModifiedUpdateCacheFrequency() {
return
viewModifiedPropSet.get(VIEW_MODIFIED_UPDATE_CACHE_FREQUENCY_BIT_SET_POS);
}
diff --git
a/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/PhoenixServerBuildIndexInputFormat.java
b/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/PhoenixServerBuildIndexInputFormat.java
index 2492394494..1ded535938 100644
---
a/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/PhoenixServerBuildIndexInputFormat.java
+++
b/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/PhoenixServerBuildIndexInputFormat.java
@@ -123,6 +123,13 @@ public class PhoenixServerBuildIndexInputFormat<T extends
DBWritable> extends Ph
Scan scan = plan.getContext().getScan();
ImmutableBytesWritable ptr = new ImmutableBytesWritable();
PTable pIndexTable = tableRef.getTable();
+ if (pIndexTable.hasConditionTTL()) {
+ // For raw scans like Index rebuild don't use the
condition ttl filter
+ // because the filters don't handle delete markers. The
only downside is
+ // you will build some extra expired rows but those will
be masked and purged
+ // when compaction runs
+ scan.setFilter(null);
+ }
PTable pDataTable =
phoenixConnection.getTable(dataTableFullName);
IndexMaintainer.serialize(pDataTable, ptr,
Collections.singletonList(pIndexTable), phoenixConnection);
scan.setAttribute(PhoenixIndexCodec.INDEX_NAME_FOR_IDX_MAINTAINER,
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TTLAsPhoenixTTLIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TTLAsPhoenixTTLIT.java
index fdc42a6236..29c303254b 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TTLAsPhoenixTTLIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TTLAsPhoenixTTLIT.java
@@ -24,18 +24,14 @@ import
org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.jdbc.PhoenixConnection;
-import org.apache.phoenix.parse.ParseNode;
-import org.apache.phoenix.parse.SQLParser;
import org.apache.phoenix.schema.ColumnNotFoundException;
import org.apache.phoenix.schema.ColumnFamilyNotFoundException;
import org.apache.phoenix.schema.PName;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTableKey;
import org.apache.phoenix.schema.TTLExpression;
-import org.apache.phoenix.schema.TypeMismatchException;
import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.PhoenixRuntime;
-import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
@@ -43,7 +39,6 @@ import org.junit.runners.Parameterized;
import java.sql.Connection;
import java.sql.DriverManager;
-import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.Arrays;
import java.util.Collection;
@@ -55,7 +50,9 @@ import static
org.apache.phoenix.exception.SQLExceptionCode.TTL_ALREADY_DEFINED_
import static
org.apache.phoenix.exception.SQLExceptionCode.TTL_SUPPORTED_FOR_TABLES_AND_VIEWS_ONLY;
import static
org.apache.phoenix.schema.TTLExpression.TTL_EXPRESSION_NOT_DEFINED;
import static org.apache.phoenix.util.PhoenixRuntime.TENANT_ID_ATTRIB;
+import static org.apache.phoenix.util.TestUtil.retainSingleQuotes;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -132,6 +129,7 @@ public class TTLAsPhoenixTTLIT extends
ParallelStatsDisabledIT{
PTable table = conn.unwrap(PhoenixConnection.class).getTable(new
PTableKey(null,
createTableWithOrWithOutTTLAsItsProperty(conn, false)));
assertTTLValue(table, TTL_EXPRESSION_NOT_DEFINED);
+ assertFalse(table.hasConditionTTL());
}
}
@@ -142,6 +140,7 @@ public class TTLAsPhoenixTTLIT extends
ParallelStatsDisabledIT{
PTable table = conn.unwrap(PhoenixConnection.class).
getTable(new PTableKey(null, tableName));
assertTTLValue(table, defaultTTL);
+ assertTrue(table.hasConditionTTL() == useExpression);
// Switch from cond ttl to value or vice versa
String alterTTL = useExpression ?
String.valueOf(DEFAULT_TTL_FOR_ALTER) :
String.format("'%s'", DEFAULT_TTL_EXPRESSION_FOR_ALTER);
@@ -163,10 +162,9 @@ public class TTLAsPhoenixTTLIT extends
ParallelStatsDisabledIT{
public void testCreateTableWithTTLWithDifferentColumnFamilies() throws
Exception {
String tableName = generateUniqueName();
String ttlExpr = "id = 'x' AND b.col2 = 7";
- String quotedTTLExpr = "id = ''x'' AND b.col2 = 7"; // to retain
single quotes in DDL
int ttlValue = 86400;
String ttl = (useExpression ?
- String.format("'%s'", quotedTTLExpr) :
+ String.format("'%s'", retainSingleQuotes(ttlExpr)) :
String.valueOf(ttlValue));
String ddl =
"create table IF NOT EXISTS " + tableName + " (" + " id
char(1) NOT NULL,"
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/mapreduce/PhoenixServerBuildIndexInputFormatIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/mapreduce/PhoenixServerBuildIndexInputFormatIT.java
index ff359a6819..8a95b287cb 100644
---
a/phoenix-core/src/it/java/org/apache/phoenix/mapreduce/PhoenixServerBuildIndexInputFormatIT.java
+++
b/phoenix-core/src/it/java/org/apache/phoenix/mapreduce/PhoenixServerBuildIndexInputFormatIT.java
@@ -18,6 +18,7 @@
package org.apache.phoenix.mapreduce;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.mapreduce.Job;
import org.apache.phoenix.compile.QueryPlan;
import org.apache.phoenix.end2end.ParallelStatsDisabledIT;
@@ -31,17 +32,36 @@ import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.SchemaUtil;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
import java.sql.Connection;
import java.sql.DriverManager;
+import java.util.Arrays;
+import java.util.Collection;
import java.util.Properties;
import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
@Category(ParallelStatsDisabledTest.class)
+@RunWith(Parameterized.class)
public class PhoenixServerBuildIndexInputFormatIT extends
ParallelStatsDisabledIT {
+ private final boolean useExpression;
+
+ @Parameterized.Parameters(name = "useExpression={0}")
+ public static synchronized Collection<Boolean[]> data() {
+ return Arrays.asList(new Boolean[][]{
+ {false}, {true}
+ });
+ }
+
+ public PhoenixServerBuildIndexInputFormatIT(boolean useExpression) {
+ this.useExpression = useExpression;
+ }
+
@Test
public void testQueryPlanWithSource() throws Exception {
PhoenixServerBuildIndexInputFormat inputFormat;
@@ -58,7 +78,8 @@ public class PhoenixServerBuildIndexInputFormatIT extends
ParallelStatsDisabled
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.createStatement().execute("CREATE TABLE " + dataTableFullName
- + " (ID INTEGER NOT NULL PRIMARY KEY, VAL1 INTEGER, VAL2
INTEGER) ");
+ + " (ID INTEGER NOT NULL PRIMARY KEY, VAL1 INTEGER, VAL2
INTEGER) " +
+ (useExpression ? "TTL = 'VAL2 = -1'" : ""));
conn.createStatement().execute(String.format(
"CREATE INDEX %s ON %s (VAL1) INCLUDE (VAL2)", indexTableName,
dataTableFullName));
conn.createStatement().execute("CREATE VIEW " + viewFullName +
@@ -105,5 +126,7 @@ public class PhoenixServerBuildIndexInputFormatIT extends
ParallelStatsDisabled
} else {
assertEquals(pIndexTable, actual);
}
+ Scan scan = queryPlan.getContext().getScan();
+ assertNull(scan.getFilter());
}
}
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/schema/ConditionTTLExpressionIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/schema/ConditionTTLExpressionIT.java
new file mode 100644
index 0000000000..87359c32bb
--- /dev/null
+++
b/phoenix-core/src/it/java/org/apache/phoenix/schema/ConditionTTLExpressionIT.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.schema;
+
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.apache.phoenix.util.TestUtil.in;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.util.Properties;
+
+import org.apache.phoenix.end2end.ParallelStatsDisabledIT;
+import org.apache.phoenix.jdbc.PhoenixResultSet;
+import org.apache.phoenix.util.EnvironmentEdgeManager;
+import org.apache.phoenix.util.ManualEnvironmentEdge;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.QueryUtil;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(ParallelStatsDisabledIT.class)
+public class ConditionTTLExpressionIT extends ParallelStatsDisabledIT {
+
+ ManualEnvironmentEdge injectEdge;
+
+ @Before
+ public void beforeTest(){
+ EnvironmentEdgeManager.reset();
+ injectEdge = new ManualEnvironmentEdge();
+ injectEdge.setValue(EnvironmentEdgeManager.currentTimeMillis());
+ }
+
+ @After
+ public synchronized void afterTest() {
+ EnvironmentEdgeManager.reset();
+ }
+
+ @Test
+ public void testMasking() throws Exception {
+ final String tablename = "T_" + generateUniqueName();
+ final String indexName = "I_" + generateUniqueName();
+ final String ddlTemplate = "create table %s (k1 bigint not null, k2
bigint not null," +
+ "val varchar, expired boolean constraint pk primary key
(k1,k2)) TTL = 'expired'";
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String ddl = String.format(ddlTemplate, tablename);
+ conn.createStatement().execute(ddl);
+ conn.commit();
+ PreparedStatement dml = conn.prepareStatement("upsert into " +
tablename + " VALUES(?, ?, ?, ?)");
+ int rows = 10, cols = 2;
+ int total = rows * cols;
+ for (int i = 0; i < rows; ++i) {
+ for (int j = 0; j < cols; ++j) {
+ dml.setInt(1, i);
+ dml.setInt(2, j);
+ dml.setString(3, "val_" +i);
+ dml.setBoolean(4, false);
+ dml.executeUpdate();
+ }
+ }
+ conn.commit();
+ PreparedStatement dql = conn.prepareStatement("select count(*)
from " + tablename);
+ ResultSet rs = dql.executeQuery();
+ assertTrue(rs.next());
+ assertEquals(total, rs.getInt(1));
+
+ // expire odd rows by setting expired to true
+ dml = conn.prepareStatement("upsert into " + tablename + "(k1, k2,
expired) VALUES(?, ?, ?)");
+ for (int i = 0; i < 10; ++i) {
+ dml.setInt(1, i);
+ dml.setInt(2, 1);
+ dml.setBoolean(3, true);
+ dml.executeUpdate();
+ }
+ conn.commit();
+ rs = dql.executeQuery();
+ assertTrue(rs.next());
+ // half the rows should be masked
+ assertEquals(total/2, rs.getInt(1));
+
+ ddl = "create index " + indexName + " ON " + tablename + " (val)
INCLUDE (expired)";
+ conn.createStatement().execute(ddl);
+ conn.commit();
+
+ rs = dql.executeQuery();
+ PhoenixResultSet prs = rs.unwrap(PhoenixResultSet.class);
+ String explainPlan =
QueryUtil.getExplainPlan(prs.getUnderlyingIterator());
+ assertTrue(explainPlan.contains(indexName));
+ assertTrue(rs.next());
+ // half the rows should be masked
+ assertEquals(total/2, rs.getInt(1));
+
+ dql = conn.prepareStatement(
+ "select k1,k2 from " + tablename + " where val='val_7'");
+ rs = dql.executeQuery();
+ prs = rs.unwrap(PhoenixResultSet.class);
+ explainPlan =
QueryUtil.getExplainPlan(prs.getUnderlyingIterator());
+ assertTrue(explainPlan.contains(indexName));
+ // only even row expected (7,0)
+ assertTrue(rs.next());
+ assertEquals(7, rs.getInt(1));
+ assertEquals(0, rs.getInt(2));
+ assertFalse(rs.next());
+
+ // now update the row again and set expired = false
+ dml.setInt(1, 7);
+ dml.setInt(2, 1);
+ dml.setBoolean(3, false);
+ dml.executeUpdate();
+ conn.commit();
+
+ // run the above query again 2 rows expected (7,0) and (7,1)
+ rs = dql.executeQuery();
+ assertTrue(rs.next());
+ assertEquals(7, rs.getInt(1));
+ assertEquals(0, rs.getInt(2));
+ assertTrue(rs.next());
+ assertEquals(7, rs.getInt(1));
+ assertEquals(1, rs.getInt(2));
+ assertFalse(rs.next());
+ }
+ }
+
+ @Test
+ public void testPhoenixRowTimestamp() throws Exception {
+ final String tablename = "T_" + generateUniqueName();
+ final String ddlTemplate = "create table %s (k1 bigint not null, k2
bigint not null," +
+ "val varchar constraint pk primary key (k1,k2))" +
+ "TTL = 'TO_NUMBER(CURRENT_TIME()) -
TO_NUMBER(PHOENIX_ROW_TIMESTAMP()) >= 50'"; // 50ms ttl
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String ddl = String.format(ddlTemplate, tablename);
+ conn.createStatement().execute(ddl);
+ conn.commit();
+ PreparedStatement dml = conn.prepareStatement("upsert into " +
tablename + " VALUES(?, ?, ?)");
+ int rows = 10, cols = 2;
+ int total = rows * cols;
+ for (int i = 0; i < rows; ++i) {
+ for (int j = 0; j < cols; ++j) {
+ dml.setInt(1, i);
+ dml.setInt(2, j);
+ dml.setString(3, "val_" +i);
+ dml.executeUpdate();
+ }
+ }
+ conn.commit();
+ PreparedStatement dql = conn.prepareStatement("select count(*)
from " + tablename);
+ ResultSet rs = dql.executeQuery();
+ assertTrue(rs.next());
+ assertEquals(total, rs.getInt(1));
+ // bump the current time to go past ttl value
+ injectEdge.incrementValue(100);
+ dql = conn.prepareStatement("select count(*) from " + tablename);
+ rs = dql.executeQuery();
+ assertTrue(rs.next());
+ assertEquals(0, rs.getInt(1));
+
+ // update 1 row
+ dml.setInt(1, 7);
+ dml.setInt(2, 1);
+ dml.setString(3, "val_foo");
+ dml.executeUpdate();
+ conn.commit();
+
+ dql = conn.prepareStatement("select count(*) from " + tablename);
+ rs = dql.executeQuery();
+ assertTrue(rs.next());
+ assertEquals(1, rs.getInt(1));
+ }
+ }
+}
diff --git
a/phoenix-core/src/test/java/org/apache/phoenix/schema/ConditionTTLExpressionTest.java
b/phoenix-core/src/test/java/org/apache/phoenix/schema/ConditionTTLExpressionTest.java
new file mode 100644
index 0000000000..9f5ed5ebb2
--- /dev/null
+++
b/phoenix-core/src/test/java/org/apache/phoenix/schema/ConditionTTLExpressionTest.java
@@ -0,0 +1,446 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.schema;
+
+import static org.apache.phoenix.schema.PTableType.INDEX;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.apache.phoenix.util.TestUtil.retainSingleQuotes;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterList;
+import org.apache.phoenix.compile.OrderByCompiler;
+import org.apache.phoenix.compile.QueryPlan;
+import org.apache.phoenix.exception.PhoenixParserException;
+import org.apache.phoenix.exception.SQLExceptionCode;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixPreparedStatement;
+import org.apache.phoenix.jdbc.PhoenixStatement;
+import org.apache.phoenix.query.BaseConnectionlessQueryTest;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.junit.Test;
+
+public class ConditionTTLExpressionTest extends BaseConnectionlessQueryTest {
+
+ private static void assertConditonTTL(Connection conn, String tableName,
String ttlExpr) throws SQLException {
+ TTLExpression expected = new ConditionTTLExpression(ttlExpr);
+ assertTTL(conn, tableName, expected);
+ }
+
+ private static void assertTTL(Connection conn, String tableName,
TTLExpression expected) throws SQLException {
+ PTable table =
conn.unwrap(PhoenixConnection.class).getTable(tableName);
+ assertEquals(expected, table.getTTL());
+ }
+
+ private static void assertScanConditionTTL(Scan scan, Scan
scanWithCondTTL) {
+ assertEquals(scan.includeStartRow(),
scanWithCondTTL.includeStartRow());
+ assertArrayEquals(scan.getStartRow(), scanWithCondTTL.getStartRow());
+ assertEquals(scan.includeStopRow(), scanWithCondTTL.includeStopRow());
+ assertArrayEquals(scan.getStopRow(), scanWithCondTTL.getStopRow());
+ Filter filter = scan.getFilter();
+ Filter filterCondTTL = scanWithCondTTL.getFilter();
+ assertNotNull(filter);
+ assertNotNull(filterCondTTL);
+ if (filter instanceof FilterList) {
+ assertTrue(filterCondTTL instanceof FilterList);
+ FilterList filterList = (FilterList) filter;
+ FilterList filterListCondTTL = (FilterList) filterCondTTL;
+ // ultimately compares the individual filters
+ assertEquals(filterList, filterListCondTTL);
+ } else {
+ assertEquals(filter, filterCondTTL);
+ }
+ }
+
+ private void compareScanWithCondTTL(Connection conn,
+ String tableNoTTL,
+ String tableWithTTL,
+ String queryTemplate,
+ String ttl) throws SQLException {
+ compareScanWithCondTTL(conn, tableNoTTL, tableWithTTL, queryTemplate,
ttl, false);
+ }
+
+ private void compareScanWithCondTTL(Connection conn,
+ String tableNoTTL,
+ String tableWithTTL,
+ String queryTemplate,
+ String ttl,
+ boolean useIndex) throws SQLException {
+ PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class);
+ String query;
+ // Modify the query by adding the cond ttl expression explicitly to
the WHERE clause
+ if (queryTemplate.toUpperCase().contains(" WHERE ")) {
+ // append the cond TTL expression to the WHERE clause
+ query = String.format(queryTemplate + " AND NOT (%s)", tableNoTTL,
ttl);
+ } else {
+ // add a WHERE clause with negative cond ttl expression
+ query = String.format(queryTemplate + " WHERE NOT (%s)",
tableNoTTL, ttl);
+ }
+ PhoenixPreparedStatement pstmt = new PhoenixPreparedStatement(pconn,
query);
+ QueryPlan plan = pstmt.optimizeQuery();
+ if (useIndex) {
+ assertTrue(plan.getTableRef().getTable().getType() == INDEX);
+ }
+ Scan scanNoTTL = plan.getContext().getScan();
+ // now execute the same query with cond ttl expression implicitly used
for masking
+ query = String.format(queryTemplate, tableWithTTL);
+ pstmt = new PhoenixPreparedStatement(pconn, query);
+ plan = pstmt.optimizeQuery();
+ if (useIndex) {
+ assertTrue(plan.getTableRef().getTable().getType() == INDEX);
+ }
+ Scan scanWithCondTTL = plan.getContext().getScan();
+ assertScanConditionTTL(scanNoTTL, scanWithCondTTL);
+ }
+
+ @Test
+ public void testBasicExpression() throws SQLException {
+ String ddlTemplate = "create table %s (k1 bigint not null, k2 bigint
not null, col1 varchar, col2 date " +
+ "constraint pk primary key (k1,k2 desc))";
+ String ddlTemplateWithTTL = ddlTemplate + " TTL = '%s'";
+ String tableNoTTL = generateUniqueName();
+ String tableWithTTL = generateUniqueName();
+ String ttl = "k1 > 5 AND col1 < 'zzzzzz'";
+ try (Connection conn = DriverManager.getConnection(getUrl(),
PropertiesUtil.deepCopy(TEST_PROPERTIES))) {
+ String ddl = String.format(ddlTemplate, tableNoTTL);
+ conn.createStatement().execute(ddl);
+ ddl = String.format(ddlTemplateWithTTL, tableWithTTL,
retainSingleQuotes(ttl));
+ conn.createStatement().execute(ddl);
+ assertConditonTTL(conn, tableWithTTL, ttl);
+ String queryTemplate = "SELECT k1, k2, col1, col2 from %s where k1
> 3";
+ compareScanWithCondTTL(conn, tableNoTTL, tableWithTTL,
queryTemplate, ttl);
+ }
+ }
+
+ @Test(expected = TypeMismatchException.class)
+ public void testNotBooleanExpr() throws SQLException {
+ String ddlTemplate = "create table %s (k1 bigint not null, k2 bigint
not null, col1 varchar, col2 date " +
+ "constraint pk primary key (k1,k2 desc)) TTL = '%s'";
+ String ttl = "k1 + 100";
+ String tableName = generateUniqueName();
+ String ddl = String.format(ddlTemplate, tableName, ttl);
+ try (Connection conn = DriverManager.getConnection(getUrl(),
PropertiesUtil.deepCopy(TEST_PROPERTIES))) {
+ conn.createStatement().execute(ddl);
+ }
+ }
+
+ @Test(expected = TypeMismatchException.class)
+ public void testWrongArgumentValue() throws SQLException {
+ String ddlTemplate = "create table %s (k1 bigint not null, k2 bigint
not null, col1 varchar, col2 date " +
+ "constraint pk primary key (k1,k2 desc)) TTL = '%s'";
+ String ttl = "k1 = ''abc''";
+ String tableName = generateUniqueName();
+ String ddl = String.format(ddlTemplate, tableName, ttl);
+ try (Connection conn = DriverManager.getConnection(getUrl(),
PropertiesUtil.deepCopy(TEST_PROPERTIES))) {
+ conn.createStatement().execute(ddl);
+ }
+ }
+
+ @Test(expected = PhoenixParserException.class)
+ public void testParsingError() throws SQLException {
+ String ddlTemplate = "create table %s (k1 bigint not null, k2 bigint
not null, col1 varchar, col2 date " +
+ "constraint pk primary key (k1,k2 desc)) TTL = '%s'";
+ String ttl = "k2 == 23";
+ String tableName = generateUniqueName();
+ String ddl = String.format(ddlTemplate, tableName, ttl);
+ try (Connection conn = DriverManager.getConnection(getUrl(),
PropertiesUtil.deepCopy(TEST_PROPERTIES))) {
+ conn.createStatement().execute(ddl);
+ }
+ }
+
+ @Test
+ public void testAggregateExpressionNotAllowed() throws SQLException {
+ String ddlTemplate = "create table %s (k1 bigint not null, k2 bigint
not null, col1 varchar, col2 date " +
+ "constraint pk primary key (k1,k2 desc)) TTL = '%s'";
+ String ttl = "SUM(k2) > 23";
+ String tableName = generateUniqueName();
+ String ddl = String.format(ddlTemplate, tableName, ttl);
+ try (Connection conn = DriverManager.getConnection(getUrl(),
PropertiesUtil.deepCopy(TEST_PROPERTIES))) {
+ conn.createStatement().execute(ddl);
+ fail();
+ } catch (SQLException e) {
+
assertEquals(SQLExceptionCode.AGGREGATE_EXPRESSION_NOT_ALLOWED_IN_TTL_EXPRESSION.getErrorCode(),
+ e.getErrorCode());
+ } catch (Exception e) {
+ fail("Unknown exception " + e);
+ }
+ }
+
+ @Test
+ public void testNullExpression() throws SQLException {
+ String ddlTemplate = "create table %s (k1 bigint not null, k2 bigint
not null, col1 varchar, col2 date " +
+ "constraint pk primary key (k1,k2 desc))";
+ String ddlTemplateWithTTL = ddlTemplate + " TTL = '%s'";
+ String tableNoTTL = generateUniqueName();
+ String tableWithTTL = generateUniqueName();
+ String ttl = "col1 is NULL AND col2 < CURRENT_DATE() + 30000";
+ try (Connection conn = DriverManager.getConnection(getUrl(),
PropertiesUtil.deepCopy(TEST_PROPERTIES))) {
+ String ddl = String.format(ddlTemplate, tableNoTTL);
+ conn.createStatement().execute(ddl);
+ ddl = String.format(ddlTemplateWithTTL, tableWithTTL, ttl);
+ conn.createStatement().execute(ddl);
+ assertConditonTTL(conn, tableWithTTL, ttl);
+ String queryTemplate = "SELECT * from %s";
+ compareScanWithCondTTL(conn, tableNoTTL, tableWithTTL,
queryTemplate, ttl);
+ }
+ }
+
+ @Test
+ public void testBooleanColumn() throws SQLException {
+ String ddlTemplate = "create table %s (k1 bigint not null, k2 bigint
not null, val varchar, expired BOOLEAN " +
+ "constraint pk primary key (k1,k2 desc))";
+ String ddlTemplateWithTTL = ddlTemplate + " TTL = '%s'";
+ String tableNoTTL = generateUniqueName();
+ String indexNoTTL = "I_" + tableNoTTL;
+ String tableWithTTL = generateUniqueName();
+ String indexWithTTL = "I_" + tableWithTTL;
+ String indexTemplate = "create index %s on %s (val) include (expired)";
+ String ttl = "expired";
+ try (Connection conn = DriverManager.getConnection(getUrl(),
PropertiesUtil.deepCopy(TEST_PROPERTIES))) {
+ String ddl = String.format(ddlTemplate, tableNoTTL);
+ conn.createStatement().execute(ddl);
+ ddl = String.format(ddlTemplateWithTTL, tableWithTTL, ttl);
+ conn.createStatement().execute(ddl);
+ assertConditonTTL(conn, tableWithTTL, ttl);
+
+ String queryTemplate = "SELECT k1, k2 from %s where (k1,k2) IN
((1,2), (3,4))";
+ compareScanWithCondTTL(conn, tableNoTTL, tableWithTTL,
queryTemplate, ttl);
+
+ queryTemplate = "SELECT COUNT(*) from %s";
+ compareScanWithCondTTL(conn, tableNoTTL, tableWithTTL,
queryTemplate, ttl);
+
+ ddl = String.format(indexTemplate, indexNoTTL, tableNoTTL);
+ conn.createStatement().execute(ddl);
+ ddl = String.format(indexTemplate, indexWithTTL, tableWithTTL);
+ conn.createStatement().execute(ddl);
+ assertTTL(conn, indexNoTTL,
TTLExpression.TTL_EXPRESSION_NOT_DEFINED);
+ assertConditonTTL(conn, indexWithTTL, ttl);
+ compareScanWithCondTTL(conn, tableNoTTL, tableWithTTL,
queryTemplate, ttl, true);
+ }
+ }
+
+ @Test
+ public void testNot() throws SQLException {
+ String ddlTemplate = "create table %s (k1 bigint not null, k2 bigint
not null, expired BOOLEAN " +
+ "constraint pk primary key (k1,k2 desc)) TTL = '%s'";
+ String ttl = "NOT expired";
+ String tableName = generateUniqueName();
+ String ddl = String.format(ddlTemplate, tableName, ttl);
+ try (Connection conn = DriverManager.getConnection(getUrl(),
PropertiesUtil.deepCopy(TEST_PROPERTIES))) {
+ conn.createStatement().execute(ddl);
+ assertConditonTTL(conn, tableName, ttl);
+ }
+ }
+
+ @Test
+ public void testPhoenixRowTimestamp() throws SQLException {
+ String ddlTemplate = "create table %s (k1 bigint not null, k2 bigint
not null, col1 varchar " +
+ "constraint pk primary key (k1,k2 desc))";
+ String ddlTemplateWithTTL = ddlTemplate + " TTL = '%s'";
+ String tableNoTTL = generateUniqueName();
+ String tableWithTTL = generateUniqueName();
+ String ttl = "PHOENIX_ROW_TIMESTAMP() < CURRENT_DATE() - 100";
+ try (Connection conn = DriverManager.getConnection(getUrl(),
PropertiesUtil.deepCopy(TEST_PROPERTIES))) {
+ String ddl = String.format(ddlTemplate, tableNoTTL);
+ conn.createStatement().execute(ddl);
+ ddl = String.format(ddlTemplateWithTTL, tableWithTTL, ttl);
+ conn.createStatement().execute(ddl);
+ assertConditonTTL(conn, tableWithTTL, ttl);
+ String queryTemplate = "select * from %s where k1 = 7 AND k2 > 12";
+ compareScanWithCondTTL(conn, tableNoTTL, tableWithTTL,
queryTemplate, ttl);
+ queryTemplate = "select * from %s where k1 = 7 AND k2 > 12 AND
col1 = 'abc'";
+ compareScanWithCondTTL(conn, tableNoTTL, tableWithTTL,
queryTemplate, ttl);
+ }
+ }
+
+ @Test
+ public void testBooleanCaseExpression() throws SQLException {
+ String ddlTemplate = "create table %s (k1 bigint not null, k2 bigint
not null, col1 varchar, status char(1) " +
+ "constraint pk primary key (k1,k2 desc)) TTL = '%s'";
+ String ttl = "CASE WHEN status = ''E'' THEN TRUE ELSE FALSE END";
+ String expectedTTLExpr = "CASE WHEN status = 'E' THEN TRUE ELSE FALSE
END";
+ String tableName = generateUniqueName();
+ String ddl = String.format(ddlTemplate, tableName, ttl);
+ try (Connection conn = DriverManager.getConnection(getUrl(),
PropertiesUtil.deepCopy(TEST_PROPERTIES))) {
+ conn.createStatement().execute(ddl);
+ assertConditonTTL(conn, tableName, expectedTTLExpr);
+ }
+ }
+
+ @Test
+ public void testCondTTLOnTopLevelView() throws SQLException {
+ String ddlTemplate = "create table %s (k1 bigint not null primary
key," +
+ "k2 bigint, col1 varchar, status char(1))";
+ String tableName1 = generateUniqueName();
+ String tableName2 = generateUniqueName();
+ String viewNameNoTTL = generateUniqueName();
+ String viewNameWithTTL = generateUniqueName();
+ String viewTemplate = "create view %s (k3 smallint) as select * from
%s WHERE k1=7 TTL = '%s'";
+ String ttl = "k2 = 34 and k3 = -1";
+ try (Connection conn = DriverManager.getConnection(getUrl(),
PropertiesUtil.deepCopy(TEST_PROPERTIES))) {
+ String ddl = String.format(ddlTemplate, tableName1);
+ conn.createStatement().execute(ddl);
+ ddl = String.format(viewTemplate, viewNameWithTTL, tableName1,
ttl);
+ conn.createStatement().execute(ddl);
+ assertTTL(conn, tableName1,
TTLExpression.TTL_EXPRESSION_NOT_DEFINED);
+ assertConditonTTL(conn, viewNameWithTTL, ttl);
+ ddl = String.format(ddlTemplate, tableName2);
+ conn.createStatement().execute(ddl);
+ ddl = String.format(viewTemplate, viewNameNoTTL, tableName2,
"NONE");
+ conn.createStatement().execute(ddl);
+ assertTTL(conn, tableName1,
TTLExpression.TTL_EXPRESSION_NOT_DEFINED);
+ String queryTemplate = "select * from %s";
+ compareScanWithCondTTL(conn, viewNameNoTTL, viewNameWithTTL,
queryTemplate, ttl);
+ }
+ }
+
+ @Test
+ public void testCondTTLOnMultiLevelView() throws SQLException {
+ String ddlTemplate = "create table %s (k1 bigint not null primary
key," +
+ "k2 bigint, col1 varchar, status char(1))";
+ String tableName = generateUniqueName();
+ String parentView = generateUniqueName();
+ String childView = generateUniqueName();
+ String parentViewTemplate = "create view %s (k3 smallint) as select *
from %s WHERE k1=7";
+ String childViewTemplate = "create view %s as select * from %s TTL =
'%s'";
+ String ttl = "k2 = 34 and k3 = -1";
+ String ddl = String.format(ddlTemplate, tableName);
+ try (Connection conn = DriverManager.getConnection(getUrl(),
PropertiesUtil.deepCopy(TEST_PROPERTIES))) {
+ conn.createStatement().execute(ddl);
+ ddl = String.format(parentViewTemplate, parentView, tableName);
+ conn.createStatement().execute(ddl);
+ ddl = String.format(childViewTemplate, childView, parentView, ttl);
+ conn.createStatement().execute(ddl);
+ assertTTL(conn, tableName,
TTLExpression.TTL_EXPRESSION_NOT_DEFINED);
+ assertTTL(conn, parentView,
TTLExpression.TTL_EXPRESSION_NOT_DEFINED);
+ assertConditonTTL(conn, childView, ttl);
+ }
+ }
+
+ @Test
+ public void testRVCUsingPkColsReturnedByPlanShouldUseIndex() throws
Exception {
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ conn.createStatement().execute(
+ "CREATE TABLE T (k VARCHAR NOT NULL PRIMARY KEY, v1
CHAR(15), v2 VARCHAR) " +
+ "TTL='v1=''EXPIRED'''");
+ conn.createStatement().execute("CREATE INDEX IDX ON T(v1, v2)");
+ PhoenixStatement stmt =
conn.createStatement().unwrap(PhoenixStatement.class);
+ String query = "select * from t where (v1, v2, k) > ('1', '2',
'3')";
+ QueryPlan plan = stmt.optimizeQuery(query);
+ assertEquals("IDX",
plan.getTableRef().getTable().getTableName().getString());
+ Scan scan = plan.getContext().getScan();
+ Filter filter = scan.getFilter();
+ assertEquals(filter.toString(), "NOT (TO_CHAR(\"V1\") =
'EXPIRED')");
+ }
+ }
+
+ @Test
+ public void testInListTTLExpr() throws Exception {
+ String ddlTemplate = "create table %s (id varchar not null primary
key, " +
+ "col1 integer, col2 varchar)";
+ String ddlTemplateWithTTL = ddlTemplate + " TTL = '%s'";
+ String tableNoTTL = generateUniqueName();
+ String tableWithTTL = generateUniqueName();
+ String ttl = "col2 IN ('expired', 'cancelled')";
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ String ddl = String.format(ddlTemplate, tableNoTTL);
+ conn.createStatement().execute(ddl);
+ ddl = String.format(ddlTemplateWithTTL, tableWithTTL,
retainSingleQuotes(ttl));
+ conn.createStatement().execute(ddl);
+ assertConditonTTL(conn, tableWithTTL, ttl);
+ String queryTemplate = "select * from %s where id IN ('abc',
'fff')";
+ compareScanWithCondTTL(conn, tableNoTTL, tableWithTTL,
queryTemplate, ttl);
+ }
+ }
+
+ @Test
+ public void testPartialIndex() throws Exception {
+ String ddlTemplate = "create table %s (id varchar not null primary
key, " +
+ "col1 integer, col2 integer, col3 double, col4 varchar)";
+ String ddlTemplateWithTTL = ddlTemplate + " TTL = '%s'";
+ String tableNoTTL = generateUniqueName();
+ String tableWithTTL = generateUniqueName();
+ String indexTemplate = "create index %s on %s (col1) " +
+ "include (col2, col3, col4) where col1 > 50";
+ String indexNoTTL = generateUniqueName();
+ String indexWithTTL = generateUniqueName();
+ String ttl = "col2 > 100 AND col4='expired'";
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ String ddl = String.format(ddlTemplate, tableNoTTL);
+ conn.createStatement().execute(ddl);
+ ddl = String.format(ddlTemplateWithTTL, tableWithTTL,
retainSingleQuotes(ttl));
+ conn.createStatement().execute(ddl);
+ assertConditonTTL(conn, tableWithTTL, ttl);
+ ddl = String.format(indexTemplate, indexNoTTL, tableNoTTL);
+ conn.createStatement().execute(ddl);
+ ddl = String.format(indexTemplate, indexWithTTL, tableWithTTL);
+ conn.createStatement().execute(ddl);
+ assertConditonTTL(conn, indexWithTTL, ttl);
+ String queryTemplate = "select * from %s where col1 > 60";
+ compareScanWithCondTTL(conn, tableNoTTL, tableWithTTL,
queryTemplate, ttl, true);
+ }
+ }
+
+ @Test
+ public void testOrderByOptimizedOut() throws Exception {
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ conn.createStatement().execute(
+ "CREATE TABLE foo (k VARCHAR NOT NULL PRIMARY KEY, v
VARCHAR) " +
+ "IMMUTABLE_ROWS=true,TTL='v=''EXPIRED'''");
+ PhoenixStatement stmt =
conn.createStatement().unwrap(PhoenixStatement.class);
+ QueryPlan plan = stmt.optimizeQuery("SELECT * FROM foo ORDER BY
k");
+ assertEquals(OrderByCompiler.OrderBy.FWD_ROW_KEY_ORDER_BY,
plan.getOrderBy());
+ Scan scan = plan.getContext().getScan();
+ Filter filter = scan.getFilter();
+ assertEquals(filter.toString(), "NOT (V = 'EXPIRED')");
+ }
+ }
+
+ @Test
+ public void testTableSelectionWithMultipleIndexes() throws Exception {
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ conn.createStatement().execute(
+ "CREATE TABLE t (k INTEGER NOT NULL PRIMARY KEY, v1
VARCHAR, v2 VARCHAR) " +
+ "IMMUTABLE_ROWS=true,TTL='v2=''EXPIRED'''");
+ conn.createStatement().execute("CREATE INDEX idx ON t(v1)");
+ PhoenixStatement stmt =
conn.createStatement().unwrap(PhoenixStatement.class);
+ QueryPlan plan = stmt.optimizeQuery("SELECT v1 FROM t WHERE v1 =
'bar'");
+ // T is chosen because TTL expression is on v2 which is not
present in index
+ assertEquals("T",
plan.getTableRef().getTable().getTableName().getString());
+ Scan scan = plan.getContext().getScan();
+ Filter filter = scan.getFilter();
+ assertEquals(filter.toString(), "(V1 = 'bar' AND NOT (V2 =
'EXPIRED'))");
+ conn.createStatement().execute("CREATE INDEX idx2 ON t(v1,v2)");
+ plan = stmt.optimizeQuery("SELECT v1 FROM t WHERE v1 = 'bar'");
+ // Now IDX2 should be chosen
+ assertEquals("IDX2",
plan.getTableRef().getTable().getTableName().getString());
+ scan = plan.getContext().getScan();
+ filter = scan.getFilter();
+ assertEquals(filter.toString(), "NOT (\"V2\" = 'EXPIRED')");
+ }
+ }
+}
diff --git
a/phoenix-core/src/test/java/org/apache/phoenix/schema/ConditionalTTLExpressionDDLTest.java
b/phoenix-core/src/test/java/org/apache/phoenix/schema/ConditionalTTLExpressionDDLTest.java
deleted file mode 100644
index 4a473464d3..0000000000
---
a/phoenix-core/src/test/java/org/apache/phoenix/schema/ConditionalTTLExpressionDDLTest.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.phoenix.schema;
-
-import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.SQLException;
-
-import org.apache.phoenix.exception.PhoenixParserException;
-import org.apache.phoenix.exception.SQLExceptionCode;
-import org.apache.phoenix.jdbc.PhoenixConnection;
-import org.apache.phoenix.query.BaseConnectionlessQueryTest;
-import org.apache.phoenix.util.PropertiesUtil;
-import org.junit.Test;
-
-public class ConditionalTTLExpressionDDLTest extends
BaseConnectionlessQueryTest {
-
- private static void assertConditonTTL(Connection conn, String tableName,
String ttlExpr) throws SQLException {
- TTLExpression expected = new ConditionTTLExpression(ttlExpr);
- assertConditonTTL(conn, tableName, expected);
- }
-
- private static void assertConditonTTL(Connection conn, String tableName,
TTLExpression expected) throws SQLException {
- PTable table =
conn.unwrap(PhoenixConnection.class).getTable(tableName);
- assertEquals(expected, table.getTTL());
- }
-
- @Test
- public void testBasicExpression() throws SQLException {
- String ddlTemplate = "create table %s (k1 bigint not null, k2 bigint
not null, col1 varchar, col2 date " +
- "constraint pk primary key (k1,k2 desc)) TTL = '%s'";
- String quotedValue = "k1 > 5 AND col1 < ''zzzzzz''";
- String ttl = "k1 > 5 AND col1 < 'zzzzzz'";
- String tableName = generateUniqueName();
- String ddl = String.format(ddlTemplate, tableName, quotedValue);
- try (Connection conn = DriverManager.getConnection(getUrl(),
PropertiesUtil.deepCopy(TEST_PROPERTIES))) {
- conn.createStatement().execute(ddl);
- assertConditonTTL(conn, tableName, ttl);
- }
- }
-
- @Test(expected = TypeMismatchException.class)
- public void testNotBooleanExpr() throws SQLException {
- String ddlTemplate = "create table %s (k1 bigint not null, k2 bigint
not null, col1 varchar, col2 date " +
- "constraint pk primary key (k1,k2 desc)) TTL = '%s'";
- String ttl = "k1 + 100";
- String tableName = generateUniqueName();
- String ddl = String.format(ddlTemplate, tableName, ttl);
- try (Connection conn = DriverManager.getConnection(getUrl(),
PropertiesUtil.deepCopy(TEST_PROPERTIES))) {
- conn.createStatement().execute(ddl);
- }
- }
-
- @Test(expected = TypeMismatchException.class)
- public void testWrongArgumentValue() throws SQLException {
- String ddlTemplate = "create table %s (k1 bigint not null, k2 bigint
not null, col1 varchar, col2 date " +
- "constraint pk primary key (k1,k2 desc)) TTL = '%s'";
- String ttl = "k1 = ''abc''";
- String tableName = generateUniqueName();
- String ddl = String.format(ddlTemplate, tableName, ttl);
- try (Connection conn = DriverManager.getConnection(getUrl(),
PropertiesUtil.deepCopy(TEST_PROPERTIES))) {
- conn.createStatement().execute(ddl);
- }
- }
-
- @Test(expected = PhoenixParserException.class)
- public void testParsingError() throws SQLException {
- String ddlTemplate = "create table %s (k1 bigint not null, k2 bigint
not null, col1 varchar, col2 date " +
- "constraint pk primary key (k1,k2 desc)) TTL = '%s'";
- String ttl = "k2 == 23";
- String tableName = generateUniqueName();
- String ddl = String.format(ddlTemplate, tableName, ttl);
- try (Connection conn = DriverManager.getConnection(getUrl(),
PropertiesUtil.deepCopy(TEST_PROPERTIES))) {
- conn.createStatement().execute(ddl);
- }
- }
-
- @Test
- public void testAggregateExpressionNotAllowed() throws SQLException {
- String ddlTemplate = "create table %s (k1 bigint not null, k2 bigint
not null, col1 varchar, col2 date " +
- "constraint pk primary key (k1,k2 desc)) TTL = '%s'";
- String ttl = "SUM(k2) > 23";
- String tableName = generateUniqueName();
- String ddl = String.format(ddlTemplate, tableName, ttl);
- try (Connection conn = DriverManager.getConnection(getUrl(),
PropertiesUtil.deepCopy(TEST_PROPERTIES))) {
- conn.createStatement().execute(ddl);
- fail();
- } catch (SQLException e) {
-
assertEquals(SQLExceptionCode.AGGREGATE_EXPRESSION_NOT_ALLOWED_IN_TTL_EXPRESSION.getErrorCode(),
- e.getErrorCode());
- } catch (Exception e) {
- fail("Unknown exception " + e);
- }
- }
-
- @Test
- public void testNullExpression() throws SQLException {
- String ddlTemplate = "create table %s (k1 bigint not null, k2 bigint
not null, col1 varchar, col2 date " +
- "constraint pk primary key (k1,k2 desc)) TTL = '%s'";
- String ttl = "col1 is NULL AND col2 < CURRENT_DATE() + 30000";
- String tableName = generateUniqueName();
- String ddl = String.format(ddlTemplate, tableName, ttl);
- try (Connection conn = DriverManager.getConnection(getUrl(),
PropertiesUtil.deepCopy(TEST_PROPERTIES))) {
- conn.createStatement().execute(ddl);
- assertConditonTTL(conn, tableName, ttl);
- }
- }
-
- @Test
- public void testBooleanColumn() throws SQLException {
- String ddlTemplate = "create table %s (k1 bigint not null, k2 bigint
not null, expired BOOLEAN " +
- "constraint pk primary key (k1,k2 desc)) TTL = '%s'";
- String ttl = "expired";
- String tableName = generateUniqueName();
- String ddl = String.format(ddlTemplate, tableName, ttl);
- try (Connection conn = DriverManager.getConnection(getUrl(),
PropertiesUtil.deepCopy(TEST_PROPERTIES))) {
- conn.createStatement().execute(ddl);
- assertConditonTTL(conn, tableName, ttl);
- }
- }
-
- @Test
- public void testNot() throws SQLException {
- String ddlTemplate = "create table %s (k1 bigint not null, k2 bigint
not null, expired BOOLEAN " +
- "constraint pk primary key (k1,k2 desc)) TTL = '%s'";
- String ttl = "NOT expired";
- String tableName = generateUniqueName();
- String ddl = String.format(ddlTemplate, tableName, ttl);
- try (Connection conn = DriverManager.getConnection(getUrl(),
PropertiesUtil.deepCopy(TEST_PROPERTIES))) {
- conn.createStatement().execute(ddl);
- assertConditonTTL(conn, tableName, ttl);
- }
- }
-
- @Test
- public void testPhoenixRowTimestamp() throws SQLException {
- String ddlTemplate = "create table %s (k1 bigint not null, k2 bigint
not null, col1 varchar " +
- "constraint pk primary key (k1,k2 desc)) TTL = '%s'";
- String ttl = "PHOENIX_ROW_TIMESTAMP() < CURRENT_DATE() - 100";
- String tableName = generateUniqueName();
- String ddl = String.format(ddlTemplate, tableName, ttl);
- try (Connection conn = DriverManager.getConnection(getUrl(),
PropertiesUtil.deepCopy(TEST_PROPERTIES))) {
- conn.createStatement().execute(ddl);
- assertConditonTTL(conn, tableName, ttl);
- }
- }
-
- @Test
- public void testBooleanCaseExpression() throws SQLException {
- String ddlTemplate = "create table %s (k1 bigint not null, k2 bigint
not null, col1 varchar, status char(1) " +
- "constraint pk primary key (k1,k2 desc)) TTL = '%s'";
- String ttl = "CASE WHEN status = ''E'' THEN TRUE ELSE FALSE END";
- String expectedTTLExpr = "CASE WHEN status = 'E' THEN TRUE ELSE FALSE
END";
- String tableName = generateUniqueName();
- String ddl = String.format(ddlTemplate, tableName, ttl);
- try (Connection conn = DriverManager.getConnection(getUrl(),
PropertiesUtil.deepCopy(TEST_PROPERTIES))) {
- conn.createStatement().execute(ddl);
- assertConditonTTL(conn, tableName, expectedTTLExpr);
- }
- }
-
- @Test
- public void testCondTTLOnTopLevelView() throws SQLException {
- String ddlTemplate = "create table %s (k1 bigint not null primary
key," +
- "k2 bigint, col1 varchar, status char(1))";
- String tableName = generateUniqueName();
- String viewName = generateUniqueName();
- String viewTemplate = "create view %s (k3 smallint) as select * from
%s WHERE k1=7 TTL = '%s'";
- String ttl = "k2 = 34 and k3 = -1";
- String ddl = String.format(ddlTemplate, tableName);
- try (Connection conn = DriverManager.getConnection(getUrl(),
PropertiesUtil.deepCopy(TEST_PROPERTIES))) {
- conn.createStatement().execute(ddl);
- ddl = String.format(viewTemplate, viewName, tableName, ttl);
- conn.createStatement().execute(ddl);
- assertConditonTTL(conn, tableName,
TTLExpression.TTL_EXPRESSION_NOT_DEFINED);
- assertConditonTTL(conn, viewName, ttl);
- }
- }
-
- @Test
- public void testCondTTLOnMultiLevelView() throws SQLException {
- String ddlTemplate = "create table %s (k1 bigint not null primary
key," +
- "k2 bigint, col1 varchar, status char(1))";
- String tableName = generateUniqueName();
- String parentView = generateUniqueName();
- String childView = generateUniqueName();
- String parentViewTemplate = "create view %s (k3 smallint) as select *
from %s WHERE k1=7";
- String childViewTemplate = "create view %s as select * from %s TTL =
'%s'";
- String ttl = "k2 = 34 and k3 = -1";
- String ddl = String.format(ddlTemplate, tableName);
- try (Connection conn = DriverManager.getConnection(getUrl(),
PropertiesUtil.deepCopy(TEST_PROPERTIES))) {
- conn.createStatement().execute(ddl);
- ddl = String.format(parentViewTemplate, parentView, tableName);
- conn.createStatement().execute(ddl);
- ddl = String.format(childViewTemplate, childView, parentView, ttl);
- conn.createStatement().execute(ddl);
- assertConditonTTL(conn, tableName,
TTLExpression.TTL_EXPRESSION_NOT_DEFINED);
- assertConditonTTL(conn, parentView,
TTLExpression.TTL_EXPRESSION_NOT_DEFINED);
- assertConditonTTL(conn, childView, ttl);
- }
- }
-}
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
index 9263a963f5..436c5302a3 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
@@ -1452,4 +1452,16 @@ public class TestUtil {
return
Files.createTempDirectory(Paths.get(System.getProperty("java.io.tmpdir")),
null);
}
+ public static String retainSingleQuotes(String input) {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < input.length(); ++i) {
+ char ch = input.charAt(i);
+ sb.append(ch);
+ if (ch == '\'') {
+ sb.append('\'');
+ }
+ }
+ return sb.toString();
+ }
+
}
