This is an automated email from the ASF dual-hosted git repository.
vjasani pushed a commit to branch PHOENIX-6978-feature
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/PHOENIX-6978-feature by this
push:
new 40a30ac66a PHOENIX-6979 (Addendum) Use HBase TTL as TTL for Tables
only at Phoenix Level when phoenix.table.ttl.enabled is true (#1670)
40a30ac66a is described below
commit 40a30ac66a7655003f52e816525c3e209d44a083
Author: Lokesh Khurana <khuranalokes...@gmail.com>
AuthorDate: Thu Sep 14 16:21:20 2023 -0700
PHOENIX-6979 (Addendum) Use HBase TTL as TTL for Tables only at Phoenix
Level when phoenix.table.ttl.enabled is true (#1670)
---
.../org/apache/phoenix/end2end/CreateTableIT.java | 15 +++++
.../apache/phoenix/end2end/PropertiesInSyncIT.java | 2 +-
.../org/apache/phoenix/end2end/SetPropertyIT.java | 58 +++++++++++++++++++
.../apache/phoenix/end2end/TTLAsPhoenixTTLIT.java | 66 ++++++++++++++++++++++
.../phoenix/query/ConnectionQueryServicesImpl.java | 13 +++++
.../org/apache/phoenix/schema/MetaDataClient.java | 17 ++++--
6 files changed, 165 insertions(+), 6 deletions(-)
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
index cb493abebd..13bd8732cf 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
@@ -357,6 +357,9 @@ public class CreateTableIT extends ParallelStatsDisabledIT {
admin.getDescriptor(TableName.valueOf(tableName)).getColumnFamilies();
assertEquals(1, columnFamilies.length);
assertEquals(86400, columnFamilies[0].getTimeToLive());
+ //Check if TTL is stored in SYSCAT as well and we are getting ttl from
get api in PTable
+ assertEquals(86400, conn.unwrap(PhoenixConnection.class).getTable(
+ new PTableKey(null, tableName)).getPhoenixTTL());
}
@Test
@@ -413,6 +416,9 @@ public class CreateTableIT extends ParallelStatsDisabledIT {
assertEquals("B", columnFamilies[0].getNameAsString());
assertEquals(86400, columnFamilies[1].getTimeToLive());
assertEquals("C", columnFamilies[1].getNameAsString());
+ //Check if TTL is stored in SYSCAT as well and we are getting ttl from
get api in PTable
+ assertEquals(86400, conn.unwrap(PhoenixConnection.class).getTable(
+ new PTableKey(null, tableName)).getPhoenixTTL());
}
/**
@@ -439,6 +445,9 @@ public class CreateTableIT extends ParallelStatsDisabledIT {
assertEquals(86400, columnFamilies[0].getTimeToLive());
assertEquals("B", columnFamilies[1].getNameAsString());
assertEquals(86400, columnFamilies[1].getTimeToLive());
+ //Check if TTL is stored in SYSCAT as well and we are getting ttl from
get api in PTable
+ assertEquals(86400, conn.unwrap(PhoenixConnection.class).getTable(
+ new PTableKey(null, tableName)).getPhoenixTTL());
}
/**
@@ -517,6 +526,9 @@ public class CreateTableIT extends ParallelStatsDisabledIT {
assertEquals(1, columnFamilies.length);
assertEquals("a", columnFamilies[0].getNameAsString());
assertEquals(10000, columnFamilies[0].getTimeToLive());
+ //Check if TTL is stored in SYSCAT as well and we are getting ttl from
get api in PTable
+ assertEquals(10000, conn.unwrap(PhoenixConnection.class).getTable(
+ new PTableKey(null, tableName)).getPhoenixTTL());
}
/**
@@ -539,6 +551,9 @@ public class CreateTableIT extends ParallelStatsDisabledIT {
assertEquals(1, columnFamilies.length);
assertEquals("a", columnFamilies[0].getNameAsString());
assertEquals(10000, columnFamilies[0].getTimeToLive());
+ //Check if TTL is stored in SYSCAT as well and we are getting ttl from
get api in PTable
+ assertEquals(10000, conn.unwrap(PhoenixConnection.class).getTable(
+ new PTableKey(null, tableName)).getPhoenixTTL());
}
@Test
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/PropertiesInSyncIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/PropertiesInSyncIT.java
index 934ad98dc0..1706e09a91 100644
---
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/PropertiesInSyncIT.java
+++
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/PropertiesInSyncIT.java
@@ -323,7 +323,7 @@ public class PropertiesInSyncIT extends
ParallelStatsDisabledIT {
for (String propName: SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES) {
try {
conn.createStatement().execute("alter table " +
globalIndexName + " set "
- + propName + "=" + DUMMY_PROP_VALUE);
+ + propName + "=" + INITIAL_TTL_VALUE);
fail("Should fail with SQLException when altering synced
property for a global index");
} catch (SQLException sqlE) {
assertEquals("Should fail to alter synced property for a
global index",
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SetPropertyIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SetPropertyIT.java
index 19f40510c9..7a70eb1b0b 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SetPropertyIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SetPropertyIT.java
@@ -29,6 +29,7 @@ import java.sql.SQLException;
import java.util.Map;
import java.util.Properties;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeepDeletedCells;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
@@ -39,6 +40,7 @@ import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.phoenix.coprocessor.BaseScannerRegionObserver;
import org.apache.phoenix.exception.SQLExceptionCode;
import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTableKey;
@@ -426,6 +428,9 @@ public abstract class SetPropertyIT extends
ParallelStatsDisabledIT {
assertEquals(1000, columnFamilies[1].getTimeToLive());
assertEquals(ColumnFamilyDescriptorBuilder.DEFAULT_BLOCKSIZE,
columnFamilies[1].getBlocksize());
assertEquals(Boolean.toString(false),
tableDesc.getValue(TableDescriptorBuilder.COMPACTION_ENABLED));
+ //Check if Alter Table TTL also changes TTL stored in SYSCAT with
phoenix.table.ttl disabled
+ assertEquals(1000, conn.unwrap(PhoenixConnection.class).getTable(
+ new PTableKey(null, dataTableFullName)).getPhoenixTTL());
}
}
@@ -786,6 +791,9 @@ public abstract class SetPropertyIT extends
ParallelStatsDisabledIT {
assertEquals(1, columnFamilies.length);
assertEquals("XYZ", columnFamilies[0].getNameAsString());
assertEquals(86400, columnFamilies[0].getTimeToLive());
+ //Check if TTL is stored in SYSCAT as well and we are getting
ttl from get api in PTable
+ assertEquals(86400,
conn.unwrap(PhoenixConnection.class).getTable(
+ new PTableKey(null,
dataTableFullName)).getPhoenixTTL());
}
ddl = "ALTER TABLE " + dataTableFullName + " SET TTL=30";
conn.createStatement().execute(ddl);
@@ -796,6 +804,50 @@ public abstract class SetPropertyIT extends
ParallelStatsDisabledIT {
assertEquals(1, columnFamilies.length);
assertEquals(30, columnFamilies[0].getTimeToLive());
assertEquals("XYZ", columnFamilies[0].getNameAsString());
+ //Check if Alter Table TTL also changes TTL stored in SYSCAT
with phoenix.table.ttl disabled
+ assertEquals(30, conn.unwrap(PhoenixConnection.class).getTable(
+ new PTableKey(null,
dataTableFullName)).getPhoenixTTL());
+ }
+ } finally {
+ conn.close();
+ }
+ }
+
+ @Test
+ public void testSetTTLForTableWithForeverAndNoneValue() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ conn.setAutoCommit(false);
+ try {
+ String ddl = "create table " + dataTableFullName + " ("
+ + " id char(1) NOT NULL,"
+ + " col1 integer NOT NULL,"
+ + " col2 bigint NOT NULL,"
+ + " CONSTRAINT NAME_PK PRIMARY KEY (id, col1, col2)"
+ + " ) " + generateDDLOptions("TTL=NONE, SALT_BUCKETS = 4,
DEFAULT_COLUMN_FAMILY='XYZ'");
+ conn.createStatement().execute(ddl);
+ try (Admin admin =
conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin()) {
+ TableDescriptor tableDesc =
admin.getDescriptor(TableName.valueOf(dataTableFullName));
+ ColumnFamilyDescriptor[] columnFamilies =
tableDesc.getColumnFamilies();
+ assertEquals(1, columnFamilies.length);
+ assertEquals("XYZ", columnFamilies[0].getNameAsString());
+ assertEquals(HConstants.FOREVER,
columnFamilies[0].getTimeToLive());
+ //Check if TTL is stored in SYSCAT as well and we are getting
ttl from get api in PTable
+ assertEquals(PhoenixDatabaseMetaData.PHOENIX_TTL_NOT_DEFINED,
conn.unwrap(PhoenixConnection.class).getTable(
+ new PTableKey(null,
dataTableFullName)).getPhoenixTTL());
+ }
+ ddl = "ALTER TABLE " + dataTableFullName + " SET TTL=FOREVER";
+ conn.createStatement().execute(ddl);
+ conn.commit();
+ try (Admin admin =
conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin()) {
+ TableDescriptor tableDesc =
admin.getDescriptor(TableName.valueOf(dataTableFullName));
+ ColumnFamilyDescriptor[] columnFamilies =
tableDesc.getColumnFamilies();
+ assertEquals(1, columnFamilies.length);
+ assertEquals(HConstants.FOREVER,
columnFamilies[0].getTimeToLive());
+ assertEquals("XYZ", columnFamilies[0].getNameAsString());
+ //Check if Alter Table TTL also changes TTL stored in SYSCAT
with phoenix.table.ttl disabled
+ assertEquals(HConstants.LATEST_TIMESTAMP,
conn.unwrap(PhoenixConnection.class).getTable(
+ new PTableKey(null,
dataTableFullName)).getPhoenixTTL());
}
} finally {
conn.close();
@@ -942,6 +994,9 @@ public abstract class SetPropertyIT extends
ParallelStatsDisabledIT {
assertEquals("XYZ", columnFamilies[1].getNameAsString());
assertEquals(false, columnFamilies[1].isInMemory());
assertEquals(86400, columnFamilies[1].getTimeToLive());
+ //Check if TTL is stored in SYSCAT as well and we are getting
ttl from get api in PTable
+ assertEquals(86400,
conn.unwrap(PhoenixConnection.class).getTable(
+ new PTableKey(null,
dataTableFullName)).getPhoenixTTL());
}
ddl = "ALTER TABLE " + dataTableFullName + " SET TTL=1000";
@@ -957,6 +1012,9 @@ public abstract class SetPropertyIT extends
ParallelStatsDisabledIT {
assertEquals("XYZ", columnFamilies[1].getNameAsString());
assertEquals(false, columnFamilies[1].isInMemory());
assertEquals(1000, columnFamilies[1].getTimeToLive());
+ //Check if Alter Table TTL also changes TTL stored in SYSCAT
with phoenix.table.ttl disabled
+ assertEquals(1000,
conn.unwrap(PhoenixConnection.class).getTable(
+ new PTableKey(null,
dataTableFullName)).getPhoenixTTL());
}
// the new column will be assigned to the column family XYZ. With
the a KV column getting added for XYZ,
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TTLAsPhoenixTTLIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TTLAsPhoenixTTLIT.java
index d5a1fb5ab3..4a006ab213 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TTLAsPhoenixTTLIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TTLAsPhoenixTTLIT.java
@@ -17,6 +17,11 @@
*/
package org.apache.phoenix.end2end;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.schema.PTable;
@@ -70,6 +75,61 @@ public class TTLAsPhoenixTTLIT extends
ParallelStatsDisabledIT{
Connection conn = DriverManager.getConnection(getUrl());
conn.createStatement().execute(ddl);
assertTTLValueOfTableOrView(conn.unwrap(PhoenixConnection.class),
DEFAULT_TTL_FOR_TEST, tableName);
+ //Setting TTL should not be stored as CF Descriptor properties when
+ //phoenix.table.ttl.enabled is true
+ Admin admin = driver.getConnectionQueryServices(getUrl(), new
Properties()).getAdmin();
+ ColumnFamilyDescriptor[] columnFamilies =
+
admin.getDescriptor(TableName.valueOf(tableName)).getColumnFamilies();
+ assertEquals(ColumnFamilyDescriptorBuilder.DEFAULT_TTL,
columnFamilies[0].getTimeToLive());
+
+ }
+
+ @Test
+ public void testCreateAndAlterTableDDLWithForeverAndNoneTTLValues() throws
Exception {
+ String tableName = generateUniqueName();
+ String ddl =
+ "create table IF NOT EXISTS " + tableName + " (" + " id
char(1) NOT NULL,"
+ + " col1 integer NOT NULL," + " b.col2 bigint," + "
col3 bigint, "
+ + " CONSTRAINT NAME_PK PRIMARY KEY (id, col1)"
+ + " ) TTL=FOREVER";
+ Connection conn = DriverManager.getConnection(getUrl());
+ conn.createStatement().execute(ddl);
+ assertTTLValueOfTableOrView(conn.unwrap(PhoenixConnection.class),
+ HConstants.LATEST_TIMESTAMP, tableName);
+
+ ddl = "ALTER TABLE " + tableName
+ + " SET TTL=NONE";
+ conn.createStatement().execute(ddl);
+ assertTTLValueOfTableOrView(conn.unwrap(PhoenixConnection.class),
+ PhoenixDatabaseMetaData.PHOENIX_TTL_NOT_DEFINED, tableName);
+ //Setting TTL should not be stored as CF Descriptor properties when
+ //phoenix.table.ttl.enabled is true
+ Admin admin = driver.getConnectionQueryServices(getUrl(), new
Properties()).getAdmin();
+ ColumnFamilyDescriptor[] columnFamilies =
+
admin.getDescriptor(TableName.valueOf(tableName)).getColumnFamilies();
+ assertEquals(ColumnFamilyDescriptorBuilder.DEFAULT_TTL,
columnFamilies[0].getTimeToLive());
+
+ tableName = generateUniqueName();
+ ddl =
+ "create table IF NOT EXISTS " + tableName + " (" + " id
char(1) NOT NULL,"
+ + " col1 integer NOT NULL," + " b.col2 bigint," + "
col3 bigint, "
+ + " CONSTRAINT NAME_PK PRIMARY KEY (id, col1)"
+ + " ) TTL=NONE";
+ conn.createStatement().execute(ddl);
+ assertTTLValueOfTableOrView(conn.unwrap(PhoenixConnection.class),
+ PhoenixDatabaseMetaData.PHOENIX_TTL_NOT_DEFINED, tableName);
+
+ ddl = "ALTER TABLE " + tableName
+ + " SET TTL=FOREVER";
+ conn.createStatement().execute(ddl);
+ assertTTLValueOfTableOrView(conn.unwrap(PhoenixConnection.class),
+ HConstants.LATEST_TIMESTAMP, tableName);
+ //Setting TTL should not be stored as CF Descriptor properties when
+ //phoenix.table.ttl.enabled is true
+ columnFamilies =
+
admin.getDescriptor(TableName.valueOf(tableName)).getColumnFamilies();
+ assertEquals(ColumnFamilyDescriptorBuilder.DEFAULT_TTL,
columnFamilies[0].getTimeToLive());
+
}
@Test
@@ -83,6 +143,12 @@ public class TTLAsPhoenixTTLIT extends
ParallelStatsDisabledIT{
+ " SET TTL=1000";
conn.createStatement().execute(ddl);
assertTTLValueOfTableOrView(conn.unwrap(PhoenixConnection.class),
1000, tableName);
+ //Asserting TTL should not be stored as CF Descriptor properties
when
+ //phoenix.table.ttl.enabled is true
+ Admin admin = driver.getConnectionQueryServices(getUrl(), new
Properties()).getAdmin();
+ ColumnFamilyDescriptor[] columnFamilies =
+
admin.getDescriptor(TableName.valueOf(tableName)).getColumnFamilies();
+ assertEquals(ColumnFamilyDescriptorBuilder.DEFAULT_TTL,
columnFamilies[0].getTimeToLive());
}
}
diff --git
a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index ae4e537368..9373f4b74b 100644
---
a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++
b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -2814,6 +2814,8 @@ public class ConnectionQueryServicesImpl extends
DelegateQueryServices implement
.setMessage("Property: " +
propName).build()
.buildException();
}
+ //Handle FOREVER and NONE case
+ propValue =
convertForeverAndNoneTTLValue(propValue);
//If Phoenix level TTL is enabled we are using
TTL as phoenix
//Table level property.
if (!isPhoenixTTLEnabled()) {
@@ -3134,6 +3136,17 @@ public class ConnectionQueryServicesImpl extends
DelegateQueryServices implement
tableDesc.getColumnFamily(SchemaUtil.getEmptyColumnFamily(table)).getTimeToLive();
}
+ public static Object convertForeverAndNoneTTLValue(Object propValue) {
+ //Handle FOREVER and NONE value for TTL at HBase level TTL.
+ if (propValue instanceof String) {
+ String strValue = (String) propValue;
+ if ("FOREVER".equalsIgnoreCase(strValue) ||
"NONE".equalsIgnoreCase(strValue)) {
+ propValue = HConstants.FOREVER;
+ }
+ }
+ return propValue;
+ }
+
/**
* Keep the TTL, KEEP_DELETED_CELLS and REPLICATION_SCOPE properties of
new column families
* in sync with the existing column families. Note that we use the new
values for these properties in case they
diff --git
a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index 1c1cf2e6c7..709b211b00 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -154,6 +154,7 @@ import java.util.Set;
import java.util.HashSet;
import org.apache.hadoop.conf.Configuration;
+import org.apache.phoenix.query.ConnectionQueryServicesImpl;
import org.apache.phoenix.schema.task.SystemTaskParams;
import
org.apache.phoenix.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.hbase.HConstants;
@@ -1058,11 +1059,17 @@ public class MetaDataClient {
.setMessage("Property: " + prop.getFirst()).build()
.buildException();
}
- //If Phoenix Level TTL is enabled use TTL as Phoenix Table
Property as skip
- //TTL at HTableDescriptor level.
- if (isPhoenixTTLEnabled() &&
prop.getFirst().equalsIgnoreCase(TTL)
- && tableType != PTableType.SYSTEM) {
+ //Keeping TTL value as Phoenix Table property irrespective of
PhoenixTTLEnabled or
+ //not to store the value in SYSCAT. To keep
PTableImpl.getPhoenixTTL() consistent
+ //for client side.
+ if (prop.getFirst().equalsIgnoreCase(TTL) && tableType !=
PTableType.SYSTEM) {
tableProps.put(prop.getFirst(), prop.getSecond());
+ if (!isPhoenixTTLEnabled()) {
+ //Handling FOREVER and NONE case for TTL when
phoenix.table.ttl.enable is false.
+ Object value =
ConnectionQueryServicesImpl.convertForeverAndNoneTTLValue(prop.getSecond());
+ commonFamilyProps.put(prop.getFirst(), value);
+ }
+ //If phoenix.table.ttl.enabled is true doesn't store TTL
as columnFamilyProp
continue;
}
@@ -5299,7 +5306,7 @@ public class MetaDataClient {
metaProperties.setColumnEncodedBytesProp(QualifierEncodingScheme.fromSerializedValue((byte)value));
} else if
(propName.equalsIgnoreCase(USE_STATS_FOR_PARALLELIZATION)) {
metaProperties.setUseStatsForParallelizationProp((Boolean)value);
- } else if (propName.equalsIgnoreCase(TTL) &&
isPhoenixTTLEnabled()) {
+ } else if (propName.equalsIgnoreCase(TTL)) {
metaProperties.setPhoenixTTL((Long)value);
} else if
(propName.equalsIgnoreCase(CHANGE_DETECTION_ENABLED)) {
metaProperties.setChangeDetectionEnabled((Boolean)
value);
