http://git-wip-us.apache.org/repos/asf/hive/blob/222b4aca/standalone-metastore/src/main/sql/derby/upgrade-3.1.0-to-4.0.0.derby.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/derby/upgrade-3.1.0-to-4.0.0.derby.sql b/standalone-metastore/src/main/sql/derby/upgrade-3.1.0-to-4.0.0.derby.sql index 38eecd9..41ffe1c 100644 --- a/standalone-metastore/src/main/sql/derby/upgrade-3.1.0-to-4.0.0.derby.sql +++ b/standalone-metastore/src/main/sql/derby/upgrade-3.1.0-to-4.0.0.derby.sql @@ -2,8 +2,10 @@ -- HIVE-19416 ALTER TABLE "APP"."TBLS" ADD WRITEID_LIST CLOB; ALTER TABLE "APP"."TBLS" ADD TXN_ID bigint DEFAULT 0; +ALTER TABLE "APP"."TBLS" ADD WRITE_ID bigint DEFAULT 0; ALTER TABLE "APP"."PARTITIONS" ADD WRITEID_LIST CLOB; ALTER TABLE "APP"."PARTITIONS" ADD TXN_ID bigint DEFAULT 0; +ALTER TABLE "APP"."PARTITIONS" ADD WRITE_ID bigint DEFAULT 0; -- This needs to be the last thing done. Insert any changes above this line. UPDATE "APP".VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1;
http://git-wip-us.apache.org/repos/asf/hive/blob/222b4aca/standalone-metastore/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql index 1f31341..6ce2639 100644 --- a/standalone-metastore/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql +++ b/standalone-metastore/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql @@ -381,6 +381,7 @@ CREATE TABLE TBLS VIEW_ORIGINAL_TEXT text NULL, IS_REWRITE_ENABLED bit NOT NULL DEFAULT 0, TXN_ID bigint NULL, + WRITE_ID bigint NULL, WRITEID_LIST text NULL ); http://git-wip-us.apache.org/repos/asf/hive/blob/222b4aca/standalone-metastore/src/main/sql/mssql/upgrade-3.1.0-to-4.0.0.mssql.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/mssql/upgrade-3.1.0-to-4.0.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/upgrade-3.1.0-to-4.0.0.mssql.sql index 594d165..44939d2 100644 --- a/standalone-metastore/src/main/sql/mssql/upgrade-3.1.0-to-4.0.0.mssql.sql +++ b/standalone-metastore/src/main/sql/mssql/upgrade-3.1.0-to-4.0.0.mssql.sql @@ -3,8 +3,10 @@ SELECT 'Upgrading MetaStore schema from 3.1.0 to 4.0.0' AS MESSAGE; -- HIVE-19416 ALTER TABLE TBLS ADD WRITEID_LIST text NULL; ALTER TABLE TBLS ADD TXN_ID bigint NULL; +ALTER TABLE TBLS ADD WRITE_ID bigint NULL; ALTER TABLE PARTITIONS ADD WRITEID_LIST text NULL; ALTER TABLE PARTITIONS ADD TXN_ID bigint NULL; +ALTER TABLE PARTITIONS ADD WRITE_ID bigint NULL; -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1; http://git-wip-us.apache.org/repos/asf/hive/blob/222b4aca/standalone-metastore/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql index 90f45ac..86fef26 100644 --- a/standalone-metastore/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql +++ b/standalone-metastore/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql @@ -225,6 +225,7 @@ CREATE TABLE IF NOT EXISTS `PARTITIONS` ( `SD_ID` bigint(20) DEFAULT NULL, `TBL_ID` bigint(20) DEFAULT NULL, `TXN_ID` bigint(20) DEFAULT 0, + `WRITE_ID` bigint(20) DEFAULT 0, `WRITEID_LIST` text DEFAULT NULL, PRIMARY KEY (`PART_ID`), UNIQUE KEY `UNIQUEPARTITION` (`PART_NAME`,`TBL_ID`), @@ -632,6 +633,7 @@ CREATE TABLE IF NOT EXISTS `TBLS` ( `VIEW_ORIGINAL_TEXT` mediumtext, `IS_REWRITE_ENABLED` bit(1) NOT NULL DEFAULT 0, `TXN_ID` bigint(20) DEFAULT 0, + `WRITE_ID` bigint(20) DEFAULT 0, `WRITEID_LIST` text DEFAULT NULL, PRIMARY KEY (`TBL_ID`), UNIQUE KEY `UNIQUETABLE` (`TBL_NAME`,`DB_ID`), http://git-wip-us.apache.org/repos/asf/hive/blob/222b4aca/standalone-metastore/src/main/sql/mysql/upgrade-3.1.0-to-4.0.0.mysql.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/mysql/upgrade-3.1.0-to-4.0.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/upgrade-3.1.0-to-4.0.0.mysql.sql index dc011c2..9228cca 100644 --- a/standalone-metastore/src/main/sql/mysql/upgrade-3.1.0-to-4.0.0.mysql.sql +++ b/standalone-metastore/src/main/sql/mysql/upgrade-3.1.0-to-4.0.0.mysql.sql @@ -2,8 +2,10 @@ SELECT 'Upgrading MetaStore schema from 3.1.0 to 4.0.0' AS ' '; -- HIVE-19416 ALTER TABLE TBLS ADD TXN_ID bigint; +ALTER TABLE TBLS ADD WRITE_ID bigint; ALTER TABLE TBLS ADD WRITEID_LIST CLOB; ALTER TABLE PARTITIONS ADD TXN_ID bigint; +ALTER TABLE PARTITIONS ADD WRITE_ID bigint; ALTER TABLE PARTITIONS ADD WRITEID_LIST CLOB; -- These lines need to be last. Insert any changes above. http://git-wip-us.apache.org/repos/asf/hive/blob/222b4aca/standalone-metastore/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql index cc08dc1..dde7025 100644 --- a/standalone-metastore/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql +++ b/standalone-metastore/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql @@ -165,6 +165,7 @@ CREATE TABLE PARTITIONS SD_ID NUMBER NULL, TBL_ID NUMBER NULL, TXN_ID NUMBER NULL, + WRITE_ID NUMBER NULL, WRITEID_LIST CLOB NULL ); @@ -402,6 +403,7 @@ CREATE TABLE TBLS VIEW_ORIGINAL_TEXT CLOB NULL, IS_REWRITE_ENABLED NUMBER(1) DEFAULT 0 NOT NULL CHECK (IS_REWRITE_ENABLED IN (1,0)), TXN_ID NUMBER NULL, + WRITE_ID NUMBER NULL, WRITEID_LIST CLOB NULL ); http://git-wip-us.apache.org/repos/asf/hive/blob/222b4aca/standalone-metastore/src/main/sql/oracle/upgrade-3.1.0-to-4.0.0.oracle.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/oracle/upgrade-3.1.0-to-4.0.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/upgrade-3.1.0-to-4.0.0.oracle.sql index 9e1e6cb..0b75831 100644 --- a/standalone-metastore/src/main/sql/oracle/upgrade-3.1.0-to-4.0.0.oracle.sql +++ b/standalone-metastore/src/main/sql/oracle/upgrade-3.1.0-to-4.0.0.oracle.sql @@ -1,8 +1,10 @@ SELECT 'Upgrading MetaStore schema from 3.1.0 to 4.0.0' AS Status from dual; ALTER TABLE TBLS ADD TXN_ID number NULL; +ALTER TABLE TBLS ADD WRITE_ID number NULL; ALTER TABLE TBLS ADD WRITEID_LIST CLOB NULL; ALTER TABLE PARTITIONS ADD TXN_ID number NULL; +ALTER TABLE PARTITIONS ADD WRITE_ID number NULL; ALTER TABLE PARTITIONS ADD WRITEID_LIST CLOB NULL; -- These lines need to be last. Insert any changes above. http://git-wip-us.apache.org/repos/asf/hive/blob/222b4aca/standalone-metastore/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql index c7add63..ad193f9 100644 --- a/standalone-metastore/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql +++ b/standalone-metastore/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql @@ -396,6 +396,7 @@ CREATE TABLE "TBLS" ( "VIEW_ORIGINAL_TEXT" text, "IS_REWRITE_ENABLED" boolean NOT NULL DEFAULT false, "TXN_ID" bigint, + "WRITE_ID" bigint, "WRITEID_LIST" text ); http://git-wip-us.apache.org/repos/asf/hive/blob/222b4aca/standalone-metastore/src/main/sql/postgres/upgrade-3.1.0-to-4.0.0.postgres.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/postgres/upgrade-3.1.0-to-4.0.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/upgrade-3.1.0-to-4.0.0.postgres.sql index 0692db1..d788719 100644 --- a/standalone-metastore/src/main/sql/postgres/upgrade-3.1.0-to-4.0.0.postgres.sql +++ b/standalone-metastore/src/main/sql/postgres/upgrade-3.1.0-to-4.0.0.postgres.sql @@ -2,8 +2,10 @@ SELECT 'Upgrading MetaStore schema from 3.1.0 to 4.0.0'; -- HIVE-19416 ALTER TABLE "TBLS" ADD "TXN_ID" bigint; +ALTER TABLE "TBLS" ADD "WRITE_ID" bigint; ALTER TABLE "TBLS" ADD "WRITEID_LIST" text; ALTER TABLE "PARTITIONS" ADD "TXN_ID" bigint; +ALTER TABLE "PARTITIONS" ADD "WRITE_ID" bigint; ALTER TABLE "PARTITIONS" ADD "WRITEID_LIST" text; -- These lines need to be last. Insert any changes above. http://git-wip-us.apache.org/repos/asf/hive/blob/222b4aca/standalone-metastore/src/main/thrift/hive_metastore.thrift ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/thrift/hive_metastore.thrift b/standalone-metastore/src/main/thrift/hive_metastore.thrift index ecd2001..1cf2d17 100644 --- a/standalone-metastore/src/main/thrift/hive_metastore.thrift +++ b/standalone-metastore/src/main/thrift/hive_metastore.thrift @@ -438,8 +438,9 @@ struct Table { 17: optional string catName, // Name of the catalog the table is in 18: optional PrincipalType ownerType = PrincipalType.USER, // owner type of this table (default to USER for backward compatibility) 19: optional i64 txnId=-1, - 20: optional string validWriteIdList, - 21: optional IsolationLevelCompliance isStatsCompliant + 20: optional i64 writeId=-1, + 21: optional string validWriteIdList, + 22: optional IsolationLevelCompliance isStatsCompliant } struct Partition { @@ -453,8 +454,9 @@ struct Partition { 8: optional PrincipalPrivilegeSet privileges, 9: optional string catName, 10: optional i64 txnId=-1, - 11: optional string validWriteIdList, - 12: optional IsolationLevelCompliance isStatsCompliant + 11: optional i64 writeId=-1, + 12: optional string validWriteIdList, + 13: optional IsolationLevelCompliance isStatsCompliant } struct PartitionWithoutSD { @@ -483,8 +485,9 @@ struct PartitionSpec { 5: optional PartitionListComposingSpec partitionList, 6: optional string catName, 7: optional i64 txnId=-1, - 8: optional string validWriteIdList, - 9: optional IsolationLevelCompliance isStatsCompliant + 8: optional i64 writeId=-1, + 9: optional string validWriteIdList, + 10: optional IsolationLevelCompliance isStatsCompliant } // column statistics @@ -596,7 +599,8 @@ struct SetPartitionsStatsRequest { 1: required list<ColumnStatistics> colStats, 2: optional bool needMerge, //stats need to be merged with the existing stats 3: optional i64 txnId=-1, // transaction id of the query that sends this structure -4: optional string validWriteIdList // valid write id list for the table for which this struct is being sent +4: optional i64 writeId=-1, // writeId for the current query that updates the stats +5: optional string validWriteIdList // valid write id list for the table for which this struct is being sent } // schema of the table/query results etc. @@ -1584,7 +1588,8 @@ struct AlterPartitionsRequest { 3: required list<Partition> partitions, 4: required EnvironmentContext environmentContext, 5: optional i64 txnId=-1, - 6: optional string validWriteIdList + 6: optional i64 writeId=-1, + 7: optional string validWriteIdList } struct AlterPartitionsResponse { http://git-wip-us.apache.org/repos/asf/hive/blob/222b4aca/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java index 001c3ed..70771cd 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java @@ -363,10 +363,10 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable { @Override public void alterPartitions(String catName, String dbName, String tblName, List<List<String>> partValsList, List<Partition> newParts, - long txnId, String writeIdList) + long txnId, String writeIdList, long writeId) throws InvalidObjectException, MetaException { objectStore.alterPartitions( - catName, dbName, tblName, partValsList, newParts, txnId, writeIdList); + catName, dbName, tblName, partValsList, newParts, txnId, writeIdList, writeId); } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/222b4aca/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java index d6a882e..e9c8993 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java @@ -364,7 +364,7 @@ public class DummyRawStoreForJdoConnection implements RawStore { @Override public void alterPartitions(String catName, String db_name, String tbl_name, List<List<String>> part_vals_list, List<Partition> new_parts, - long txnId, String writeIdList) + long txnId, String writeIdList, long writeId) throws InvalidObjectException, MetaException { } http://git-wip-us.apache.org/repos/asf/hive/blob/222b4aca/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java index 3899f03..28cb3d9 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java @@ -1641,7 +1641,7 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos @Override public void alter_partitions(String dbName, String tblName, List<Partition> newParts, EnvironmentContext environmentContext, - long txnId, String writeIdList) + long txnId, String writeIdList, long writeId) throws InvalidOperationException, MetaException, TException { AlterPartitionsRequest req = new AlterPartitionsRequest(); req.setDbName(dbName); @@ -3303,7 +3303,7 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos public void alter_partitions(String catName, String dbName, String tblName, List<Partition> newParts, EnvironmentContext environmentContext, - long txnId, String writeIdList) throws + long txnId, String writeIdList, long writeId) throws InvalidOperationException, MetaException, TException { throw new UnsupportedOperationException(); } http://git-wip-us.apache.org/repos/asf/hive/blob/222b4aca/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAlterPartitions.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAlterPartitions.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAlterPartitions.java index 0a62ac1..9b9b101 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAlterPartitions.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAlterPartitions.java @@ -843,7 +843,7 @@ public class TestAlterPartitions extends MetaStoreClientTest { createTable4PartColsParts(client); Partition part = client.listPartitions(DB_NAME, TABLE_NAME, (short)-1).get(0); client.alter_partitions("nosuch", DB_NAME, TABLE_NAME, Lists.newArrayList(part), new EnvironmentContext(), - -1, null); + -1, null, -1); } @Test(expected = InvalidOperationException.class) http://git-wip-us.apache.org/repos/asf/hive/blob/222b4aca/storage-api/src/java/org/apache/hive/common/util/TxnIdUtils.java ---------------------------------------------------------------------- diff --git a/storage-api/src/java/org/apache/hive/common/util/TxnIdUtils.java b/storage-api/src/java/org/apache/hive/common/util/TxnIdUtils.java index dc50f1b..4b00864 100644 --- a/storage-api/src/java/org/apache/hive/common/util/TxnIdUtils.java +++ b/storage-api/src/java/org/apache/hive/common/util/TxnIdUtils.java @@ -19,6 +19,8 @@ package org.apache.hive.common.util; import org.apache.hadoop.hive.common.ValidWriteIdList; +import java.util.*; + public class TxnIdUtils { /** @@ -45,15 +47,30 @@ public class TxnIdUtils { * on the table. */ public static boolean areTheseConcurrentWrites( - ValidWriteIdList older, ValidWriteIdList newer, long statsWriteId) { + ValidWriteIdList older, long olderWriteId, + ValidWriteIdList newer, long newerWriteId) { if (!older.getTableName().equalsIgnoreCase(newer.getTableName())) { return false; } assert(older.getHighWatermark() <= newer.getHighWatermark()); - // TODO: Just return false for now. - return false; + // Return false when a write id is not positive. + if (olderWriteId <= 0 || newerWriteId <= 0) { + return false; + } + + // If olderWriteId is for aborted write, return false. + if (newer.isWriteIdAborted(olderWriteId)) { + return false; + } + + // TODO: does this need to account for watermark? + // If either writeId is contained in the other's writeIdList, + // it is a concurrent INSERTs case. + int index2Older = Arrays.binarySearch(older.getInvalidWriteIds(), olderWriteId); + int index2Newer = Arrays.binarySearch(newer.getInvalidWriteIds(), newerWriteId); + return index2Older >= 0 || index2Newer >= 0; } /**
