HIVE-14644 : use metastore information on the read path appropriately (Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3e481b47 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3e481b47 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3e481b47 Branch: refs/heads/hive-14535 Commit: 3e481b4719f49a5c15ce1d745b05392f5d0f2627 Parents: 815e069 Author: Sergey Shelukhin <[email protected]> Authored: Mon Sep 12 13:20:59 2016 -0700 Committer: Sergey Shelukhin <[email protected]> Committed: Mon Sep 12 13:20:59 2016 -0700 ---------------------------------------------------------------------- .../hadoop/hive/common/ValidWriteIds.java | 158 + itests/pom.xml | 28 - itests/qtest-spark/pom.xml | 54 +- metastore/if/hive_metastore.thrift | 12 + .../upgrade/derby/037-HIVE-14637.derby.sql | 4 +- .../upgrade/derby/hive-schema-2.2.0.derby.sql | 2 +- .../upgrade/mssql/022-HIVE-14637.mssql.sql | 4 +- .../upgrade/mssql/hive-schema-2.2.0.mssql.sql | 4 +- .../upgrade/mysql/037-HIVE-14637.mysql.sql | 4 +- .../upgrade/mysql/hive-schema-2.2.0.mysql.sql | 4 +- .../upgrade/oracle/037-HIVE-14637.oracle.sql | 4 +- .../upgrade/oracle/hive-schema-2.2.0.oracle.sql | 4 +- .../postgres/036-HIVE-14637.postgres.sql | 4 +- .../postgres/hive-schema-2.2.0.postgres.sql | 4 +- .../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp | 2403 +++++----- .../gen/thrift/gen-cpp/ThriftHiveMetastore.h | 126 + .../ThriftHiveMetastore_server.skeleton.cpp | 5 + .../gen/thrift/gen-cpp/hive_metastore_types.cpp | 533 ++- .../gen/thrift/gen-cpp/hive_metastore_types.h | 115 + .../metastore/api/GetAllFunctionsResponse.java | 36 +- .../metastore/api/GetValidWriteIdsRequest.java | 490 ++ .../metastore/api/GetValidWriteIdsResult.java | 740 +++ .../hive/metastore/api/ThriftHiveMetastore.java | 4242 +++++++++++------- .../gen-php/metastore/ThriftHiveMetastore.php | 1448 +++--- .../src/gen/thrift/gen-php/metastore/Types.php | 288 +- .../hive_metastore/ThriftHiveMetastore-remote | 7 + .../hive_metastore/ThriftHiveMetastore.py | 1379 +++--- .../gen/thrift/gen-py/hive_metastore/ttypes.py | 212 +- .../gen/thrift/gen-rb/hive_metastore_types.rb | 44 + .../gen/thrift/gen-rb/thrift_hive_metastore.rb | 54 + .../hadoop/hive/metastore/HiveMetaStore.java | 167 +- .../hive/metastore/HiveMetaStoreClient.java | 6 + .../hadoop/hive/metastore/IMetaStoreClient.java | 3 + .../hive/metastore/MetaStoreDirectSql.java | 9 +- .../hadoop/hive/metastore/ObjectStore.java | 128 +- .../apache/hadoop/hive/metastore/RawStore.java | 11 + .../hadoop/hive/metastore/hbase/HBaseStore.java | 27 +- metastore/src/model/package.jdo | 16 +- .../DummyRawStoreControlledCommit.java | 12 + .../DummyRawStoreForJdoConnection.java | 13 +- .../java/org/apache/hadoop/hive/ql/Driver.java | 38 + .../hadoop/hive/ql/exec/FetchOperator.java | 5 + .../apache/hadoop/hive/ql/exec/FetchTask.java | 5 + .../hadoop/hive/ql/exec/FileSinkOperator.java | 49 +- .../apache/hadoop/hive/ql/exec/MoveTask.java | 14 +- .../apache/hadoop/hive/ql/exec/Utilities.java | 52 +- .../org/apache/hadoop/hive/ql/io/AcidUtils.java | 4 - .../hadoop/hive/ql/io/HiveInputFormat.java | 59 +- .../apache/hadoop/hive/ql/metadata/Hive.java | 94 +- .../hive/ql/optimizer/GenMapRedUtils.java | 2 +- .../physical/AnnotateRunTimeStatsOptimizer.java | 3 +- .../hadoop/hive/ql/parse/TaskCompiler.java | 2 +- .../org/apache/hadoop/hive/ql/plan/MapWork.java | 2 + .../apache/hadoop/hive/ql/plan/PlanUtils.java | 14 - .../apache/hadoop/hive/ql/plan/TableDesc.java | 3 +- ql/src/test/queries/clientpositive/mm_current.q | 3 +- .../clientpositive/llap/mm_current.q.out | 8 +- 57 files changed, 8788 insertions(+), 4373 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/3e481b47/common/src/java/org/apache/hadoop/hive/common/ValidWriteIds.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/common/ValidWriteIds.java b/common/src/java/org/apache/hadoop/hive/common/ValidWriteIds.java new file mode 100644 index 0000000..b25a72d --- /dev/null +++ b/common/src/java/org/apache/hadoop/hive/common/ValidWriteIds.java @@ -0,0 +1,158 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common; + +import java.util.Arrays; +import java.util.HashSet; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ValidWriteIds { + public static final ValidWriteIds NO_WRITE_IDS = new ValidWriteIds(-1, -1, false, null); + + private static final String MM_PREFIX = "mm"; + + private final static Logger LOG = LoggerFactory.getLogger(ValidWriteIds.class); + + private static final String VALID_WRITEIDS_PREFIX = "hive.valid.write.ids."; + private final long lowWatermark, highWatermark; + private final boolean areIdsValid; + private final HashSet<Long> ids; + private String source = null; + + public ValidWriteIds( + long lowWatermark, long highWatermark, boolean areIdsValid, HashSet<Long> ids) { + this.lowWatermark = lowWatermark; + this.highWatermark = highWatermark; + this.areIdsValid = areIdsValid; + this.ids = ids; + } + + public static ValidWriteIds createFromConf(Configuration conf, String dbName, String tblName) { + return createFromConf(conf, dbName + "." + tblName); + } + + public static ValidWriteIds createFromConf(Configuration conf, String fullTblName) { + String idStr = conf.get(createConfKey(fullTblName), null); + if (idStr == null || idStr.isEmpty()) return null; + return new ValidWriteIds(idStr); + } + + private static String createConfKey(String dbName, String tblName) { + return createConfKey(dbName + "." + tblName); + } + + private static String createConfKey(String fullName) { + return VALID_WRITEIDS_PREFIX + fullName; + } + + private ValidWriteIds(String src) { + // TODO: lifted from ACID config implementation... optimize if needed? e.g. ranges, base64 + String[] values = src.split(":"); + highWatermark = Long.parseLong(values[0]); + lowWatermark = Long.parseLong(values[1]); + if (values.length > 2) { + areIdsValid = Long.parseLong(values[2]) > 0; + ids = new HashSet<Long>(); + for(int i = 3; i < values.length; ++i) { + ids.add(Long.parseLong(values[i])); + } + } else { + areIdsValid = false; + ids = null; + } + } + + public void addToConf(Configuration conf, String dbName, String tblName) { + if (source == null) { + source = toString(); + } + if (LOG.isDebugEnabled()) { + LOG.debug("Setting " + createConfKey(dbName, tblName) + " => " + source); + } + conf.set(createConfKey(dbName, tblName), source); + } + + public String toString() { + // TODO: lifted from ACID config implementation... optimize if needed? e.g. ranges, base64 + StringBuilder buf = new StringBuilder(); + buf.append(highWatermark); + buf.append(':'); + buf.append(lowWatermark); + if (ids != null) { + buf.append(':'); + buf.append(areIdsValid ? 1 : 0); + for (long id : ids) { + buf.append(':'); + buf.append(id); + } + } + return buf.toString(); + } + + public boolean isValid(long writeId) { + if (writeId < 0) throw new RuntimeException("Incorrect write ID " + writeId); + if (writeId <= lowWatermark) return true; + if (writeId >= highWatermark) return false; + return ids != null && (areIdsValid == ids.contains(writeId)); + } + + public boolean isValidInput(Path file) { + String fileName = file.getName(); + String[] parts = fileName.split("_", 3); + if (parts.length < 2 || !MM_PREFIX.equals(parts[0])) { + LOG.info("Ignoring unknown file for a MM table: " + file + + " (" + Arrays.toString(parts) + ")"); + return false; + } + long writeId = -1; + try { + writeId = Long.parseLong(parts[1]); + } catch (NumberFormatException ex) { + LOG.info("Ignoring unknown file for a MM table: " + file + + "; parsing " + parts[1] + " got " + ex.getMessage()); + return false; + } + return isValid(writeId); + } + + public static String getMmFilePrefix(long mmWriteId) { + return MM_PREFIX + "_" + mmWriteId; + } + + + public static class IdPathFilter implements PathFilter { + private final String prefix, tmpPrefix; + private final boolean isMatch; + public IdPathFilter(long writeId, boolean isMatch) { + this.prefix = ValidWriteIds.getMmFilePrefix(writeId); + this.tmpPrefix = "_tmp." + prefix; + this.isMatch = isMatch; + } + + @Override + public boolean accept(Path path) { + String name = path.getName(); + return isMatch == (name.startsWith(prefix) || name.startsWith(tmpPrefix)); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/3e481b47/itests/pom.xml ---------------------------------------------------------------------- diff --git a/itests/pom.xml b/itests/pom.xml index a452db3..8b5b6b7 100644 --- a/itests/pom.xml +++ b/itests/pom.xml @@ -72,34 +72,6 @@ <target> <echo file="target/download.sh"> set -x - /bin/pwd - BASE_DIR=./target - HIVE_ROOT=$BASE_DIR/../../../ - DOWNLOAD_DIR=./../thirdparty - download() { - url=$1; - finalName=$2 - tarName=$(basename $url) - rm -rf $BASE_DIR/$finalName - if [[ ! -f $DOWNLOAD_DIR/$tarName ]] - then - curl -Sso $DOWNLOAD_DIR/$tarName $url - else - local md5File="$tarName".md5sum - curl -Sso $DOWNLOAD_DIR/$md5File "$url".md5sum - cd $DOWNLOAD_DIR - if ! md5sum -c $md5File; then - curl -Sso $DOWNLOAD_DIR/$tarName $url || return 1 - fi - - cd - - fi - tar -zxf $DOWNLOAD_DIR/$tarName -C $BASE_DIR - mv $BASE_DIR/spark-${spark.version}-bin-hadoop2-without-hive $BASE_DIR/$finalName - } - mkdir -p $DOWNLOAD_DIR - download "http://d3jw87u4immizc.cloudfront.net/spark-tarball/spark-${spark.version}-bin-hadoop2-without-hive.tgz"; "spark" - cp -f $HIVE_ROOT/data/conf/spark/log4j2.properties $BASE_DIR/spark/conf/ </echo> </target> </configuration> http://git-wip-us.apache.org/repos/asf/hive/blob/3e481b47/itests/qtest-spark/pom.xml ---------------------------------------------------------------------- diff --git a/itests/qtest-spark/pom.xml b/itests/qtest-spark/pom.xml index 1e6c3a2..07282fb 100644 --- a/itests/qtest-spark/pom.xml +++ b/itests/qtest-spark/pom.xml @@ -348,6 +348,38 @@ </executions> </plugin> <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <version>${maven.build-helper.plugin.version}</version> + <executions> + <execution> + <id>add-test-sources</id> + <phase>generate-test-sources</phase> + <goals> + <goal>add-test-source</goal> + </goals> + <configuration> + <sources> + <source>target/generated-test-sources/java</source> + </sources> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + +<profiles> + <profile> + <id>spark-test</id> + <activation> + <property> + <name>!skipSparkTests</name> + </property> + </activation> + <build> + <plugins> + <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-antrun-plugin</artifactId> <executions> @@ -388,26 +420,8 @@ </execution> </executions> </plugin> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>build-helper-maven-plugin</artifactId> - <version>${maven.build-helper.plugin.version}</version> - <executions> - <execution> - <id>add-test-sources</id> - <phase>generate-test-sources</phase> - <goals> - <goal>add-test-source</goal> - </goals> - <configuration> - <sources> - <source>target/generated-test-sources/java</source> - </sources> - </configuration> - </execution> - </executions> - </plugin> </plugins> </build> - +</profile> +</profiles> </project> http://git-wip-us.apache.org/repos/asf/hive/blob/3e481b47/metastore/if/hive_metastore.thrift ---------------------------------------------------------------------- diff --git a/metastore/if/hive_metastore.thrift b/metastore/if/hive_metastore.thrift index 95eee27..0101eab 100755 --- a/metastore/if/hive_metastore.thrift +++ b/metastore/if/hive_metastore.thrift @@ -918,6 +918,17 @@ struct HeartbeatWriteIdRequest { struct HeartbeatWriteIdResult { } +struct GetValidWriteIdsRequest { + 1: required string dbName, + 2: required string tblName +} +struct GetValidWriteIdsResult { + 1: required i64 lowWatermarkId, + 2: required i64 highWatermarkId, + 3: optional bool areIdsValid, + 4: optional list<i64> ids +} + struct GetAllFunctionsResponse { 1: optional list<Function> functions @@ -1470,6 +1481,7 @@ service ThriftHiveMetastore extends fb303.FacebookService GetNextWriteIdResult get_next_write_id(1:GetNextWriteIdRequest req) FinalizeWriteIdResult finalize_write_id(1:FinalizeWriteIdRequest req) HeartbeatWriteIdResult heartbeat_write_id(1:HeartbeatWriteIdRequest req) + GetValidWriteIdsResult get_valid_write_ids(1:GetValidWriteIdsRequest req) } // * Note about the DDL_TIME: When creating or altering a table or a partition, http://git-wip-us.apache.org/repos/asf/hive/blob/3e481b47/metastore/scripts/upgrade/derby/037-HIVE-14637.derby.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/derby/037-HIVE-14637.derby.sql b/metastore/scripts/upgrade/derby/037-HIVE-14637.derby.sql index 8cea9f1..88a48f0 100644 --- a/metastore/scripts/upgrade/derby/037-HIVE-14637.derby.sql +++ b/metastore/scripts/upgrade/derby/037-HIVE-14637.derby.sql @@ -1,5 +1,5 @@ -ALTER TABLE "TBLS" ADD "MM_WATERMARK_WRITE_ID" BIGINT; -ALTER TABLE "TBLS" ADD "MM_NEXT_WRITE_ID" BIGINT; +ALTER TABLE "TBLS" ADD "MM_WATERMARK_WRITE_ID" BIGINT DEFAULT -1; +ALTER TABLE "TBLS" ADD "MM_NEXT_WRITE_ID" BIGINT DEFAULT 0; CREATE TABLE "APP"."TBL_WRITES" ("TW_ID" BIGINT NOT NULL, "TBL_ID" BIGINT NOT NULL, "WRITE_ID" BIGINT NOT NULL, "STATE" CHAR(1) NOT NULL, "LAST_HEARTBEAT" BIGINT); ALTER TABLE "APP"."TBL_WRITES" ADD CONSTRAINT "TBL_WRITES_PK" PRIMARY KEY ("TW_ID"); ALTER TABLE "APP"."TBL_WRITES" ADD CONSTRAINT "TBL_WRITES_FK1" FOREIGN KEY ("TBL_ID") REFERENCES "APP"."TBLS" ("TBL_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; http://git-wip-us.apache.org/repos/asf/hive/blob/3e481b47/metastore/scripts/upgrade/derby/hive-schema-2.2.0.derby.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/derby/hive-schema-2.2.0.derby.sql b/metastore/scripts/upgrade/derby/hive-schema-2.2.0.derby.sql index 3c4ba4b..f86ee4a 100644 --- a/metastore/scripts/upgrade/derby/hive-schema-2.2.0.derby.sql +++ b/metastore/scripts/upgrade/derby/hive-schema-2.2.0.derby.sql @@ -60,7 +60,7 @@ CREATE TABLE "APP"."COLUMNS" ("SD_ID" BIGINT NOT NULL, "COMMENT" VARCHAR(256), " CREATE TABLE "APP"."ROLES" ("ROLE_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "OWNER_NAME" VARCHAR(128), "ROLE_NAME" VARCHAR(128)); -CREATE TABLE "APP"."TBLS" ("TBL_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "DB_ID" BIGINT, "LAST_ACCESS_TIME" INTEGER NOT NULL, "OWNER" VARCHAR(767), "RETENTION" INTEGER NOT NULL, "SD_ID" BIGINT, "TBL_NAME" VARCHAR(128), "TBL_TYPE" VARCHAR(128), "VIEW_EXPANDED_TEXT" LONG VARCHAR, "VIEW_ORIGINAL_TEXT" LONG VARCHAR, "MM_WATERMARK_WRITE_ID" BIGINT, "MM_NEXT_WRITE_ID" BIGINT); +CREATE TABLE "APP"."TBLS" ("TBL_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "DB_ID" BIGINT, "LAST_ACCESS_TIME" INTEGER NOT NULL, "OWNER" VARCHAR(767), "RETENTION" INTEGER NOT NULL, "SD_ID" BIGINT, "TBL_NAME" VARCHAR(128), "TBL_TYPE" VARCHAR(128), "VIEW_EXPANDED_TEXT" LONG VARCHAR, "VIEW_ORIGINAL_TEXT" LONG VARCHAR, "MM_WATERMARK_WRITE_ID" BIGINT DEFAULT -1, "MM_NEXT_WRITE_ID" BIGINT DEFAULT 0); CREATE TABLE "APP"."PARTITION_KEYS" ("TBL_ID" BIGINT NOT NULL, "PKEY_COMMENT" VARCHAR(4000), "PKEY_NAME" VARCHAR(128) NOT NULL, "PKEY_TYPE" VARCHAR(767) NOT NULL, "INTEGER_IDX" INTEGER NOT NULL); http://git-wip-us.apache.org/repos/asf/hive/blob/3e481b47/metastore/scripts/upgrade/mssql/022-HIVE-14637.mssql.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/mssql/022-HIVE-14637.mssql.sql b/metastore/scripts/upgrade/mssql/022-HIVE-14637.mssql.sql index bb42935..5d6f99f 100644 --- a/metastore/scripts/upgrade/mssql/022-HIVE-14637.mssql.sql +++ b/metastore/scripts/upgrade/mssql/022-HIVE-14637.mssql.sql @@ -1,5 +1,5 @@ -ALTER TABLE TBLS ADD MM_WATERMARK_WRITE_ID BIGINT NULL; -ALTER TABLE TBLS ADD MM_NEXT_WRITE_ID BIGINT NULL; +ALTER TABLE TBLS ADD MM_WATERMARK_WRITE_ID BIGINT DEFAULT -1; +ALTER TABLE TBLS ADD MM_NEXT_WRITE_ID BIGINT DEFAULT 0; CREATE TABLE TBL_WRITES ( http://git-wip-us.apache.org/repos/asf/hive/blob/3e481b47/metastore/scripts/upgrade/mssql/hive-schema-2.2.0.mssql.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/mssql/hive-schema-2.2.0.mssql.sql b/metastore/scripts/upgrade/mssql/hive-schema-2.2.0.mssql.sql index 6bd0d87..26b2ab3 100644 --- a/metastore/scripts/upgrade/mssql/hive-schema-2.2.0.mssql.sql +++ b/metastore/scripts/upgrade/mssql/hive-schema-2.2.0.mssql.sql @@ -359,8 +359,8 @@ CREATE TABLE TBLS TBL_TYPE nvarchar(128) NULL, VIEW_EXPANDED_TEXT text NULL, VIEW_ORIGINAL_TEXT text NULL, - MM_WATERMARK_WRITE_ID BIGINT NULL, - MM_NEXT_WRITE_ID BIGINT NULL + MM_WATERMARK_WRITE_ID BIGINT NULL DEFAULT -1, + MM_NEXT_WRITE_ID BIGINT NULL DEFAULT 0 ); ALTER TABLE TBLS ADD CONSTRAINT TBLS_PK PRIMARY KEY (TBL_ID); http://git-wip-us.apache.org/repos/asf/hive/blob/3e481b47/metastore/scripts/upgrade/mysql/037-HIVE-14637.mysql.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/mysql/037-HIVE-14637.mysql.sql b/metastore/scripts/upgrade/mysql/037-HIVE-14637.mysql.sql index 1b740d5..c024584 100644 --- a/metastore/scripts/upgrade/mysql/037-HIVE-14637.mysql.sql +++ b/metastore/scripts/upgrade/mysql/037-HIVE-14637.mysql.sql @@ -1,5 +1,5 @@ -alter table `TBLS` ADD COLUMN `MM_WATERMARK_WRITE_ID` bigint(20); -alter table `TBLS` ADD COLUMN `MM_NEXT_WRITE_ID` bigint(20); +alter table `TBLS` ADD COLUMN `MM_WATERMARK_WRITE_ID` bigint(20) DEFAULT -1; +alter table `TBLS` ADD COLUMN `MM_NEXT_WRITE_ID` bigint(20) DEFAULT 0; CREATE TABLE IF NOT EXISTS `TBL_WRITES` ( http://git-wip-us.apache.org/repos/asf/hive/blob/3e481b47/metastore/scripts/upgrade/mysql/hive-schema-2.2.0.mysql.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/mysql/hive-schema-2.2.0.mysql.sql b/metastore/scripts/upgrade/mysql/hive-schema-2.2.0.mysql.sql index f7ef948..b295950 100644 --- a/metastore/scripts/upgrade/mysql/hive-schema-2.2.0.mysql.sql +++ b/metastore/scripts/upgrade/mysql/hive-schema-2.2.0.mysql.sql @@ -587,8 +587,8 @@ CREATE TABLE IF NOT EXISTS `TBLS` ( `TBL_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, `VIEW_EXPANDED_TEXT` mediumtext, `VIEW_ORIGINAL_TEXT` mediumtext, - `MM_WATERMARK_WRITE_ID` bigint(20), - `MM_NEXT_WRITE_ID` bigint(20), + `MM_WATERMARK_WRITE_ID` bigint(20) DEFAULT -1, + `MM_NEXT_WRITE_ID` bigint(20) DEFAULT 0, PRIMARY KEY (`TBL_ID`), UNIQUE KEY `UNIQUETABLE` (`TBL_NAME`,`DB_ID`), KEY `TBLS_N50` (`SD_ID`), http://git-wip-us.apache.org/repos/asf/hive/blob/3e481b47/metastore/scripts/upgrade/oracle/037-HIVE-14637.oracle.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/oracle/037-HIVE-14637.oracle.sql b/metastore/scripts/upgrade/oracle/037-HIVE-14637.oracle.sql index bc5fb6b..9f6dbb2 100644 --- a/metastore/scripts/upgrade/oracle/037-HIVE-14637.oracle.sql +++ b/metastore/scripts/upgrade/oracle/037-HIVE-14637.oracle.sql @@ -1,5 +1,5 @@ -ALTER TABLE TBLS ADD MM_WATERMARK_WRITE_ID NUMBER; -ALTER TABLE TBLS ADD MM_NEXT_WRITE_ID NUMBER; +ALTER TABLE TBLS ADD MM_WATERMARK_WRITE_ID NUMBER DEFAULT -1; +ALTER TABLE TBLS ADD MM_NEXT_WRITE_ID NUMBER DEFAULT 0; CREATE TABLE TBL_WRITES ( http://git-wip-us.apache.org/repos/asf/hive/blob/3e481b47/metastore/scripts/upgrade/oracle/hive-schema-2.2.0.oracle.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/oracle/hive-schema-2.2.0.oracle.sql b/metastore/scripts/upgrade/oracle/hive-schema-2.2.0.oracle.sql index 503ce09..6972c20 100644 --- a/metastore/scripts/upgrade/oracle/hive-schema-2.2.0.oracle.sql +++ b/metastore/scripts/upgrade/oracle/hive-schema-2.2.0.oracle.sql @@ -376,8 +376,8 @@ CREATE TABLE TBLS TBL_TYPE VARCHAR2(128) NULL, VIEW_EXPANDED_TEXT CLOB NULL, VIEW_ORIGINAL_TEXT CLOB NULL, - MM_WATERMARK_WRITE_ID NUMBER NULL, - MM_NEXT_WRITE_ID NUMBER NULL + MM_WATERMARK_WRITE_ID NUMBER DEFAULT -1, + MM_NEXT_WRITE_ID NUMBER DEFAULT 0 ); ALTER TABLE TBLS ADD CONSTRAINT TBLS_PK PRIMARY KEY (TBL_ID); http://git-wip-us.apache.org/repos/asf/hive/blob/3e481b47/metastore/scripts/upgrade/postgres/036-HIVE-14637.postgres.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/postgres/036-HIVE-14637.postgres.sql b/metastore/scripts/upgrade/postgres/036-HIVE-14637.postgres.sql index d94c19d..f153837 100644 --- a/metastore/scripts/upgrade/postgres/036-HIVE-14637.postgres.sql +++ b/metastore/scripts/upgrade/postgres/036-HIVE-14637.postgres.sql @@ -1,6 +1,6 @@ -ALTER TABLE "TBLS" ADD COLUMN "MM_WATERMARK_WRITE_ID" bigint; -ALTER TABLE "TBLS" ADD COLUMN "MM_NEXT_WRITE_ID" bigint; +ALTER TABLE "TBLS" ADD COLUMN "MM_WATERMARK_WRITE_ID" bigint DEFAULT -1; +ALTER TABLE "TBLS" ADD COLUMN "MM_NEXT_WRITE_ID" bigint DEFAULT 0; CREATE TABLE "TBL_WRITES" ( http://git-wip-us.apache.org/repos/asf/hive/blob/3e481b47/metastore/scripts/upgrade/postgres/hive-schema-2.2.0.postgres.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/postgres/hive-schema-2.2.0.postgres.sql b/metastore/scripts/upgrade/postgres/hive-schema-2.2.0.postgres.sql index bf1d769..de997d3 100644 --- a/metastore/scripts/upgrade/postgres/hive-schema-2.2.0.postgres.sql +++ b/metastore/scripts/upgrade/postgres/hive-schema-2.2.0.postgres.sql @@ -373,8 +373,8 @@ CREATE TABLE "TBLS" ( "TBL_TYPE" character varying(128) DEFAULT NULL::character varying, "VIEW_EXPANDED_TEXT" text, "VIEW_ORIGINAL_TEXT" text, - "MM_WATERMARK_WRITE_ID" bigint, - "MM_NEXT_WRITE_ID" bigint + "MM_WATERMARK_WRITE_ID" bigint DEFAULT -1, + "MM_NEXT_WRITE_ID" bigint DEFAULT 0 );
