This is an automated email from the ASF dual-hosted git repository.
turcsanyi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 4083e07447 NIFI-10801: Fix hive metastore based tests in Iceberg
processors
4083e07447 is described below
commit 4083e074476cd64d73094fd3720c0dbff20d8f6d
Author: Mark Bathori <[email protected]>
AuthorDate: Fri Nov 11 15:58:07 2022 +0100
NIFI-10801: Fix hive metastore based tests in Iceberg processors
This closes #6656.
Signed-off-by: Peter Turcsanyi <[email protected]>
---
.../nifi-iceberg-processors/pom.xml | 16 ++
.../iceberg/metastore/MetastoreCore.java | 2 +-
...rby.sql => hive-schema-4.0.0-alpha-2.derby.sql} | 199 ++++++++++++++++++---
.../nifi-iceberg-services-api-nar/pom.xml | 12 ++
.../nifi-iceberg-services-api/pom.xml | 12 +-
5 files changed, 212 insertions(+), 29 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-processors/pom.xml
b/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-processors/pom.xml
index f24be5a078..ac96763044 100644
--- a/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-processors/pom.xml
+++ b/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-processors/pom.xml
@@ -155,6 +155,22 @@
<groupId>org.bouncycastle</groupId>
<artifactId>bcprov-jdk18on</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<!-- Test dependencies -->
<dependency>
diff --git
a/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-processors/src/test/java/org/apache/nifi/processors/iceberg/metastore/MetastoreCore.java
b/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-processors/src/test/java/org/apache/nifi/processors/iceberg/metastore/MetastoreCore.java
index dc917a358c..0f9247a19b 100644
---
a/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-processors/src/test/java/org/apache/nifi/processors/iceberg/metastore/MetastoreCore.java
+++
b/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-processors/src/test/java/org/apache/nifi/processors/iceberg/metastore/MetastoreCore.java
@@ -163,7 +163,7 @@ class MetastoreCore {
final Connection connection = DriverManager.getConnection(dbURL);
ScriptRunner scriptRunner = new ScriptRunner(connection);
- final URL initScript =
getClass().getClassLoader().getResource("hive-schema-3.2.0.derby.sql");
+ final URL initScript =
getClass().getClassLoader().getResource("hive-schema-4.0.0-alpha-2.derby.sql");
final Reader reader = new BufferedReader(new
FileReader(initScript.getFile()));
scriptRunner.runScript(reader);
}
diff --git
a/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-processors/src/test/resources/hive-schema-3.2.0.derby.sql
b/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-processors/src/test/resources/hive-schema-4.0.0-alpha-2.derby.sql
similarity index 82%
rename from
nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-processors/src/test/resources/hive-schema-3.2.0.derby.sql
rename to
nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-processors/src/test/resources/hive-schema-4.0.0-alpha-2.derby.sql
index cbc7c935ab..b6beb25324 100644
---
a/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-processors/src/test/resources/hive-schema-3.2.0.derby.sql
+++
b/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-processors/src/test/resources/hive-schema-4.0.0-alpha-2.derby.sql
@@ -36,7 +36,11 @@ CREATE TABLE "APP"."DBS" (
"OWNER_NAME" VARCHAR(128),
"OWNER_TYPE" VARCHAR(10),
"CTLG_NAME" VARCHAR(256) NOT NULL DEFAULT 'hive',
- "CREATE_TIME" INTEGER
+ "CREATE_TIME" INTEGER,
+ "DB_MANAGED_LOCATION_URI" VARCHAR(4000),
+ "TYPE" VARCHAR(32) DEFAULT 'NATIVE' NOT NULL,
+ "DATACONNECTOR_NAME" VARCHAR(128),
+ "REMOTE_DBNAME" VARCHAR(128)
);
CREATE TABLE "APP"."TBL_PRIVS" ("TBL_GRANT_ID" BIGINT NOT NULL, "CREATE_TIME"
INTEGER NOT NULL, "GRANT_OPTION" SMALLINT NOT NULL, "GRANTOR" VARCHAR(128),
"GRANTOR_TYPE" VARCHAR(128), "PRINCIPAL_NAME" VARCHAR(128), "PRINCIPAL_TYPE"
VARCHAR(128), "TBL_PRIV" VARCHAR(128), "TBL_ID" BIGINT, "AUTHORIZER"
VARCHAR(128));
@@ -57,11 +61,13 @@ CREATE TABLE "APP"."PARTITION_KEY_VALS" ("PART_ID" BIGINT
NOT NULL, "PART_KEY_VA
CREATE TABLE "APP"."DB_PRIVS" ("DB_GRANT_ID" BIGINT NOT NULL, "CREATE_TIME"
INTEGER NOT NULL, "DB_ID" BIGINT, "GRANT_OPTION" SMALLINT NOT NULL, "GRANTOR"
VARCHAR(128), "GRANTOR_TYPE" VARCHAR(128), "PRINCIPAL_NAME" VARCHAR(128),
"PRINCIPAL_TYPE" VARCHAR(128), "DB_PRIV" VARCHAR(128), "AUTHORIZER"
VARCHAR(128));
+CREATE TABLE "APP"."DC_PRIVS" ("DC_GRANT_ID" BIGINT NOT NULL, "CREATE_TIME"
INTEGER NOT NULL, "NAME" VARCHAR(128), "GRANT_OPTION" SMALLINT NOT NULL,
"GRANTOR" VARCHAR(128), "GRANTOR_TYPE" VARCHAR(128), "PRINCIPAL_NAME"
VARCHAR(128), "PRINCIPAL_TYPE" VARCHAR(128), "DC_PRIV" VARCHAR(128),
"AUTHORIZER" VARCHAR(128));
+
CREATE TABLE "APP"."IDXS" ("INDEX_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER
NOT NULL, "DEFERRED_REBUILD" CHAR(1) NOT NULL, "INDEX_HANDLER_CLASS"
VARCHAR(4000), "INDEX_NAME" VARCHAR(128), "INDEX_TBL_ID" BIGINT,
"LAST_ACCESS_TIME" INTEGER NOT NULL, "ORIG_TBL_ID" BIGINT, "SD_ID" BIGINT);
CREATE TABLE "APP"."INDEX_PARAMS" ("INDEX_ID" BIGINT NOT NULL, "PARAM_KEY"
VARCHAR(256) NOT NULL, "PARAM_VALUE" VARCHAR(4000));
-CREATE TABLE "APP"."PARTITIONS" ("PART_ID" BIGINT NOT NULL, "CREATE_TIME"
INTEGER NOT NULL, "LAST_ACCESS_TIME" INTEGER NOT NULL, "PART_NAME"
VARCHAR(767), "SD_ID" BIGINT, "TBL_ID" BIGINT);
+CREATE TABLE "APP"."PARTITIONS" ("PART_ID" BIGINT NOT NULL, "CREATE_TIME"
INTEGER NOT NULL, "LAST_ACCESS_TIME" INTEGER NOT NULL, "PART_NAME"
VARCHAR(767), "SD_ID" BIGINT, "TBL_ID" BIGINT, "WRITE_ID" BIGINT DEFAULT 0);
CREATE TABLE "APP"."SERDES" ("SERDE_ID" BIGINT NOT NULL, "NAME" VARCHAR(128),
"SLIB" VARCHAR(4000), "DESCRIPTION" VARCHAR(4000), "SERIALIZER_CLASS"
VARCHAR(4000), "DESERIALIZER_CLASS" VARCHAR(4000), SERDE_TYPE INTEGER);
@@ -89,7 +95,7 @@ CREATE TABLE "APP"."COLUMNS" ("SD_ID" BIGINT NOT NULL,
"COMMENT" VARCHAR(256), "
CREATE TABLE "APP"."ROLES" ("ROLE_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER
NOT NULL, "OWNER_NAME" VARCHAR(128), "ROLE_NAME" VARCHAR(128));
-CREATE TABLE "APP"."TBLS" ("TBL_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT
NULL, "DB_ID" BIGINT, "LAST_ACCESS_TIME" INTEGER NOT NULL, "OWNER"
VARCHAR(767), "OWNER_TYPE" VARCHAR(10), "RETENTION" INTEGER NOT NULL, "SD_ID"
BIGINT, "TBL_NAME" VARCHAR(256), "TBL_TYPE" VARCHAR(128), "VIEW_EXPANDED_TEXT"
LONG VARCHAR, "VIEW_ORIGINAL_TEXT" LONG VARCHAR, "IS_REWRITE_ENABLED" CHAR(1)
NOT NULL DEFAULT 'N');
+CREATE TABLE "APP"."TBLS" ("TBL_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT
NULL, "DB_ID" BIGINT, "LAST_ACCESS_TIME" INTEGER NOT NULL, "OWNER"
VARCHAR(767), "OWNER_TYPE" VARCHAR(10), "RETENTION" INTEGER NOT NULL, "SD_ID"
BIGINT, "TBL_NAME" VARCHAR(256), "TBL_TYPE" VARCHAR(128), "VIEW_EXPANDED_TEXT"
LONG VARCHAR, "VIEW_ORIGINAL_TEXT" LONG VARCHAR, "IS_REWRITE_ENABLED" CHAR(1)
NOT NULL DEFAULT 'N', "WRITE_ID" BIGINT DEFAULT 0);
CREATE TABLE "APP"."PARTITION_KEYS" ("TBL_ID" BIGINT NOT NULL, "PKEY_COMMENT"
VARCHAR(4000), "PKEY_NAME" VARCHAR(128) NOT NULL, "PKEY_TYPE" VARCHAR(767) NOT
NULL, "INTEGER_IDX" INTEGER NOT NULL);
@@ -120,7 +126,8 @@ CREATE TABLE "APP"."TAB_COL_STATS"(
"LAST_ANALYZED" BIGINT,
"CS_ID" BIGINT NOT NULL,
"TBL_ID" BIGINT NOT NULL,
- "BIT_VECTOR" BLOB
+ "BIT_VECTOR" BLOB,
+ "ENGINE" VARCHAR(128) NOT NULL
);
CREATE TABLE "APP"."TABLE_PARAMS" ("TBL_ID" BIGINT NOT NULL, "PARAM_KEY"
VARCHAR(256) NOT NULL, "PARAM_VALUE" CLOB);
@@ -169,7 +176,8 @@ CREATE TABLE "APP"."PART_COL_STATS"(
"NUM_FALSES" BIGINT,
"LAST_ANALYZED" BIGINT,
"CS_ID" BIGINT NOT NULL,
- "PART_ID" BIGINT NOT NULL
+ "PART_ID" BIGINT NOT NULL,
+ "ENGINE" VARCHAR(128) NOT NULL
);
CREATE TABLE "APP"."VERSION" ("VER_ID" BIGINT NOT NULL, "SCHEMA_VERSION"
VARCHAR(127) NOT NULL, "VERSION_COMMENT" VARCHAR(255));
@@ -190,13 +198,29 @@ CREATE TABLE "APP"."NOTIFICATION_LOG" (
"MESSAGE_FORMAT" VARCHAR(16)
);
+CREATE UNIQUE INDEX "APP"."NOTIFICATION_LOG_EVENT_ID" ON
"APP"."NOTIFICATION_LOG" ("EVENT_ID");
+
CREATE TABLE "APP"."NOTIFICATION_SEQUENCE" ("NNI_ID" BIGINT NOT NULL,
"NEXT_EVENT_ID" BIGINT NOT NULL);
-CREATE TABLE "APP"."KEY_CONSTRAINTS" ("CHILD_CD_ID" BIGINT,
"CHILD_INTEGER_IDX" INTEGER, "CHILD_TBL_ID" BIGINT, "PARENT_CD_ID" BIGINT ,
"PARENT_INTEGER_IDX" INTEGER, "PARENT_TBL_ID" BIGINT NOT NULL, "POSITION"
BIGINT NOT NULL, "CONSTRAINT_NAME" VARCHAR(400) NOT NULL, "CONSTRAINT_TYPE"
SMALLINT NOT NULL, "UPDATE_RULE" SMALLINT, "DELETE_RULE" SMALLINT,
"ENABLE_VALIDATE_RELY" SMALLINT NOT NULL, "DEFAULT_VALUE" VARCHAR(400));
+CREATE TABLE "APP"."KEY_CONSTRAINTS" (
+ "CHILD_CD_ID" BIGINT,
+ "CHILD_INTEGER_IDX" INTEGER,
+ "CHILD_TBL_ID" BIGINT,
+ "PARENT_CD_ID" BIGINT,
+ "PARENT_INTEGER_IDX" INTEGER,
+ "PARENT_TBL_ID" BIGINT NOT NULL,
+ "POSITION" BIGINT NOT NULL,
+ "CONSTRAINT_NAME" VARCHAR(400) NOT NULL,
+ "CONSTRAINT_TYPE" SMALLINT NOT NULL,
+ "UPDATE_RULE" SMALLINT,
+ "DELETE_RULE" SMALLINT,
+ "ENABLE_VALIDATE_RELY" SMALLINT NOT NULL,
+ "DEFAULT_VALUE" VARCHAR(400)
+);
CREATE TABLE "APP"."METASTORE_DB_PROPERTIES" ("PROPERTY_KEY" VARCHAR(255) NOT
NULL, "PROPERTY_VALUE" VARCHAR(1000) NOT NULL, "DESCRIPTION" VARCHAR(1000));
-CREATE TABLE "APP"."WM_RESOURCEPLAN" (RP_ID BIGINT NOT NULL, NAME VARCHAR(128)
NOT NULL, QUERY_PARALLELISM INTEGER, STATUS VARCHAR(20) NOT NULL,
DEFAULT_POOL_ID BIGINT);
+CREATE TABLE "APP"."WM_RESOURCEPLAN" (RP_ID BIGINT NOT NULL, NS VARCHAR(128),
NAME VARCHAR(128) NOT NULL, QUERY_PARALLELISM INTEGER, STATUS VARCHAR(20) NOT
NULL, DEFAULT_POOL_ID BIGINT);
CREATE TABLE "APP"."WM_POOL" (POOL_ID BIGINT NOT NULL, RP_ID BIGINT NOT NULL,
PATH VARCHAR(1024) NOT NULL, ALLOC_FRACTION DOUBLE, QUERY_PARALLELISM INTEGER,
SCHEDULING_POLICY VARCHAR(1024));
@@ -217,7 +241,10 @@ CREATE TABLE "APP"."MV_CREATION_METADATA" (
CREATE TABLE "APP"."MV_TABLES_USED" (
"MV_CREATION_METADATA_ID" BIGINT NOT NULL,
- "TBL_ID" BIGINT NOT NULL
+ "TBL_ID" BIGINT NOT NULL,
+ "INSERTED_COUNT" BIGINT NOT NULL DEFAULT 0,
+ "UPDATED_COUNT" BIGINT NOT NULL DEFAULT 0,
+ "DELETED_COUNT" BIGINT NOT NULL DEFAULT 0
);
CREATE TABLE "APP"."CTLGS" (
@@ -228,7 +255,8 @@ CREATE TABLE "APP"."CTLGS" (
"CREATE_TIME" INTEGER);
-- Insert a default value. The location is TBD. Hive will fix this when it
starts
-INSERT INTO "APP"."CTLGS" VALUES (1, 'hive', 'Default catalog for Hive',
'TBD', NULL);
+INSERT INTO "APP"."CTLGS" ("CTLG_ID", "NAME", "DESC", "LOCATION_URI",
"CREATE_TIME")
+VALUES (1, 'hive', 'Default catalog for Hive', 'TBD', NULL);
-- ----------------------------------------------
-- DML Statements
@@ -248,6 +276,8 @@ CREATE INDEX "APP"."TABLECOLUMNPRIVILEGEINDEX" ON
"APP"."TBL_COL_PRIVS" ("AUTHOR
CREATE UNIQUE INDEX "APP"."DBPRIVILEGEINDEX" ON "APP"."DB_PRIVS"
("AUTHORIZER", "DB_ID", "PRINCIPAL_NAME", "PRINCIPAL_TYPE", "DB_PRIV",
"GRANTOR", "GRANTOR_TYPE");
+CREATE UNIQUE INDEX "APP"."DCPRIVILEGEINDEX" ON "APP"."DC_PRIVS"
("AUTHORIZER", "NAME", "PRINCIPAL_NAME", "PRINCIPAL_TYPE", "DC_PRIV",
"GRANTOR", "GRANTOR_TYPE");
+
CREATE INDEX "APP"."PCS_STATS_IDX" ON "APP"."PART_COL_STATS" ("CAT_NAME",
"DB_NAME","TABLE_NAME","COLUMN_NAME","PARTITION_NAME");
CREATE INDEX "APP"."TAB_COL_STATS_IDX" ON "APP"."TAB_COL_STATS" ("CAT_NAME",
"DB_NAME", "TABLE_NAME", "COLUMN_NAME");
@@ -282,7 +312,7 @@ CREATE INDEX "APP"."CONSTRAINTS_PARENT_TBL_ID_INDEX" ON
"APP"."KEY_CONSTRAINTS"(
CREATE INDEX "APP"."CONSTRAINTS_CONSTRAINT_TYPE_INDEX" ON
"APP"."KEY_CONSTRAINTS"("CONSTRAINT_TYPE");
-CREATE UNIQUE INDEX "APP"."UNIQUE_WM_RESOURCEPLAN" ON "APP"."WM_RESOURCEPLAN"
("NAME");
+CREATE UNIQUE INDEX "APP"."UNIQUE_WM_RESOURCEPLAN" ON "APP"."WM_RESOURCEPLAN"
("NS", "NAME");
CREATE UNIQUE INDEX "APP"."UNIQUE_WM_POOL" ON "APP"."WM_POOL" ("RP_ID",
"PATH");
@@ -308,6 +338,8 @@ ALTER TABLE "APP"."CDS" ADD CONSTRAINT "SQL110922153006460"
PRIMARY KEY ("CD_ID"
ALTER TABLE "APP"."DB_PRIVS" ADD CONSTRAINT "DB_PRIVS_PK" PRIMARY KEY
("DB_GRANT_ID");
+ALTER TABLE "APP"."DC_PRIVS" ADD CONSTRAINT "DC_PRIVS_PK" PRIMARY KEY
("DC_GRANT_ID");
+
ALTER TABLE "APP"."INDEX_PARAMS" ADD CONSTRAINT "INDEX_PARAMS_PK" PRIMARY KEY
("INDEX_ID", "PARAM_KEY");
ALTER TABLE "APP"."PARTITION_KEYS" ADD CONSTRAINT "PARTITION_KEY_PK" PRIMARY
KEY ("TBL_ID", "PKEY_NAME");
@@ -386,7 +418,9 @@ ALTER TABLE "APP"."NOTIFICATION_LOG" ADD CONSTRAINT
"NOTIFICATION_LOG_PK" PRIMAR
ALTER TABLE "APP"."NOTIFICATION_SEQUENCE" ADD CONSTRAINT
"NOTIFICATION_SEQUENCE_PK" PRIMARY KEY ("NNI_ID");
-ALTER TABLE "APP"."KEY_CONSTRAINTS" ADD CONSTRAINT "CONSTRAINTS_PK" PRIMARY
KEY ("CONSTRAINT_NAME", "POSITION");
+ALTER TABLE "APP"."NOTIFICATION_SEQUENCE" ADD CONSTRAINT "ONE_ROW_CONSTRAINT"
CHECK (NNI_ID = 1);
+
+ALTER TABLE "APP"."KEY_CONSTRAINTS" ADD CONSTRAINT "CONSTRAINTS_PK" PRIMARY
KEY ("PARENT_TBL_ID", "CONSTRAINT_NAME", "POSITION");
ALTER TABLE "APP"."METASTORE_DB_PROPERTIES" ADD CONSTRAINT "PROPERTY_KEY_PK"
PRIMARY KEY ("PROPERTY_KEY");
@@ -498,6 +532,8 @@ ALTER TABLE "APP"."WM_MAPPING" ADD CONSTRAINT
"WM_MAPPING_FK1" FOREIGN KEY ("RP_
ALTER TABLE "APP"."WM_MAPPING" ADD CONSTRAINT "WM_MAPPING_FK2" FOREIGN KEY
("POOL_ID") REFERENCES "APP"."WM_POOL" ("POOL_ID") ON DELETE NO ACTION ON
UPDATE NO ACTION;
+ALTER TABLE "APP"."MV_TABLES_USED" ADD CONSTRAINT "MV_TABLES_USED_PK" PRIMARY
KEY ("TBL_ID", "MV_CREATION_METADATA_ID");
+
ALTER TABLE "APP"."MV_TABLES_USED" ADD CONSTRAINT "MV_TABLES_USED_FK1" FOREIGN
KEY ("MV_CREATION_METADATA_ID") REFERENCES "APP"."MV_CREATION_METADATA"
("MV_CREATION_METADATA_ID") ON DELETE NO ACTION ON UPDATE NO ACTION;
ALTER TABLE "APP"."MV_TABLES_USED" ADD CONSTRAINT "MV_TABLES_USED_FK2" FOREIGN
KEY ("TBL_ID") REFERENCES "APP"."TBLS" ("TBL_ID") ON DELETE NO ACTION ON UPDATE
NO ACTION;
@@ -516,7 +552,7 @@ ALTER TABLE "APP"."SDS" ADD CONSTRAINT "SQL110318025505550"
CHECK (IS_COMPRESSED
-- Transaction and Lock Tables
-- ----------------------------
CREATE TABLE TXNS (
- TXN_ID bigint PRIMARY KEY,
+ TXN_ID bigint PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY,
TXN_STATE char(1) NOT NULL,
TXN_STARTED bigint NOT NULL,
TXN_LAST_HEARTBEAT bigint NOT NULL,
@@ -528,10 +564,13 @@ CREATE TABLE TXNS (
TXN_TYPE integer
);
+INSERT INTO TXNS (TXN_ID, TXN_STATE, TXN_STARTED, TXN_LAST_HEARTBEAT,
TXN_USER, TXN_HOST)
+ VALUES(0, 'c', 0, 0, '', '');
+
CREATE TABLE TXN_COMPONENTS (
TC_TXNID bigint NOT NULL REFERENCES TXNS (TXN_ID),
TC_DATABASE varchar(128) NOT NULL,
- TC_TABLE varchar(128),
+ TC_TABLE varchar(256),
TC_PARTITION varchar(767),
TC_OPERATION_TYPE char(1) NOT NULL,
TC_WRITEID bigint
@@ -551,17 +590,17 @@ CREATE TABLE COMPLETED_TXN_COMPONENTS (
CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS
(CTC_DATABASE, CTC_TABLE, CTC_PARTITION);
-CREATE TABLE NEXT_TXN_ID (
- NTXN_NEXT bigint NOT NULL
+CREATE TABLE TXN_LOCK_TBL (
+ TXN_LOCK bigint NOT NULL
);
-INSERT INTO NEXT_TXN_ID VALUES(1);
+INSERT INTO TXN_LOCK_TBL VALUES(1);
CREATE TABLE HIVE_LOCKS (
HL_LOCK_EXT_ID bigint NOT NULL,
HL_LOCK_INT_ID bigint NOT NULL,
HL_TXNID bigint NOT NULL,
HL_DB varchar(128) NOT NULL,
- HL_TABLE varchar(128),
+ HL_TABLE varchar(256),
HL_PARTITION varchar(767),
HL_LOCK_STATE char(1) NOT NULL,
HL_LOCK_TYPE char(1) NOT NULL,
@@ -586,17 +625,27 @@ INSERT INTO NEXT_LOCK_ID VALUES(1);
CREATE TABLE COMPACTION_QUEUE (
CQ_ID bigint PRIMARY KEY,
CQ_DATABASE varchar(128) NOT NULL,
- CQ_TABLE varchar(128) NOT NULL,
+ CQ_TABLE varchar(256) NOT NULL,
CQ_PARTITION varchar(767),
CQ_STATE char(1) NOT NULL,
CQ_TYPE char(1) NOT NULL,
CQ_TBLPROPERTIES varchar(2048),
CQ_WORKER_ID varchar(128),
+ CQ_ENQUEUE_TIME bigint,
CQ_START bigint,
CQ_RUN_AS varchar(128),
CQ_HIGHEST_WRITE_ID bigint,
CQ_META_INFO varchar(2048) for bit data,
- CQ_HADOOP_JOB_ID varchar(32)
+ CQ_HADOOP_JOB_ID varchar(32),
+ CQ_ERROR_MESSAGE clob,
+ CQ_NEXT_TXN_ID bigint,
+ CQ_TXN_ID bigint,
+ CQ_COMMIT_TIME bigint,
+ CQ_INITIATOR_ID varchar(128),
+ CQ_INITIATOR_VERSION varchar(128),
+ CQ_WORKER_VERSION varchar(128),
+ CQ_CLEANER_START bigint,
+ CQ_RETRY_RETENTION bigint NOT NULL DEFAULT 0
);
CREATE TABLE NEXT_COMPACTION_QUEUE_ID (
@@ -607,18 +656,38 @@ INSERT INTO NEXT_COMPACTION_QUEUE_ID VALUES(1);
CREATE TABLE COMPLETED_COMPACTIONS (
CC_ID bigint PRIMARY KEY,
CC_DATABASE varchar(128) NOT NULL,
- CC_TABLE varchar(128) NOT NULL,
+ CC_TABLE varchar(256) NOT NULL,
CC_PARTITION varchar(767),
CC_STATE char(1) NOT NULL,
CC_TYPE char(1) NOT NULL,
CC_TBLPROPERTIES varchar(2048),
CC_WORKER_ID varchar(128),
+ CC_ENQUEUE_TIME bigint,
CC_START bigint,
CC_END bigint,
CC_RUN_AS varchar(128),
CC_HIGHEST_WRITE_ID bigint,
CC_META_INFO varchar(2048) for bit data,
- CC_HADOOP_JOB_ID varchar(32)
+ CC_HADOOP_JOB_ID varchar(32),
+ CC_ERROR_MESSAGE clob,
+ CC_NEXT_TXN_ID bigint,
+ CC_TXN_ID bigint,
+ CC_COMMIT_TIME bigint,
+ CC_INITIATOR_ID varchar(128),
+ CC_INITIATOR_VERSION varchar(128),
+ CC_WORKER_VERSION varchar(128)
+);
+
+CREATE INDEX COMPLETED_COMPACTIONS_RES ON COMPLETED_COMPACTIONS
(CC_DATABASE,CC_TABLE,CC_PARTITION);
+
+-- HIVE-25842
+CREATE TABLE COMPACTION_METRICS_CACHE (
+ CMC_DATABASE varchar(128) NOT NULL,
+ CMC_TABLE varchar(256) NOT NULL,
+ CMC_PARTITION varchar(767),
+ CMC_METRIC_TYPE varchar(128) NOT NULL,
+ CMC_METRIC_VALUE integer NOT NULL,
+ CMC_VERSION integer NOT NULL
);
CREATE TABLE AUX_TABLE (
@@ -632,7 +701,7 @@ CREATE TABLE AUX_TABLE (
--This is a good candidate for Index orgainzed table
CREATE TABLE WRITE_SET (
WS_DATABASE varchar(128) NOT NULL,
- WS_TABLE varchar(128) NOT NULL,
+ WS_TABLE varchar(256) NOT NULL,
WS_PARTITION varchar(767),
WS_TXNID bigint NOT NULL,
WS_COMMIT_ID bigint NOT NULL,
@@ -722,7 +791,7 @@ CREATE TABLE TXN_WRITE_NOTIFICATION_LOG (
WNL_TXNID bigint NOT NULL,
WNL_WRITEID bigint NOT NULL,
WNL_DATABASE varchar(128) NOT NULL,
- WNL_TABLE varchar(128) NOT NULL,
+ WNL_TABLE varchar(256) NOT NULL,
WNL_PARTITION varchar(767) NOT NULL,
WNL_TABLE_OBJ clob NOT NULL,
WNL_PARTITION_OBJ clob,
@@ -732,7 +801,89 @@ CREATE TABLE TXN_WRITE_NOTIFICATION_LOG (
);
INSERT INTO SEQUENCE_TABLE (SEQUENCE_NAME, NEXT_VAL) VALUES
('org.apache.hadoop.hive.metastore.model.MTxnWriteNotificationLog', 1);
+CREATE TABLE "APP"."SCHEDULED_QUERIES" (
+ "SCHEDULED_QUERY_ID" bigint primary key not null,
+ "SCHEDULE_NAME" varchar(256) not null,
+ "ENABLED" CHAR(1) NOT NULL DEFAULT 'N',
+ "CLUSTER_NAMESPACE" varchar(256) not null,
+ "USER" varchar(128) not null,
+ "SCHEDULE" varchar(256) not null,
+ "QUERY" varchar(4000) not null,
+ "NEXT_EXECUTION" integer,
+ "ACTIVE_EXECUTION_ID" bigint
+);
+
+CREATE INDEX NEXTEXECUTIONINDEX ON APP.SCHEDULED_QUERIES
(ENABLED,CLUSTER_NAMESPACE,NEXT_EXECUTION);
+CREATE UNIQUE INDEX UNIQUE_SCHEDULED_QUERIES_NAME ON APP.SCHEDULED_QUERIES
(SCHEDULE_NAME,CLUSTER_NAMESPACE);
+
+CREATE TABLE APP.SCHEDULED_EXECUTIONS (
+ SCHEDULED_EXECUTION_ID bigint primary key not null,
+ EXECUTOR_QUERY_ID VARCHAR(256),
+ SCHEDULED_QUERY_ID bigint not null,
+ START_TIME integer not null,
+ END_TIME INTEGER,
+ LAST_UPDATE_TIME INTEGER,
+ ERROR_MESSAGE VARCHAR(2000),
+ STATE VARCHAR(256),
+ CONSTRAINT SCHEDULED_EXECUTIONS_SCHQ_FK FOREIGN KEY
(SCHEDULED_QUERY_ID) REFERENCES APP.SCHEDULED_QUERIES(SCHEDULED_QUERY_ID) ON
DELETE CASCADE
+);
+
+CREATE INDEX LASTUPDATETIMEINDEX ON APP.SCHEDULED_EXECUTIONS
(LAST_UPDATE_TIME);
+CREATE INDEX SCHEDULED_EXECUTIONS_SCHQID ON APP.SCHEDULED_EXECUTIONS
(SCHEDULED_QUERY_ID);
+CREATE UNIQUE INDEX SCHEDULED_EXECUTIONS_UNIQUE_ID ON APP.SCHEDULED_EXECUTIONS
(SCHEDULED_EXECUTION_ID);
+
+--HIVE-23516
+CREATE TABLE "APP"."REPLICATION_METRICS" (
+ "RM_SCHEDULED_EXECUTION_ID" bigint NOT NULL,
+ "RM_POLICY" varchar(256) NOT NULL,
+ "RM_DUMP_EXECUTION_ID" bigint NOT NULL,
+ "RM_METADATA" varchar(4000),
+ "RM_PROGRESS" varchar(10000),
+ "RM_START_TIME" integer not null,
+ "MESSAGE_FORMAT" VARCHAR(16) DEFAULT 'json-0.2',
+ PRIMARY KEY("RM_SCHEDULED_EXECUTION_ID")
+);
+
+CREATE INDEX "POLICY_IDX" ON "APP"."REPLICATION_METRICS" ("RM_POLICY");
+CREATE INDEX "DUMP_IDX" ON "APP"."REPLICATION_METRICS"
("RM_DUMP_EXECUTION_ID");
+
+-- Create stored procedure tables
+CREATE TABLE "APP"."STORED_PROCS" (
+ "SP_ID" BIGINT NOT NULL,
+ "CREATE_TIME" INTEGER NOT NULL,
+ "DB_ID" BIGINT NOT NULL,
+ "NAME" VARCHAR(256) NOT NULL,
+ "OWNER_NAME" VARCHAR(128) NOT NULL,
+ "SOURCE" clob NOT NULL,
+ PRIMARY KEY ("SP_ID")
+);
+
+CREATE UNIQUE INDEX "UNIQUESTOREDPROC" ON "STORED_PROCS" ("NAME", "DB_ID");
+ALTER TABLE "STORED_PROCS" ADD CONSTRAINT "STOREDPROC_FK1" FOREIGN KEY
("DB_ID") REFERENCES "DBS" ("DB_ID");
+
+CREATE TABLE "APP"."DATACONNECTORS" ("NAME" VARCHAR(128) NOT NULL, "TYPE"
VARCHAR(32) NOT NULL, "URL" VARCHAR(4000) NOT NULL, "COMMENT" VARCHAR(256),
"OWNER_NAME" VARCHAR(256), "OWNER_TYPE" VARCHAR(10), "CREATE_TIME" INTEGER NOT
NULL);
+CREATE TABLE "APP"."DATACONNECTOR_PARAMS" ("NAME" VARCHAR(128) NOT NULL,
"PARAM_KEY" VARCHAR(180) NOT NULL, "PARAM_VALUE" VARCHAR(4000));
+ALTER TABLE "APP"."DATACONNECTORS" ADD CONSTRAINT "DATACONNECTORS_KEY_PK"
PRIMARY KEY ("NAME");
+ALTER TABLE "APP"."DATACONNECTOR_PARAMS" ADD CONSTRAINT
"DATACONNECTOR_PARAMS_KEY_PK" PRIMARY KEY ("NAME", "PARAM_KEY");
+ALTER TABLE "APP"."DATACONNECTOR_PARAMS" ADD CONSTRAINT "NAME_FK1" FOREIGN KEY
("NAME") REFERENCES "APP"."DATACONNECTORS" ("NAME") ON DELETE NO ACTION ON
UPDATE NO ACTION;
+ALTER TABLE "APP"."DC_PRIVS" ADD CONSTRAINT "DC_PRIVS_FK1" FOREIGN KEY
("NAME") REFERENCES "APP"."DATACONNECTORS" ("NAME") ON DELETE NO ACTION ON
UPDATE NO ACTION;
+
+-- Create stored procedure packages
+CREATE TABLE "APP"."PACKAGES" (
+ "PKG_ID" BIGINT NOT NULL,
+ "CREATE_TIME" INTEGER NOT NULL,
+ "DB_ID" BIGINT NOT NULL,
+ "NAME" VARCHAR(256) NOT NULL,
+ "OWNER_NAME" VARCHAR(128) NOT NULL,
+ "HEADER" clob NOT NULL,
+ "BODY" clob NOT NULL,
+ PRIMARY KEY ("PKG_ID")
+);
+
+CREATE UNIQUE INDEX "UNIQUEPKG" ON "PACKAGES" ("NAME", "DB_ID");
+ALTER TABLE "PACKAGES" ADD CONSTRAINT "PACKAGES_FK1" FOREIGN KEY ("DB_ID")
REFERENCES "DBS" ("DB_ID");
+
-- -----------------------------------------------------------------
-- Record schema version. Should be the last step in the init script
-- -----------------------------------------------------------------
-INSERT INTO "APP"."VERSION" (VER_ID, SCHEMA_VERSION, VERSION_COMMENT) VALUES
(1, '3.2.0', 'Hive release version 3.2.0');
+INSERT INTO "APP"."VERSION" (VER_ID, SCHEMA_VERSION, VERSION_COMMENT) VALUES
(1, '4.0.0-alpha-2', 'Hive release version 4.0.0-alpha-2');
diff --git
a/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-services-api-nar/pom.xml
b/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-services-api-nar/pom.xml
index c8ed0594ee..977057cf4e 100644
--- a/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-services-api-nar/pom.xml
+++ b/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-services-api-nar/pom.xml
@@ -106,6 +106,18 @@
<artifactId>hadoop-cloud-storage</artifactId>
<version>${hadoop.version}</version>
<exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
diff --git
a/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-services-api/pom.xml
b/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-services-api/pom.xml
index 4cdc87520a..c47f39f8db 100644
--- a/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-services-api/pom.xml
+++ b/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-services-api/pom.xml
@@ -98,6 +98,14 @@
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-web</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-1.2-api</artifactId>
+ </exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
@@ -130,10 +138,6 @@
<groupId>com.tdunning</groupId>
<artifactId>json</artifactId>
</exclusion>
- <exclusion>
- <groupId>javax.transaction</groupId>
- <artifactId>transaction-api</artifactId>
- </exclusion>
<exclusion>
<groupId>com.zaxxer</groupId>
<artifactId>HikariCP</artifactId>
