Repository: incubator-ranger Updated Branches: refs/heads/master cafe86970 -> dd996d25b
RANGER-430 : Need additional database columns to support log aggregation at source Signed-off-by: Velmurugan Periasamy <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/dd996d25 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/dd996d25 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/dd996d25 Branch: refs/heads/master Commit: dd996d25bb976c9f723eab0b5f89907225f2c039 Parents: cafe869 Author: Gautam Borad <[email protected]> Authored: Mon Apr 27 22:33:09 2015 +0530 Committer: Velmurugan Periasamy <[email protected]> Committed: Mon Apr 27 16:09:00 2015 -0400 ---------------------------------------------------------------------- .../db/mysql/patches/011-auditcolumnssize.sql | 36 ---- .../patches/audit/011-auditcolumnssize.sql | 36 ++++ .../patches/audit/015-auditlogaggregation.sql | 44 ++++ .../db/oracle/patches/011-auditcolumnssize.sql | 39 ---- .../oracle/patches/012-createusersyncuser.sql | 6 +- .../patches/audit/011-auditcolumnssize.sql | 39 ++++ .../patches/audit/015-auditlogaggregation.sql | 56 +++++ .../db/postgres/xa_audit_db_postgres.sql | 3 + .../db/sqlserver/xa_audit_db_sqlserver.sql | 3 + security-admin/scripts/db_setup.py | 214 +++++++++++++++++-- .../org/apache/ranger/entity/XXAccessAudit.java | 32 +++ .../ranger/service/XAccessAuditService.java | 6 + .../ranger/service/XAccessAuditServiceBase.java | 6 + .../ranger/solr/SolrAccessAuditsService.java | 12 ++ .../org/apache/ranger/view/VXAccessAudit.java | 34 +++ .../webapp/scripts/views/reports/AuditLayout.js | 18 ++ 16 files changed, 489 insertions(+), 95 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/dd996d25/security-admin/db/mysql/patches/011-auditcolumnssize.sql ---------------------------------------------------------------------- diff --git a/security-admin/db/mysql/patches/011-auditcolumnssize.sql b/security-admin/db/mysql/patches/011-auditcolumnssize.sql deleted file mode 100644 index b38ae8c..0000000 --- a/security-admin/db/mysql/patches/011-auditcolumnssize.sql +++ /dev/null @@ -1,36 +0,0 @@ --- Licensed to the Apache Software Foundation (ASF) under one or more --- contributor license agreements. See the NOTICE file distributed with --- this work for additional information regarding copyright ownership. --- The ASF licenses this file to You under the Apache License, Version 2.0 --- (the "License"); you may not use this file except in compliance with --- the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. - -drop procedure if exists increase_column_size_of_xa_access_audit_table; - -delimiter ;; -create procedure increase_column_size_of_xa_access_audit_table() begin - - /* change request_data data size from 2000 to 4000 */ - if exists (select * from information_schema.columns where table_schema=database() and table_name = 'xa_access_audit' and column_name = 'request_data' and data_type='varchar' and CHARACTER_MAXIMUM_LENGTH=2000) then - ALTER TABLE `xa_access_audit` CHANGE `request_data` `request_data` VARCHAR(4000) NULL DEFAULT NULL ; - end if; - - /* change resource_path data size from 2000 to 4000 */ - if exists (select * from information_schema.columns where table_schema=database() and table_name = 'xa_access_audit' and column_name = 'resource_path' and data_type='varchar' and CHARACTER_MAXIMUM_LENGTH=2000) then - ALTER TABLE `xa_access_audit` CHANGE `resource_path` `resource_path` VARCHAR(4000) NULL DEFAULT NULL ; - end if; - -end;; - -delimiter ; -call increase_column_size_of_xa_access_audit_table(); - -drop procedure if exists increase_column_size_of_xa_access_audit_table; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/dd996d25/security-admin/db/mysql/patches/audit/011-auditcolumnssize.sql ---------------------------------------------------------------------- diff --git a/security-admin/db/mysql/patches/audit/011-auditcolumnssize.sql b/security-admin/db/mysql/patches/audit/011-auditcolumnssize.sql new file mode 100644 index 0000000..b38ae8c --- /dev/null +++ b/security-admin/db/mysql/patches/audit/011-auditcolumnssize.sql @@ -0,0 +1,36 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +drop procedure if exists increase_column_size_of_xa_access_audit_table; + +delimiter ;; +create procedure increase_column_size_of_xa_access_audit_table() begin + + /* change request_data data size from 2000 to 4000 */ + if exists (select * from information_schema.columns where table_schema=database() and table_name = 'xa_access_audit' and column_name = 'request_data' and data_type='varchar' and CHARACTER_MAXIMUM_LENGTH=2000) then + ALTER TABLE `xa_access_audit` CHANGE `request_data` `request_data` VARCHAR(4000) NULL DEFAULT NULL ; + end if; + + /* change resource_path data size from 2000 to 4000 */ + if exists (select * from information_schema.columns where table_schema=database() and table_name = 'xa_access_audit' and column_name = 'resource_path' and data_type='varchar' and CHARACTER_MAXIMUM_LENGTH=2000) then + ALTER TABLE `xa_access_audit` CHANGE `resource_path` `resource_path` VARCHAR(4000) NULL DEFAULT NULL ; + end if; + +end;; + +delimiter ; +call increase_column_size_of_xa_access_audit_table(); + +drop procedure if exists increase_column_size_of_xa_access_audit_table; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/dd996d25/security-admin/db/mysql/patches/audit/015-auditlogaggregation.sql ---------------------------------------------------------------------- diff --git a/security-admin/db/mysql/patches/audit/015-auditlogaggregation.sql b/security-admin/db/mysql/patches/audit/015-auditlogaggregation.sql new file mode 100644 index 0000000..c88c5b4 --- /dev/null +++ b/security-admin/db/mysql/patches/audit/015-auditlogaggregation.sql @@ -0,0 +1,44 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +drop procedure if exists add_columns_to_support_audit_log_aggregation; + +delimiter ;; +create procedure add_columns_to_support_audit_log_aggregation() begin + + if exists (select * from information_schema.columns where table_schema=database() and table_name = 'xa_access_audit') then + if not exists (select * from information_schema.columns where table_schema=database() and table_name = 'xa_access_audit' and column_name = 'seq_num') then + ALTER TABLE `xa_access_audit` ADD `seq_num` bigint NULL DEFAULT 0; + end if; + end if; + + if exists (select * from information_schema.columns where table_schema=database() and table_name = 'xa_access_audit') then + if not exists (select * from information_schema.columns where table_schema=database() and table_name = 'xa_access_audit' and column_name = 'event_count') then + ALTER TABLE `xa_access_audit` ADD `event_count` bigint NULL DEFAULT 1; + end if; + end if; + + if exists (select * from information_schema.columns where table_schema=database() and table_name = 'xa_access_audit') then + if not exists (select * from information_schema.columns where table_schema=database() and table_name = 'xa_access_audit' and column_name = 'event_dur_ms') then + ALTER TABLE `xa_access_audit` ADD `event_dur_ms` bigint NULL DEFAULT 1; + end if; + end if; + +end;; + +delimiter ; +call add_columns_to_support_audit_log_aggregation(); + +drop procedure if exists add_columns_to_support_audit_log_aggregation; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/dd996d25/security-admin/db/oracle/patches/011-auditcolumnssize.sql ---------------------------------------------------------------------- diff --git a/security-admin/db/oracle/patches/011-auditcolumnssize.sql b/security-admin/db/oracle/patches/011-auditcolumnssize.sql deleted file mode 100644 index 35a9320..0000000 --- a/security-admin/db/oracle/patches/011-auditcolumnssize.sql +++ /dev/null @@ -1,39 +0,0 @@ --- Licensed to the Apache Software Foundation (ASF) under one or more --- contributor license agreements. See the NOTICE file distributed with --- this work for additional information regarding copyright ownership. --- The ASF licenses this file to You under the Apache License, Version 2.0 --- (the "License"); you may not use this file except in compliance with --- the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. - - -DECLARE - v_column_exists number:=0; -BEGIN - Select count(*) into v_column_exists - from user_tab_cols - where column_name = upper('REQUEST_DATA') - and table_name = upper('XA_ACCESS_AUDIT') and DATA_TYPE='VARCHAR2' and DATA_LENGTH=2000; - - if (v_column_exists = 1) then - execute immediate 'ALTER TABLE XA_ACCESS_AUDIT modify(REQUEST_DATA VARCHAR(4000) DEFAULT NULL)'; - commit; - end if; - v_column_exists:=0; - Select count(*) into v_column_exists - from user_tab_cols - where column_name = upper('RESOURCE_PATH') - and table_name = upper('XA_ACCESS_AUDIT') and DATA_TYPE='VARCHAR2' and DATA_LENGTH=2000; - - if (v_column_exists = 1) then - execute immediate 'ALTER TABLE XA_ACCESS_AUDIT modify(RESOURCE_PATH VARCHAR(4000) DEFAULT NULL)'; - commit; - end if; -end;/ \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/dd996d25/security-admin/db/oracle/patches/012-createusersyncuser.sql ---------------------------------------------------------------------- diff --git a/security-admin/db/oracle/patches/012-createusersyncuser.sql b/security-admin/db/oracle/patches/012-createusersyncuser.sql index 5b99b1f..b5d0764 100644 --- a/security-admin/db/oracle/patches/012-createusersyncuser.sql +++ b/security-admin/db/oracle/patches/012-createusersyncuser.sql @@ -17,9 +17,9 @@ DECLARE v_count number:=0; loginID number:=0; sql_stmt VARCHAR2(1000); - first_name VARCHAR2(10):='rangerusersync'; - scr_name VARCHAR2(10):='rangerusersync'; - login_name VARCHAR2(10):='rangerusersync'; + first_name VARCHAR2(20):='rangerusersync'; + scr_name VARCHAR2(20):='rangerusersync'; + login_name VARCHAR2(20):='rangerusersync'; password VARCHAR2(50):='70b8374d3dfe0325aaa5002a688c7e3b'; user_role VARCHAR2(50):='ROLE_SYS_ADMIN'; email VARCHAR2(20):='rangerusersync'; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/dd996d25/security-admin/db/oracle/patches/audit/011-auditcolumnssize.sql ---------------------------------------------------------------------- diff --git a/security-admin/db/oracle/patches/audit/011-auditcolumnssize.sql b/security-admin/db/oracle/patches/audit/011-auditcolumnssize.sql new file mode 100644 index 0000000..35a9320 --- /dev/null +++ b/security-admin/db/oracle/patches/audit/011-auditcolumnssize.sql @@ -0,0 +1,39 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + + +DECLARE + v_column_exists number:=0; +BEGIN + Select count(*) into v_column_exists + from user_tab_cols + where column_name = upper('REQUEST_DATA') + and table_name = upper('XA_ACCESS_AUDIT') and DATA_TYPE='VARCHAR2' and DATA_LENGTH=2000; + + if (v_column_exists = 1) then + execute immediate 'ALTER TABLE XA_ACCESS_AUDIT modify(REQUEST_DATA VARCHAR(4000) DEFAULT NULL)'; + commit; + end if; + v_column_exists:=0; + Select count(*) into v_column_exists + from user_tab_cols + where column_name = upper('RESOURCE_PATH') + and table_name = upper('XA_ACCESS_AUDIT') and DATA_TYPE='VARCHAR2' and DATA_LENGTH=2000; + + if (v_column_exists = 1) then + execute immediate 'ALTER TABLE XA_ACCESS_AUDIT modify(RESOURCE_PATH VARCHAR(4000) DEFAULT NULL)'; + commit; + end if; +end;/ \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/dd996d25/security-admin/db/oracle/patches/audit/015-auditlogaggregation.sql ---------------------------------------------------------------------- diff --git a/security-admin/db/oracle/patches/audit/015-auditlogaggregation.sql b/security-admin/db/oracle/patches/audit/015-auditlogaggregation.sql new file mode 100644 index 0000000..756ee61 --- /dev/null +++ b/security-admin/db/oracle/patches/audit/015-auditlogaggregation.sql @@ -0,0 +1,56 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +DECLARE + v_column_exists number := 0; +BEGIN + Select count(*) into v_column_exists + from user_tab_cols + where column_name = upper('seq_num') + and table_name = upper('XA_ACCESS_AUDIT'); + + if (v_column_exists = 0) then + execute immediate 'ALTER TABLE XA_ACCESS_AUDIT ADD seq_num NUMBER(20) DEFAULT 0 NULL'; + commit; + end if; +end;/ + +DECLARE + v_column_exists number := 0; +BEGIN + Select count(*) into v_column_exists + from user_tab_cols + where column_name = upper('event_count') + and table_name = upper('XA_ACCESS_AUDIT'); + + if (v_column_exists = 0) then + execute immediate 'ALTER TABLE XA_ACCESS_AUDIT ADD event_count NUMBER(20) DEFAULT 1 NULL'; + commit; + end if; +end;/ + +DECLARE + v_column_exists number := 0; +BEGIN + Select count(*) into v_column_exists + from user_tab_cols + where column_name = upper('event_dur_ms') + and table_name = upper('XA_ACCESS_AUDIT'); + + if (v_column_exists = 0) then + execute immediate 'ALTER TABLE XA_ACCESS_AUDIT ADD event_dur_ms NUMBER(20) DEFAULT 1 NULL'; + commit; + end if; +end;/ \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/dd996d25/security-admin/db/postgres/xa_audit_db_postgres.sql ---------------------------------------------------------------------- diff --git a/security-admin/db/postgres/xa_audit_db_postgres.sql b/security-admin/db/postgres/xa_audit_db_postgres.sql index d5f73c3..c12a854 100644 --- a/security-admin/db/postgres/xa_audit_db_postgres.sql +++ b/security-admin/db/postgres/xa_audit_db_postgres.sql @@ -40,6 +40,9 @@ action VARCHAR(2000) DEFAULT NULL NULL, request_data VARCHAR(4000) DEFAULT NULL NULL, resource_path VARCHAR(4000) DEFAULT NULL NULL, resource_type VARCHAR(255) DEFAULT NULL NULL, +seq_num BIGINT DEFAULT '0' NULL, +event_count BIGINT DEFAULT '1' NULL, +event_dur_ms BIGINT DEFAULT '1' NULL, PRIMARY KEY (id) ); CREATE INDEX xa_access_audit_added_by_id ON xa_access_audit(added_by_id); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/dd996d25/security-admin/db/sqlserver/xa_audit_db_sqlserver.sql ---------------------------------------------------------------------- diff --git a/security-admin/db/sqlserver/xa_audit_db_sqlserver.sql b/security-admin/db/sqlserver/xa_audit_db_sqlserver.sql index 83d4fa1..effe509 100644 --- a/security-admin/db/sqlserver/xa_audit_db_sqlserver.sql +++ b/security-admin/db/sqlserver/xa_audit_db_sqlserver.sql @@ -43,6 +43,9 @@ CREATE TABLE [dbo].[xa_access_audit]( [request_data] [varchar](4000) DEFAULT NULL NULL, [resource_path] [varchar](4000) DEFAULT NULL NULL, [resource_type] [varchar](255) DEFAULT NULL NULL, + [seq_num] [bigint] DEFAULT 0 NULL, + [event_count] [bigint] DEFAULT 1 NULL, + [event_dur_ms] [bigint] DEFAULT 1 NULL, PRIMARY KEY CLUSTERED ( [id] ASC http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/dd996d25/security-admin/scripts/db_setup.py ---------------------------------------------------------------------- diff --git a/security-admin/scripts/db_setup.py b/security-admin/scripts/db_setup.py index 16a5ff3..e7d0a17 100644 --- a/security-admin/scripts/db_setup.py +++ b/security-admin/scripts/db_setup.py @@ -129,6 +129,21 @@ class BaseDB(object): def auditdb_operation(self, xa_db_host, audit_db_host, db_name, audit_db_name, db_user, audit_db_user, db_password, audit_db_password, file_name, TABLE_NAME): log("[I] ----------------- Audit DB operations ------------", "info") + def apply_auditdb_patches(self, xa_sqlObj,xa_db_host, audit_db_host, db_name, audit_db_name, db_user, audit_db_user, db_password, audit_db_password, PATCHES_PATH, TABLE_NAME): + #first get all patches and then apply each patch + if not os.path.exists(PATCHES_PATH): + log("[I] No patches to apply!","info") + else: + # files: coming from os.listdir() sorted alphabetically, thus not numerically + files = os.listdir(PATCHES_PATH) + if files: + sorted_files = sorted(files, key=lambda x: str(x.split('.')[0])) + for filename in sorted_files: + currentPatch = os.path.join(PATCHES_PATH, filename) + self.import_auditdb_patches(xa_sqlObj, xa_db_host, audit_db_host, db_name, audit_db_name, db_user, audit_db_user, db_password, audit_db_password, currentPatch, TABLE_NAME) + else: + log("[I] No patches to apply!","info") + def execute_java_patches(xa_db_host, db_user, db_password, db_name): log("[I] ----------------- Executing java patches ------------", "info") @@ -238,8 +253,49 @@ class MysqlConf(BaseDB): else: log("[E] "+name + " import failed!","error") sys.exit(1) + + def import_auditdb_patches(self, xa_sqlObj,xa_db_host, audit_db_host, db_name, audit_db_name, db_user, audit_db_user, db_password, audit_db_password, file_name, TABLE_NAME): + log("[I] --------- Checking XA_ACCESS_AUDIT table to apply audit db patches --------- ","info") + output = self.check_table(audit_db_name, db_user, db_password, TABLE_NAME) + if output == True: + name = basename(file_name) + if os.path.isfile(file_name): + version = name.split('-')[0] + log("[I] Executing patch on " + audit_db_name + " from file: " + name,"info") + get_cmd1 = xa_sqlObj.get_jisql_cmd(db_user, db_password, db_name) + if os_name == "LINUX": + query = get_cmd1 + " -query \"select version from x_db_version_h where version = '%s' and active = 'Y';\"" %(version) + elif os_name == "WINDOWS": + query = get_cmd1 + " -query \"select version from x_db_version_h where version = '%s' and active = 'Y';\" -c ;" %(version) + output = check_output(query) + if output.strip(version + " |"): + log("[I] Patch "+ name +" is already applied" ,"info") + else: + get_cmd2 = self.get_jisql_cmd(db_user, db_password, audit_db_name) + if os_name == "LINUX": + query = get_cmd2 + " -input %s" %file_name + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd2 + " -input %s -c ;" %file_name + ret = subprocess.call(query) + if ret == 0: + log("[I] "+name + " patch applied","info") + if os_name == "LINUX": + query = get_cmd1 + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by) values ('%s', now(), user(), now(), user()) ;\"" %(version) + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd1 + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by) values ('%s', now(), user(), now(), user()) ;\" -c ;" %(version) + ret = subprocess.call(query) + if ret == 0: + log("[I] Patch version updated", "info") + else: + log("[E] Updating patch version failed", "error") + sys.exit(1) + else: + log("[E] "+name + " import failed!","error") + sys.exit(1) else: - log("[I] Import " +name + " file not found","error") + log("[I] Table XA_ACCESS_AUDIT does not exists in " +audit_db_name,"error") sys.exit(1) def check_table(self, db_name, db_user, db_password, TABLE_NAME): @@ -405,9 +461,6 @@ class OracleConf(BaseDB): else: log("[E] "+name + " import failed!","error") sys.exit(1) - else: - log("[E] Import " +name + " sql file not found","error") - sys.exit(1) def import_db_patches(self, db_name, db_user, db_password, file_name): if os.path.isfile(file_name): @@ -446,9 +499,50 @@ class OracleConf(BaseDB): else: log("[E] "+name + " Import failed!","error") sys.exit(1) - else: - log("[I] Patch file not found","error") - sys.exit(1) + + def import_auditdb_patches(self, xa_sqlObj,xa_db_host, audit_db_host, db_name, audit_db_name, db_user, audit_db_user, db_password, audit_db_password, file_name, TABLE_NAME): + log("[I] --------- Checking XA_ACCESS_AUDIT table to apply audit db patches --------- ","info") + output = self.check_table(db_name, db_user, db_password, TABLE_NAME) + if output == True: + if os.path.isfile(file_name): + name = basename(file_name) + version = name.split('-')[0] + log("[I] Executing patch on " + audit_db_name + " from file: " + name,"info") + get_cmd1 = xa_sqlObj.get_jisql_cmd(db_user, db_password) + if os_name == "LINUX": + query = get_cmd1 + " -c \; -query \"select version from x_db_version_h where version = '%s' and active = 'Y';\"" %(version) + elif os_name == "WINDOWS": + query = get_cmd1 + " -query \"select version from x_db_version_h where version = '%s' and active = 'Y';\" -c ;" %(version) + output = check_output(query) + if output.strip(version +" |"): + log("[I] Patch "+ name +" is already applied" ,"info") + else: + get_cmd2 = self.get_jisql_cmd(db_user, db_password) + if os_name == "LINUX": + query = get_cmd2 + " -input %s -c /" %file_name + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd2 + " -input %s -c /" %file_name + ret = subprocess.call(query) + if ret == 0: + log("[I] "+name + " patch applied","info") + if os_name == "LINUX": + query = get_cmd1 + " -c \; -query \"insert into x_db_version_h (id,version, inst_at, inst_by, updated_at, updated_by) values ( X_DB_VERSION_H_SEQ.nextval,'%s', sysdate, '%s', sysdate, '%s');\"" %(version, db_user, db_user) + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd1 + " -query \"insert into x_db_version_h (id,version, inst_at, inst_by, updated_at, updated_by) values ( X_DB_VERSION_H_SEQ.nextval,'%s', sysdate, '%s', sysdate, '%s');\" -c ;" %(version, db_user, db_user) + ret = subprocess.call(query) + if ret == 0: + log("[I] Patch version updated", "info") + else: + log("[E] Updating patch version failed", "error") + sys.exit(1) + else: + log("[E] "+name + " Import failed!","error") + sys.exit(1) + else: + log("[I] Patch file not found","error") + sys.exit(1) def check_table(self, db_name, db_user, db_password, TABLE_NAME): get_cmd = self.get_jisql_cmd(db_user ,db_password) @@ -605,9 +699,6 @@ class PostgresConf(BaseDB): else: log("[E] "+name + " DB schema import failed!","error") sys.exit(1) - else: - log("[E] DB schema file " + name+ " not found","error") - sys.exit(1) def grant_audit_db_user(self, audit_db_name , db_user, audit_db_user, db_password, audit_db_password): log("[I] Granting permission to " + audit_db_user, "info") @@ -670,8 +761,49 @@ class PostgresConf(BaseDB): else: log("[E] "+name + " import failed!","error") sys.exit(1) + + def import_auditdb_patches(self, xa_sqlObj,xa_db_host, audit_db_host, db_name, audit_db_name, db_user, audit_db_user, db_password, audit_db_password, file_name, TABLE_NAME): + log("[I] --------- Checking XA_ACCESS_AUDIT table to apply audit db patches --------- ","info") + output = self.check_table(audit_db_name, db_user, db_password, TABLE_NAME) + if output == True: + name = basename(file_name) + if os.path.isfile(file_name): + version = name.split('-')[0] + log("[I] Executing patch on " + audit_db_name + " from file: " + name,"info") + get_cmd1 = xa_sqlObj.get_jisql_cmd(db_user, db_password, db_name) + if os_name == "LINUX": + query = get_cmd1 + " -query \"select version from x_db_version_h where version = '%s' and active = 'Y';\"" %(version) + elif os_name == "WINDOWS": + query = get_cmd1 + " -query \"select version from x_db_version_h where version = '%s' and active = 'Y';\" -c ;" %(version) + output = check_output(query) + if output.strip(version + " |"): + log("[I] Patch "+ name +" is already applied" ,"info") + else: + get_cmd2 = self.get_jisql_cmd(db_user, db_password, audit_db_name) + if os_name == "LINUX": + query = get_cmd2 + " -input %s" %file_name + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd2 + " -input %s -c ;" %file_name + ret = subprocess.call(query) + if ret == 0: + log("[I] "+name + " patch applied","info") + if os_name == "LINUX": + query = get_cmd1 + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by) values ('%s', now(), '%s@%s', now(), '%s@%s') ;\"" %(version,db_user,xa_db_host,db_user,xa_db_host) + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd1 + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by) values ('%s', now(), '%s@%s', now(), '%s@%s') ;\" -c ;" %(version,db_user,xa_db_host,db_user,xa_db_host) + ret = subprocess.call(query) + if ret == 0: + log("[I] Patch version updated", "info") + else: + log("[E] Updating patch version failed", "error") + sys.exit(1) + else: + log("[E] "+name + " import failed!","error") + sys.exit(1) else: - log("[E] Import " +name + " file not found","error") + log("[I] Table XA_ACCESS_AUDIT does not exists in " +audit_db_name,"error") sys.exit(1) def check_table(self, db_name, db_user, db_password, TABLE_NAME): @@ -815,9 +947,6 @@ class SqlServerConf(BaseDB): else: log("[E] "+name + " DB Schema import failed!","error") sys.exit(1) - else: - log("[I] DB Schema file " + name+ " not found","error") - sys.exit(1) def check_table(self, db_name, db_user, db_password, TABLE_NAME): get_cmd = self.get_jisql_cmd(db_user, db_password, db_name) @@ -883,10 +1012,50 @@ class SqlServerConf(BaseDB): else: log("[E] "+name + " import failed!","error") sys.exit(1) + + def import_auditdb_patches(self, xa_sqlObj,xa_db_host, audit_db_host, db_name, audit_db_name, db_user, audit_db_user, db_password, audit_db_password, file_name, TABLE_NAME): + log("[I] --------- Checking XA_ACCESS_AUDIT table to apply audit db patches --------- ","info") + output = self.check_table(audit_db_name, db_user, db_password, TABLE_NAME) + if output == True: + name = basename(file_name) + if os.path.isfile(file_name): + version = name.split('-')[0] + log("[I] Executing patch on " + audit_db_name + " from file: " + name,"info") + get_cmd1 = xa_sqlObj.get_jisql_cmd(db_user, db_password, db_name) + if os_name == "LINUX": + query = get_cmd1 + " -query \"select version from x_db_version_h where version = '%s' and active = 'Y';\"" %(version) + elif os_name == "WINDOWS": + query = get_cmd1 + " -query \"select version from x_db_version_h where version = '%s' and active = 'Y';\" -c ;" %(version) + output = check_output(query) + if output.strip(version + " |"): + log("[I] Patch "+ name +" is already applied" ,"info") + else: + get_cmd2 = self.get_jisql_cmd(db_user, db_password, audit_db_name) + if os_name == "LINUX": + query = get_cmd2 + " -input %s" %file_name + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd2 + " -input %s" %file_name + ret = subprocess.call(query) + if ret == 0: + log("[I] "+name + " patch applied","info") + if os_name == "LINUX": + query = get_cmd1 + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by) values ('%s', GETDATE(), '%s@%s', GETDATE(), '%s@%s') ;\" -c \;" %(version,db_user,xa_db_host,db_user,xa_db_host) + ret = subprocess.call(shlex.split(query)) + elif os_name == "WINDOWS": + query = get_cmd1 + " -query \"insert into x_db_version_h (version, inst_at, inst_by, updated_at, updated_by) values ('%s', GETDATE(), '%s@%s', GETDATE(), '%s@%s') ;\" -c ;" %(version,db_user,xa_db_host,db_user,xa_db_host) + ret = subprocess.call(query) + if ret == 0: + log("[I] Patch version updated", "info") + else: + log("[E] Updating patch version failed", "error") + sys.exit(1) + else: + log("[E] "+name + " import failed!","error") + sys.exit(1) else: - log("[E] Import " +name + " file not found","error") + log("[I] Table XA_ACCESS_AUDIT does not exists in " +audit_db_name,"error") sys.exit(1) - def auditdb_operation(self, xa_db_host, audit_db_host, db_name, audit_db_name,db_user, audit_db_user, db_password, audit_db_password, file_name, TABLE_NAME): log("[I] --------- Check admin user connection --------- ","info") self.check_connection(audit_db_name, db_user, db_password) @@ -1000,21 +1169,25 @@ def main(argv): mysql_core_file = globalDict['mysql_core_file'] mysql_audit_file = globalDict['mysql_audit_file'] mysql_patches = os.path.join('db','mysql','patches') + mysql_auditdb_patches = os.path.join('db','mysql','patches','audit') oracle_dbversion_catalog = os.path.join('db','oracle','create_dbversion_catalog.sql') oracle_core_file = globalDict['oracle_core_file'] oracle_audit_file = globalDict['oracle_audit_file'] oracle_patches = os.path.join('db','oracle','patches') + oracle_auditdb_patches = os.path.join('db','oracle','patches','audit') postgres_dbversion_catalog = os.path.join('db','postgres','create_dbversion_catalog.sql') postgres_core_file = globalDict['postgres_core_file'] postgres_audit_file = globalDict['postgres_audit_file'] postgres_patches = os.path.join('db','postgres','patches') + postgres_auditdb_patches = os.path.join('db','postgres','patches','audit') sqlserver_dbversion_catalog = os.path.join('db','sqlserver','create_dbversion_catalog.sql') sqlserver_core_file = globalDict['sqlserver_core_file'] sqlserver_audit_file = globalDict['sqlserver_audit_file'] sqlserver_patches = os.path.join('db','sqlserver','patches') + sqlserver_auditdb_patches = os.path.join('db','sqlserver','patches','audit') db_name = globalDict['db_name'] db_user = globalDict['db_user'] @@ -1034,6 +1207,7 @@ def main(argv): xa_db_version_file = os.path.join(RANGER_ADMIN_HOME , mysql_dbversion_catalog) xa_db_core_file = os.path.join(RANGER_ADMIN_HOME , mysql_core_file) xa_patch_file = os.path.join(RANGER_ADMIN_HOME ,mysql_patches) + audit_patch_file = os.path.join(RANGER_ADMIN_HOME ,mysql_auditdb_patches) elif XA_DB_FLAVOR == "ORACLE": ORACLE_CONNECTOR_JAR=globalDict['SQL_CONNECTOR_JAR'] @@ -1041,6 +1215,7 @@ def main(argv): xa_db_version_file = os.path.join(RANGER_ADMIN_HOME ,oracle_dbversion_catalog) xa_db_core_file = os.path.join(RANGER_ADMIN_HOME ,oracle_core_file) xa_patch_file = os.path.join(RANGER_ADMIN_HOME ,oracle_patches) + audit_patch_file = os.path.join(RANGER_ADMIN_HOME ,oracle_auditdb_patches) elif XA_DB_FLAVOR == "POSTGRES": POSTGRES_CONNECTOR_JAR = globalDict['SQL_CONNECTOR_JAR'] @@ -1048,6 +1223,7 @@ def main(argv): xa_db_version_file = os.path.join(RANGER_ADMIN_HOME , postgres_dbversion_catalog) xa_db_core_file = os.path.join(RANGER_ADMIN_HOME , postgres_core_file) xa_patch_file = os.path.join(RANGER_ADMIN_HOME , postgres_patches) + audit_patch_file = os.path.join(RANGER_ADMIN_HOME ,postgres_auditdb_patches) elif XA_DB_FLAVOR == "SQLSERVER": SQLSERVER_CONNECTOR_JAR = globalDict['SQL_CONNECTOR_JAR'] @@ -1055,6 +1231,7 @@ def main(argv): xa_db_version_file = os.path.join(RANGER_ADMIN_HOME ,sqlserver_dbversion_catalog) xa_db_core_file = os.path.join(RANGER_ADMIN_HOME , sqlserver_core_file) xa_patch_file = os.path.join(RANGER_ADMIN_HOME , sqlserver_patches) + audit_patch_file = os.path.join(RANGER_ADMIN_HOME ,sqlserver_auditdb_patches) else: log("[E] --------- NO SUCH SUPPORTED DB FLAVOUR!! ---------", "error") sys.exit(1) @@ -1099,9 +1276,12 @@ def main(argv): if output == False: log("[I] --------- Creating version history table ---------","info") xa_sqlObj.upgrade_db(db_name, db_user, db_password, xa_db_version_file) - log("[I] --------- Applying patches ---------","info") + log("[I] --------- Applying Ranger DB patches ---------","info") xa_sqlObj.apply_patches(db_name, db_user, db_password, xa_patch_file) + log("[I] --------- Starting Audit Operation ---------","info") audit_sqlObj.auditdb_operation(xa_db_host, audit_db_host, db_name, audit_db_name, db_user, audit_db_user, db_password, audit_db_password, audit_db_file, xa_access_audit) + log("[I] --------- Applying Audit DB patches ---------","info") + audit_sqlObj.apply_auditdb_patches(xa_sqlObj,xa_db_host, audit_db_host, db_name, audit_db_name, db_user, audit_db_user, db_password, audit_db_password, audit_patch_file, xa_access_audit) # ''' if len(argv)>1: for i in range(len(argv)): http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/dd996d25/security-admin/src/main/java/org/apache/ranger/entity/XXAccessAudit.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/entity/XXAccessAudit.java b/security-admin/src/main/java/org/apache/ranger/entity/XXAccessAudit.java index 5696080..6e35fa4 100644 --- a/security-admin/src/main/java/org/apache/ranger/entity/XXAccessAudit.java +++ b/security-admin/src/main/java/org/apache/ranger/entity/XXAccessAudit.java @@ -240,6 +240,16 @@ public class XXAccessAudit extends XXDBBase implements java.io.Serializable { @Column(name="RESOURCE_TYPE" , length=255) protected String resourceType; + @Column(name="SEQ_NUM") + protected long sequenceNumber; + + @Column(name="EVENT_COUNT") + protected long eventCount; + + //event duration in ms + @Column(name="EVENT_DUR_MS") + protected long eventDuration; + /** * Default constructor. This will set all the attributes to default value. */ @@ -559,6 +569,25 @@ public class XXAccessAudit extends XXDBBase implements java.io.Serializable { return this.resourceType; } + + public long getSequenceNumber() { + return sequenceNumber; + } + public void setSequenceNumber(long sequenceNumber) { + this.sequenceNumber = sequenceNumber; + } + public long getEventCount() { + return eventCount; + } + public void setEventCount(long eventCount) { + this.eventCount = eventCount; + } + public long getEventDuration() { + return eventDuration; + } + public void setEventDuration(long eventDuration) { + this.eventDuration = eventDuration; + } /** * This return the bean content in string format * @return formatedStr @@ -585,6 +614,9 @@ public class XXAccessAudit extends XXDBBase implements java.io.Serializable { str += "requestData={" + requestData + "} "; str += "resourcePath={" + resourcePath + "} "; str += "resourceType={" + resourceType + "} "; + str += "sequenceNumber={" + sequenceNumber + "}"; + str += "eventCount={" + eventCount + "}"; + str += "eventDuration={" + eventDuration + "}"; str += "}"; return str; } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/dd996d25/security-admin/src/main/java/org/apache/ranger/service/XAccessAuditService.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/service/XAccessAuditService.java b/security-admin/src/main/java/org/apache/ranger/service/XAccessAuditService.java index a8999a9..2c143b8 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/XAccessAuditService.java +++ b/security-admin/src/main/java/org/apache/ranger/service/XAccessAuditService.java @@ -118,6 +118,9 @@ public class XAccessAuditService extends XAccessAuditServiceBase<XXAccessAudit, mObj.setResourceType(vObj.getResourceType()); mObj.setClientIP(vObj.getClientIP()); mObj.setClientType(vObj.getClientType()); + mObj.setSequenceNumber( vObj.getSequenceNumber()); + mObj.setEventCount( vObj.getEventCount()); + mObj.setEventDuration( vObj.getEventDuration()); return mObj; } @@ -139,6 +142,9 @@ public class XAccessAuditService extends XAccessAuditServiceBase<XXAccessAudit, vObj.setResourceType( mObj.getResourceType()); vObj.setClientIP( mObj.getClientIP()); vObj.setClientType( mObj.getClientType()); + vObj.setSequenceNumber( mObj.getSequenceNumber()); + vObj.setEventCount( mObj.getEventCount()); + vObj.setEventDuration( mObj.getEventDuration()); return vObj; } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/dd996d25/security-admin/src/main/java/org/apache/ranger/service/XAccessAuditServiceBase.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/service/XAccessAuditServiceBase.java b/security-admin/src/main/java/org/apache/ranger/service/XAccessAuditServiceBase.java index 2914ba1..55391d0 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/XAccessAuditServiceBase.java +++ b/security-admin/src/main/java/org/apache/ranger/service/XAccessAuditServiceBase.java @@ -60,6 +60,9 @@ public abstract class XAccessAuditServiceBase<T extends XXAccessAudit, V extends mObj.setRequestData( vObj.getRequestData()); mObj.setResourcePath( vObj.getResourcePath()); mObj.setResourceType( vObj.getResourceType()); + mObj.setSequenceNumber( vObj.getSequenceNumber()); + mObj.setEventCount( vObj.getEventCount()); + mObj.setEventDuration( vObj.getEventDuration()); return mObj; } @@ -84,6 +87,9 @@ public abstract class XAccessAuditServiceBase<T extends XXAccessAudit, V extends vObj.setRequestData( mObj.getRequestData()); vObj.setResourcePath( mObj.getResourcePath()); vObj.setResourceType( mObj.getResourceType()); + vObj.setSequenceNumber( mObj.getSequenceNumber()); + vObj.setEventCount( mObj.getEventCount()); + vObj.setEventDuration( mObj.getEventDuration()); return vObj; } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/dd996d25/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java b/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java index 6b4ef6a..ad19822 100644 --- a/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java +++ b/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java @@ -238,6 +238,18 @@ public class SolrAccessAuditsService { if (value != null) { accessAudit.setEventTime(solrUtil.toDate(value)); } + value = doc.getFieldValue("seq_num"); + if (value != null) { + accessAudit.setSequenceNumber(solrUtil.toLong(value)); + } + value = doc.getFieldValue("event_count"); + if (value != null) { + accessAudit.setEventCount(solrUtil.toLong(value)); + } + value = doc.getFieldValue("event_dur_ms"); + if (value != null) { + accessAudit.setEventDuration(solrUtil.toLong(value)); + } return accessAudit; } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/dd996d25/security-admin/src/main/java/org/apache/ranger/view/VXAccessAudit.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/view/VXAccessAudit.java b/security-admin/src/main/java/org/apache/ranger/view/VXAccessAudit.java index f46bc26..16b6718 100644 --- a/security-admin/src/main/java/org/apache/ranger/view/VXAccessAudit.java +++ b/security-admin/src/main/java/org/apache/ranger/view/VXAccessAudit.java @@ -26,6 +26,7 @@ import java.util.Date; +import javax.persistence.Column; import javax.xml.bind.annotation.XmlRootElement; import org.apache.ranger.common.AppConstants; @@ -120,6 +121,12 @@ public class VXAccessAudit extends VXDataObject implements java.io.Serializable */ protected String resourceType; + protected long sequenceNumber; + + protected long eventCount; + + //event duration in ms + protected long eventDuration; /** * Default constructor. This will set all the attributes to default value. */ @@ -433,6 +440,30 @@ public class VXAccessAudit extends VXDataObject implements java.io.Serializable return this.resourceType; } + public long getSequenceNumber() { + return sequenceNumber; + } + + public void setSequenceNumber(long sequenceNumber) { + this.sequenceNumber = sequenceNumber; + } + + public long getEventCount() { + return eventCount; + } + + public void setEventCount(long eventCount) { + this.eventCount = eventCount; + } + + public long getEventDuration() { + return eventDuration; + } + + public void setEventDuration(long eventDuration) { + this.eventDuration = eventDuration; + } + @Override public int getMyClassType( ) { return AppConstants.CLASS_TYPE_XA_ACCESS_AUDIT; @@ -463,6 +494,9 @@ public class VXAccessAudit extends VXDataObject implements java.io.Serializable str += "requestData={" + requestData + "} "; str += "resourcePath={" + resourcePath + "} "; str += "resourceType={" + resourceType + "} "; + str += "sequenceNumber={" + sequenceNumber + "}"; + str += "eventCount={" + eventCount + "}"; + str += "eventDuration={" + eventDuration + "}"; str += "}"; return str; } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/dd996d25/security-admin/src/main/webapp/scripts/views/reports/AuditLayout.js ---------------------------------------------------------------------- diff --git a/security-admin/src/main/webapp/scripts/views/reports/AuditLayout.js b/security-admin/src/main/webapp/scripts/views/reports/AuditLayout.js index 9a3cbde..a41c4c5 100644 --- a/security-admin/src/main/webapp/scripts/views/reports/AuditLayout.js +++ b/security-admin/src/main/webapp/scripts/views/reports/AuditLayout.js @@ -178,6 +178,8 @@ define(function(require) { <th class="renderable cip"></th>\ <th class="renderable cip"> </th>\ <th class="renderable aip" > </th>\ + <th class="renderable aip" > </th>\ + <th class="renderable aip" > </th>\ </tr>'); }, renderDateFields : function(){ @@ -904,6 +906,22 @@ define(function(require) { drag : false, sortable:false, editable:false + }, + eventCount : { + label : 'Event Count', + cell: "string", + click : false, + drag : false, + sortable:false, + editable:false + }, + eventDuration : { + label : 'Event Duration(ms)', + cell: "string", + click : false, + drag : false, + sortable:false, + editable:false } }; return this.accessAuditList.constructor.getTableCols(cols, this.accessAuditList);
