This is an automated email from the ASF dual-hosted git repository. madhan pushed a commit to branch RANGER-3923 in repository https://gitbox.apache.org/repos/asf/ranger.git
commit e5dd94c07b8d9724de5167b3e295338bcb5fd883 Author: Madhan Neethiraj <mad...@apache.org> AuthorDate: Wed Oct 11 22:17:22 2023 -0700 RANGER-4440: added column x_security_zone.gz_jsonData to store compressed bytes - #3 --- .../ranger/authorization/utils/StringUtil.java | 24 +++++------------ .../optimized/current/ranger_core_db_mysql.sql | 1 + .../optimized/current/ranger_core_db_postgres.sql | 1 + .../apache/ranger/entity/XXSecurityZoneBase.java | 12 ++++++--- .../service/RangerSecurityZoneServiceService.java | 30 +++++++++++++++------- 5 files changed, 38 insertions(+), 30 deletions(-) diff --git a/agents-common/src/main/java/org/apache/ranger/authorization/utils/StringUtil.java b/agents-common/src/main/java/org/apache/ranger/authorization/utils/StringUtil.java index 12a89889a..db04e5539 100644 --- a/agents-common/src/main/java/org/apache/ranger/authorization/utils/StringUtil.java +++ b/agents-common/src/main/java/org/apache/ranger/authorization/utils/StringUtil.java @@ -46,8 +46,6 @@ import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; public class StringUtil { private static final TimeZone gmtTimeZone = TimeZone.getTimeZone("GMT+0"); - private static final String COMPRESS_GZIP_MAGIC = "GZip:"; - public static boolean equals(String str1, String str2) { boolean ret = false; @@ -526,11 +524,7 @@ public class StringUtil { if (StringUtils.isEmpty(input)) { ret = input; } else { - byte[] bytes = input.getBytes(StandardCharsets.ISO_8859_1); - - bytes = gzipCompress(bytes); - - ret = COMPRESS_GZIP_MAGIC + new String(bytes, StandardCharsets.ISO_8859_1); + ret = new String(gzipCompress(input), StandardCharsets.ISO_8859_1); } return ret; @@ -541,30 +535,24 @@ public class StringUtil { if (StringUtils.isEmpty(input)) { ret = input; - } else if (input.startsWith(COMPRESS_GZIP_MAGIC)) { - byte[] bytes = input.substring(COMPRESS_GZIP_MAGIC.length()).getBytes(StandardCharsets.ISO_8859_1); - - bytes = gzipDecompress(bytes); - - ret = new String(bytes, StandardCharsets.ISO_8859_1); } else { - ret = input; + ret = gzipDecompress(input.getBytes(StandardCharsets.ISO_8859_1)); } return ret; } - public static byte[] gzipCompress(byte[] input) throws IOException { + public static byte[] gzipCompress(String input) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); GZIPOutputStream gos = new GZIPOutputStream(out); - gos.write(input); + gos.write(input.getBytes(StandardCharsets.ISO_8859_1)); gos.close(); return out.toByteArray(); } - public static byte[] gzipDecompress(byte[] input) throws IOException { + public static String gzipDecompress(byte[] input) throws IOException { ByteArrayInputStream in = new ByteArrayInputStream(input); GZIPInputStream gis = new GZIPInputStream(in); ByteArrayOutputStream out = new ByteArrayOutputStream(); @@ -582,6 +570,6 @@ public class StringUtil { gis.close(); - return out.toByteArray(); + return new String(out.toByteArray(), StandardCharsets.ISO_8859_1); } } diff --git a/security-admin/db/mysql/optimized/current/ranger_core_db_mysql.sql b/security-admin/db/mysql/optimized/current/ranger_core_db_mysql.sql old mode 100755 new mode 100644 index dbeeaf423..3cbca48ef --- a/security-admin/db/mysql/optimized/current/ranger_core_db_mysql.sql +++ b/security-admin/db/mysql/optimized/current/ranger_core_db_mysql.sql @@ -588,6 +588,7 @@ CREATE TABLE IF NOT EXISTS `x_security_zone`( `version` bigint(20) NULL DEFAULT NULL, `name` varchar(255) NOT NULL, `jsonData` MEDIUMTEXT NULL DEFAULT NULL, +`gz_jsonData` LONGBLOB NULL DEFAULT NULL, `description` varchar(1024) DEFAULT NULL, PRIMARY KEY (`id`), UNIQUE KEY `x_security_zone_UK_name`(`name`(190)), diff --git a/security-admin/db/postgres/optimized/current/ranger_core_db_postgres.sql b/security-admin/db/postgres/optimized/current/ranger_core_db_postgres.sql old mode 100755 new mode 100644 index 065bae0df..4cdf7e27d --- a/security-admin/db/postgres/optimized/current/ranger_core_db_postgres.sql +++ b/security-admin/db/postgres/optimized/current/ranger_core_db_postgres.sql @@ -595,6 +595,7 @@ upd_by_id BIGINT DEFAULT NULL NULL, version BIGINT DEFAULT NULL NULL, name varchar(255) NOT NULL, jsonData text DEFAULT NULL NULL, +gz_jsonData BYTEA NULL DEFAULT NULL, description VARCHAR(1024) DEFAULT NULL NULL, primary key (id), CONSTRAINT x_security_zone_UK_name UNIQUE (name), diff --git a/security-admin/src/main/java/org/apache/ranger/entity/XXSecurityZoneBase.java b/security-admin/src/main/java/org/apache/ranger/entity/XXSecurityZoneBase.java index ecdaf2485..e46b2c8a8 100644 --- a/security-admin/src/main/java/org/apache/ranger/entity/XXSecurityZoneBase.java +++ b/security-admin/src/main/java/org/apache/ranger/entity/XXSecurityZoneBase.java @@ -36,12 +36,16 @@ public abstract class XXSecurityZoneBase extends XXDBBase { @Column(name = "jsonData") protected String jsonData; + @Column(name = "gz_jsonData") + protected byte[] gzJsonData; + @Column(name = "description") protected String description; public Long getVersion() { return version; } public String getName() { return name; } public String getJsonData() { return jsonData; } + public byte[] getGzJsonData() { return gzJsonData; } public String getDescription() { return description; } public void setName(String name) { @@ -50,6 +54,7 @@ public abstract class XXSecurityZoneBase extends XXDBBase { public void setJsonData(String jsonData) { this.jsonData = jsonData; } + public void setGzJsonData(byte[] gzJsonData) { this.gzJsonData = gzJsonData; } public void setDescription(String description) { this.description = description; } @@ -70,19 +75,20 @@ public abstract class XXSecurityZoneBase extends XXDBBase { return Objects.equals(version, other.version) && Objects.equals(name, other.name) && - Objects.equals(jsonData, other.jsonData); + Objects.equals(jsonData, other.jsonData) && + Objects.equals(gzJsonData, other.gzJsonData); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), version, name, jsonData); + return Objects.hash(super.hashCode(), version, name, jsonData, gzJsonData); } @Override public String toString() { String str = "XXSecurityZoneBase={"; str += super.toString(); - str += " [version=" + version + ", name=" + name + ", jsonData=" + jsonData + "]"; + str += " [version=" + version + ", name=" + name + ", jsonData=" + jsonData + ", gzJsonData=" + gzJsonData + "]"; str += "}"; return str; } diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerSecurityZoneServiceService.java b/security-admin/src/main/java/org/apache/ranger/service/RangerSecurityZoneServiceService.java index 8acdd9813..d500acb8e 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/RangerSecurityZoneServiceService.java +++ b/security-admin/src/main/java/org/apache/ranger/service/RangerSecurityZoneServiceService.java @@ -98,7 +98,7 @@ public class RangerSecurityZoneServiceService extends RangerSecurityZoneServiceB RangerAdminConfig config = RangerAdminConfig.getInstance(); - compressJsonData = config.getBoolean("ranger.admin.store.security.zone.compress.json_data", false); + compressJsonData = config.getBoolean("ranger.admin.store.security.zone.compress.json_data", compressJsonData); logger.info("ranger.admin.store.security.zone.compress.json_data={}", compressJsonData); @@ -128,28 +128,40 @@ public class RangerSecurityZoneServiceService extends RangerSecurityZoneServiceB if (StringUtils.isNotEmpty(json) && compressJsonData) { try { - json = StringUtil.compressString(json); + ret.setJsonData(null); + ret.setGzJsonData(StringUtil.gzipCompress(json)); } catch (IOException excp) { logger.error("mapViewToEntityBean(): json compression failed (length={}). Will save uncompressed json", json.length(), excp); + + ret.setJsonData(json); + ret.setGzJsonData(null); } + } else { + ret.setJsonData(json); + ret.setGzJsonData(null); } - ret.setJsonData(json); - return ret; } @Override protected RangerSecurityZone mapEntityToViewBean(RangerSecurityZone securityZone, XXSecurityZone xxSecurityZone) { - RangerSecurityZone ret = super.mapEntityToViewBean(securityZone, xxSecurityZone); - String json = xxSecurityZone.getJsonData(); + RangerSecurityZone ret = super.mapEntityToViewBean(securityZone, xxSecurityZone); + byte[] gzJson = xxSecurityZone.getGzJsonData(); + String json; - if (StringUtils.isNotEmpty(json)) { + if (gzJson != null) { try { - json = StringUtil.decompressString(json); + json = StringUtil.gzipDecompress(gzJson); } catch (IOException excp) { - logger.error("mapEntityToViewBean(): json decompression failed (length={}). Will treat as uncompressed json", json.length(), excp); + json = xxSecurityZone.getJsonData(); + + logger.error("mapEntityToViewBean(): decompression of x_security_zone.gz_jsonData failed (length={}). Will use contents of x_security_zone.jsonData (length={})", gzJson.length, (json != null ? json.length() : 0), excp); } + } else { + json = xxSecurityZone.getJsonData(); + } + if (StringUtils.isNotEmpty(json)) { RangerSecurityZone zoneFromJsonData = gsonBuilder.fromJson(json, RangerSecurityZone.class); if (zoneFromJsonData == null) {