This is an automated email from the ASF dual-hosted git repository. madhan pushed a commit to branch RANGER-3923 in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/RANGER-3923 by this push: new 3afa4244e RANGER-4440: option to store compressed Json text in x_security_zone.jsonData 3afa4244e is described below commit 3afa4244e248808b56920129ad7e0e499a69b8db Author: Madhan Neethiraj <mad...@apache.org> AuthorDate: Thu Sep 28 03:03:31 2023 -0700 RANGER-4440: option to store compressed Json text in x_security_zone.jsonData (cherry picked from commit d30897b67be0f7c7ceccefd95c5f1113bae809ce) --- .../ranger/authorization/utils/StringUtil.java | 75 +++++++++++++++++++- .../ranger/authorization/utils/TestStringUtil.java | 82 ++++++++++++++++++++++ .../org/apache/ranger/biz/SecurityZoneDBStore.java | 14 +++- .../service/RangerSecurityZoneServiceService.java | 52 ++++++++++++-- 4 files changed, 214 insertions(+), 9 deletions(-) diff --git a/agents-common/src/main/java/org/apache/ranger/authorization/utils/StringUtil.java b/agents-common/src/main/java/org/apache/ranger/authorization/utils/StringUtil.java index 8202b00b2..12a89889a 100644 --- a/agents-common/src/main/java/org/apache/ranger/authorization/utils/StringUtil.java +++ b/agents-common/src/main/java/org/apache/ranger/authorization/utils/StringUtil.java @@ -19,6 +19,10 @@ package org.apache.ranger.authorization.utils; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Calendar; import java.util.Collection; @@ -31,6 +35,8 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.TimeZone; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; @@ -38,9 +44,11 @@ import org.apache.commons.lang.StringUtils; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; public class StringUtil { - private static final TimeZone gmtTimeZone = TimeZone.getTimeZone("GMT+0"); + private static final String COMPRESS_GZIP_MAGIC = "GZip:"; + + public static boolean equals(String str1, String str2) { boolean ret = false; @@ -511,4 +519,69 @@ public class StringUtil { return ret == null ? str : ret; } + + public static String compressString(String input) throws IOException { + final String ret; + + if (StringUtils.isEmpty(input)) { + ret = input; + } else { + byte[] bytes = input.getBytes(StandardCharsets.ISO_8859_1); + + bytes = gzipCompress(bytes); + + ret = COMPRESS_GZIP_MAGIC + new String(bytes, StandardCharsets.ISO_8859_1); + } + + return ret; + } + + public static String decompressString(String input) throws IOException { + final String ret; + + if (StringUtils.isEmpty(input)) { + ret = input; + } else if (input.startsWith(COMPRESS_GZIP_MAGIC)) { + byte[] bytes = input.substring(COMPRESS_GZIP_MAGIC.length()).getBytes(StandardCharsets.ISO_8859_1); + + bytes = gzipDecompress(bytes); + + ret = new String(bytes, StandardCharsets.ISO_8859_1); + } else { + ret = input; + } + + return ret; + } + + public static byte[] gzipCompress(byte[] input) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + GZIPOutputStream gos = new GZIPOutputStream(out); + + gos.write(input); + gos.close(); + + return out.toByteArray(); + } + + public static byte[] gzipDecompress(byte[] input) throws IOException { + ByteArrayInputStream in = new ByteArrayInputStream(input); + GZIPInputStream gis = new GZIPInputStream(in); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buf = new byte[1024]; + + while (true) { + int bytesRead = gis.read(buf, 0, buf.length); + + if (bytesRead == -1) { + break; + } + + out.write(buf, 0, bytesRead); + } + + gis.close(); + + return out.toByteArray(); + } } diff --git a/agents-common/src/test/java/org/apache/ranger/authorization/utils/TestStringUtil.java b/agents-common/src/test/java/org/apache/ranger/authorization/utils/TestStringUtil.java index 5317dd2d6..2f3caea9d 100644 --- a/agents-common/src/test/java/org/apache/ranger/authorization/utils/TestStringUtil.java +++ b/agents-common/src/test/java/org/apache/ranger/authorization/utils/TestStringUtil.java @@ -19,10 +19,15 @@ package org.apache.ranger.authorization.utils; +import org.apache.commons.lang3.RandomStringUtils; import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource; +import org.apache.ranger.plugin.model.RangerSecurityZone; +import org.apache.ranger.plugin.model.RangerSecurityZone.RangerSecurityZoneService; +import org.apache.ranger.plugin.util.RangerSecurityZoneHelper; import org.junit.Assert; import org.junit.Test; +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -197,6 +202,24 @@ public class TestStringUtil { } } + @Test + public void testJsonCompression() throws IOException { + int[] sizeFactors = new int[] { 1, 10, 50, 100, 250, 300, 400, 500 }; + + for (int sizeFactor : sizeFactors) { + RangerSecurityZone zone = generateLargeSecurityZone(sizeFactor); + String json = JsonUtils.objectToJson(zone); + String compressed = StringUtil.compressString(json); + String deCompressed = StringUtil.decompressString(compressed); + + System.out.println(String.format("%d: resourceCount=%d: len(json)=%,d, len(compressed)=%,d, savings=(%,d == %.03f%%)", sizeFactor, getResourceCount(zone), json.length(), compressed.length(), (json.length() - compressed.length()), ((json.length() - compressed.length()) / (float) json.length()) * 100)); + + Assert.assertTrue(compressed.length() < deCompressed.length()); + + Assert.assertEquals(json, deCompressed); + } + } + private boolean containsInstance(Collection<String> coll, String key) { boolean ret = false; @@ -216,4 +239,63 @@ public class TestStringUtil { private String getString(String str) { return str == null ? str : new String(str); } + + private RangerSecurityZone generateLargeSecurityZone(int sizeFactor) { + RangerSecurityZone zone = new RangerSecurityZone(); + int svcCount = sizeFactor; + int resourceCount = sizeFactor; + int resNameLen = (sizeFactor / 10) + 1; + + zone.setName("test-zone"); + zone.setDescription("this is a test zone"); + zone.setTagServices(generateStrings("tag-service-", 25, 1)); + zone.setAdminUsers(generateStrings("admin-", 20, 10)); + zone.setAdminUserGroups(generateStrings("admin-group-", 20, 5)); + zone.setAdminRoles(generateStrings("admin-role-", 20, 5)); + zone.setAuditUsers(generateStrings("audit-", 20, 10)); + zone.setAuditUserGroups(generateStrings("audit-group-", 20, 5)); + zone.setAuditRoles(generateStrings("audit-role-", 20, 5)); + + for (int i = 0; i < svcCount; i++) { + RangerSecurityZoneService svc = new RangerSecurityZoneService(); + + for (int j = 0; j < resourceCount; j++) { + HashMap<String, List<String>> resource = new HashMap<>(); + + resource.put("database", generateStrings("db-", resNameLen, 1)); + resource.put("table", generateStrings("tbl-", resNameLen, 2)); + resource.put("column", generateStrings("col-", resNameLen, 3)); + + svc.getResources().add(resource); + } + + zone.getServices().put("service-" + i, svc); + } + + return new RangerSecurityZoneHelper(zone, "testUser").getZone(); // add resourcesBaseInfo + } + + private int getResourceCount(RangerSecurityZone zone) { + int ret = 0; + + for (RangerSecurityZone.RangerSecurityZoneService svc : zone.getServices().values()) { + ret += svc.getResources().size(); + } + + return ret; + } + + private List<String> generateStrings(String prefix, int maxLen, int count) { + List<String> ret = new ArrayList<>(count); + + for (int i = 0; i < count; i++) { + ret.add(generateResourceName(prefix, maxLen)); + } + + return ret; + } + + private String generateResourceName(String prefix, int maxLen) { + return prefix.length() < maxLen ? (prefix + RandomStringUtils.random(maxLen - prefix.length(), true, true)) : prefix; + } } diff --git a/security-admin/src/main/java/org/apache/ranger/biz/SecurityZoneDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/SecurityZoneDBStore.java index 44bca7489..a2c4e30ca 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/SecurityZoneDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/SecurityZoneDBStore.java @@ -17,6 +17,7 @@ package org.apache.ranger.biz; +import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -26,6 +27,7 @@ import javax.annotation.PostConstruct; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; +import org.apache.ranger.authorization.utils.StringUtil; import org.apache.ranger.common.MessageEnums; import org.apache.ranger.common.RESTErrorUtil; import org.apache.ranger.db.RangerDaoManager; @@ -115,8 +117,16 @@ public class SecurityZoneDBStore implements SecurityZoneStore { throw restErrorUtil.createRESTException("security-zone with id: " + securityZone.getId() + " does not exist"); } - Gson gsonBuilder = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").create(); - RangerSecurityZone oldSecurityZone = gsonBuilder.fromJson(xxSecurityZone.getJsonData(), RangerSecurityZone.class); + Gson gsonBuilder = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").create(); + String json = xxSecurityZone.getJsonData(); + + try { + json = StringUtil.decompressString(json); + } catch (IOException excp) { + LOG.error("updateSecurityZoneById(): json decompression failed. Will treat as uncompressed json", excp); + } + + RangerSecurityZone oldSecurityZone = gsonBuilder.fromJson(json, RangerSecurityZone.class); daoMgr.getXXGlobalState().onGlobalStateChange(RANGER_GLOBAL_STATE_NAME); diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerSecurityZoneServiceService.java b/security-admin/src/main/java/org/apache/ranger/service/RangerSecurityZoneServiceService.java index 476d1511b..439d9a6de 100644 --- a/security-admin/src/main/java/org/apache/ranger/service/RangerSecurityZoneServiceService.java +++ b/security-admin/src/main/java/org/apache/ranger/service/RangerSecurityZoneServiceService.java @@ -18,6 +18,7 @@ package org.apache.ranger.service; +import java.io.IOException; import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Collection; @@ -29,6 +30,8 @@ import java.util.Set; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; +import org.apache.ranger.authorization.hadoop.config.RangerAdminConfig; +import org.apache.ranger.authorization.utils.StringUtil; import org.apache.ranger.biz.ServiceDBStore; import org.apache.ranger.common.AppConstants; import org.apache.ranger.common.view.VTrxLogAttr; @@ -49,16 +52,19 @@ import org.springframework.stereotype.Service; import com.google.gson.Gson; import com.google.gson.GsonBuilder; +import javax.annotation.PostConstruct; + @Service @Scope("singleton") public class RangerSecurityZoneServiceService extends RangerSecurityZoneServiceBase<XXSecurityZone, RangerSecurityZone> { - @Autowired RangerEnumUtil xaEnumUtil; @Autowired ServiceDBStore serviceDBStore; + boolean compressJsonData = false; + private static final Logger logger = LoggerFactory.getLogger(RangerSecurityZoneServiceService.class); private static final Gson gsonBuilder = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").create(); @@ -82,6 +88,23 @@ public class RangerSecurityZoneServiceService extends RangerSecurityZoneServiceB super(); } + @PostConstruct + public void initService() { + if (logger.isDebugEnabled()) { + logger.debug("==> RangerSecurityZoneServiceService.initService()"); + } + + RangerAdminConfig config = RangerAdminConfig.getInstance(); + + compressJsonData = config.getBoolean("ranger.admin.store.security.zone.compress.json_data", false); + + logger.info("ranger.admin.store.security.zone.compress.json_data={}", compressJsonData); + + if (logger.isDebugEnabled()) { + logger.debug("<== RangerSecurityZoneServiceService.initService()"); + } + } + @Override protected void validateForCreate(RangerSecurityZone vObj) { } @@ -99,19 +122,36 @@ public class RangerSecurityZoneServiceService extends RangerSecurityZoneServiceB protected XXSecurityZone mapViewToEntityBean(RangerSecurityZone securityZone, XXSecurityZone xxSecurityZone, int OPERATION_CONTEXT) { XXSecurityZone ret = super.mapViewToEntityBean(securityZone, xxSecurityZone, OPERATION_CONTEXT); - ret.setJsonData(gsonBuilder.toJson(securityZone)); + String json = gsonBuilder.toJson(securityZone); + + if (StringUtils.isNotEmpty(json) && compressJsonData) { + try { + json = StringUtil.compressString(json); + } catch (IOException excp) { + logger.error("mapViewToEntityBean(): json compression failed (length={}). Will save uncompressed json", json.length(), excp); + } + } + + ret.setJsonData(json); return ret; } @Override protected RangerSecurityZone mapEntityToViewBean(RangerSecurityZone securityZone, XXSecurityZone xxSecurityZone) { - RangerSecurityZone ret = super.mapEntityToViewBean(securityZone, xxSecurityZone); + RangerSecurityZone ret = super.mapEntityToViewBean(securityZone, xxSecurityZone); + String json = xxSecurityZone.getJsonData(); + + if (StringUtils.isNotEmpty(json)) { + try { + json = StringUtil.decompressString(json); + } catch (IOException excp) { + logger.error("mapEntityToViewBean(): json decompression failed (length={}). Will treat as uncompressed json", json.length(), excp); + } - if (StringUtils.isNotEmpty(xxSecurityZone.getJsonData())) { - RangerSecurityZone zoneFromJsonData = gsonBuilder.fromJson(xxSecurityZone.getJsonData(), RangerSecurityZone.class); + RangerSecurityZone zoneFromJsonData = gsonBuilder.fromJson(json, RangerSecurityZone.class); if (zoneFromJsonData == null) { - logger.info("Cannot read jsonData into RangerSecurityZone object in [" + xxSecurityZone.getJsonData() + "]!!"); + logger.info("Cannot read jsonData into RangerSecurityZone object in [" + json + "]!!"); } else { ret.setName(zoneFromJsonData.getName()); ret.setServices(zoneFromJsonData.getServices());