This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch RANGER-3923
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/RANGER-3923 by this push:
new 3afa4244e RANGER-4440: option to store compressed Json text in
x_security_zone.jsonData
3afa4244e is described below
commit 3afa4244e248808b56920129ad7e0e499a69b8db
Author: Madhan Neethiraj <[email protected]>
AuthorDate: Thu Sep 28 03:03:31 2023 -0700
RANGER-4440: option to store compressed Json text in
x_security_zone.jsonData
(cherry picked from commit d30897b67be0f7c7ceccefd95c5f1113bae809ce)
---
.../ranger/authorization/utils/StringUtil.java | 75 +++++++++++++++++++-
.../ranger/authorization/utils/TestStringUtil.java | 82 ++++++++++++++++++++++
.../org/apache/ranger/biz/SecurityZoneDBStore.java | 14 +++-
.../service/RangerSecurityZoneServiceService.java | 52 ++++++++++++--
4 files changed, 214 insertions(+), 9 deletions(-)
diff --git
a/agents-common/src/main/java/org/apache/ranger/authorization/utils/StringUtil.java
b/agents-common/src/main/java/org/apache/ranger/authorization/utils/StringUtil.java
index 8202b00b2..12a89889a 100644
---
a/agents-common/src/main/java/org/apache/ranger/authorization/utils/StringUtil.java
+++
b/agents-common/src/main/java/org/apache/ranger/authorization/utils/StringUtil.java
@@ -19,6 +19,10 @@
package org.apache.ranger.authorization.utils;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collection;
@@ -31,6 +35,8 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TimeZone;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
@@ -38,9 +44,11 @@ import org.apache.commons.lang.StringUtils;
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
public class StringUtil {
-
private static final TimeZone gmtTimeZone = TimeZone.getTimeZone("GMT+0");
+ private static final String COMPRESS_GZIP_MAGIC = "GZip:";
+
+
public static boolean equals(String str1, String str2) {
boolean ret = false;
@@ -511,4 +519,69 @@ public class StringUtil {
return ret == null ? str : ret;
}
+
+ public static String compressString(String input) throws IOException {
+ final String ret;
+
+ if (StringUtils.isEmpty(input)) {
+ ret = input;
+ } else {
+ byte[] bytes =
input.getBytes(StandardCharsets.ISO_8859_1);
+
+ bytes = gzipCompress(bytes);
+
+ ret = COMPRESS_GZIP_MAGIC + new String(bytes,
StandardCharsets.ISO_8859_1);
+ }
+
+ return ret;
+ }
+
+ public static String decompressString(String input) throws IOException {
+ final String ret;
+
+ if (StringUtils.isEmpty(input)) {
+ ret = input;
+ } else if (input.startsWith(COMPRESS_GZIP_MAGIC)) {
+ byte[] bytes =
input.substring(COMPRESS_GZIP_MAGIC.length()).getBytes(StandardCharsets.ISO_8859_1);
+
+ bytes = gzipDecompress(bytes);
+
+ ret = new String(bytes, StandardCharsets.ISO_8859_1);
+ } else {
+ ret = input;
+ }
+
+ return ret;
+ }
+
+ public static byte[] gzipCompress(byte[] input) throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ GZIPOutputStream gos = new GZIPOutputStream(out);
+
+ gos.write(input);
+ gos.close();
+
+ return out.toByteArray();
+ }
+
+ public static byte[] gzipDecompress(byte[] input) throws IOException {
+ ByteArrayInputStream in = new ByteArrayInputStream(input);
+ GZIPInputStream gis = new GZIPInputStream(in);
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ byte[] buf = new byte[1024];
+
+ while (true) {
+ int bytesRead = gis.read(buf, 0, buf.length);
+
+ if (bytesRead == -1) {
+ break;
+ }
+
+ out.write(buf, 0, bytesRead);
+ }
+
+ gis.close();
+
+ return out.toByteArray();
+ }
}
diff --git
a/agents-common/src/test/java/org/apache/ranger/authorization/utils/TestStringUtil.java
b/agents-common/src/test/java/org/apache/ranger/authorization/utils/TestStringUtil.java
index 5317dd2d6..2f3caea9d 100644
---
a/agents-common/src/test/java/org/apache/ranger/authorization/utils/TestStringUtil.java
+++
b/agents-common/src/test/java/org/apache/ranger/authorization/utils/TestStringUtil.java
@@ -19,10 +19,15 @@
package org.apache.ranger.authorization.utils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
+import org.apache.ranger.plugin.model.RangerSecurityZone;
+import
org.apache.ranger.plugin.model.RangerSecurityZone.RangerSecurityZoneService;
+import org.apache.ranger.plugin.util.RangerSecurityZoneHelper;
import org.junit.Assert;
import org.junit.Test;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -197,6 +202,24 @@ public class TestStringUtil {
}
}
+ @Test
+ public void testJsonCompression() throws IOException {
+ int[] sizeFactors = new int[] { 1, 10, 50, 100, 250, 300, 400, 500 };
+
+ for (int sizeFactor : sizeFactors) {
+ RangerSecurityZone zone =
generateLargeSecurityZone(sizeFactor);
+ String json = JsonUtils.objectToJson(zone);
+ String compressed = StringUtil.compressString(json);
+ String deCompressed =
StringUtil.decompressString(compressed);
+
+ System.out.println(String.format("%d: resourceCount=%d:
len(json)=%,d, len(compressed)=%,d, savings=(%,d == %.03f%%)", sizeFactor,
getResourceCount(zone), json.length(), compressed.length(), (json.length() -
compressed.length()), ((json.length() - compressed.length()) / (float)
json.length()) * 100));
+
+ Assert.assertTrue(compressed.length() < deCompressed.length());
+
+ Assert.assertEquals(json, deCompressed);
+ }
+ }
+
private boolean containsInstance(Collection<String> coll, String key) {
boolean ret = false;
@@ -216,4 +239,63 @@ public class TestStringUtil {
private String getString(String str) {
return str == null ? str : new String(str);
}
+
+ private RangerSecurityZone generateLargeSecurityZone(int sizeFactor) {
+ RangerSecurityZone zone = new RangerSecurityZone();
+ int svcCount = sizeFactor;
+ int resourceCount = sizeFactor;
+ int resNameLen = (sizeFactor / 10) + 1;
+
+ zone.setName("test-zone");
+ zone.setDescription("this is a test zone");
+ zone.setTagServices(generateStrings("tag-service-", 25, 1));
+ zone.setAdminUsers(generateStrings("admin-", 20, 10));
+ zone.setAdminUserGroups(generateStrings("admin-group-", 20, 5));
+ zone.setAdminRoles(generateStrings("admin-role-", 20, 5));
+ zone.setAuditUsers(generateStrings("audit-", 20, 10));
+ zone.setAuditUserGroups(generateStrings("audit-group-", 20, 5));
+ zone.setAuditRoles(generateStrings("audit-role-", 20, 5));
+
+ for (int i = 0; i < svcCount; i++) {
+ RangerSecurityZoneService svc = new RangerSecurityZoneService();
+
+ for (int j = 0; j < resourceCount; j++) {
+ HashMap<String, List<String>> resource = new HashMap<>();
+
+ resource.put("database", generateStrings("db-", resNameLen,
1));
+ resource.put("table", generateStrings("tbl-", resNameLen, 2));
+ resource.put("column", generateStrings("col-", resNameLen, 3));
+
+ svc.getResources().add(resource);
+ }
+
+ zone.getServices().put("service-" + i, svc);
+ }
+
+ return new RangerSecurityZoneHelper(zone, "testUser").getZone(); //
add resourcesBaseInfo
+ }
+
+ private int getResourceCount(RangerSecurityZone zone) {
+ int ret = 0;
+
+ for (RangerSecurityZone.RangerSecurityZoneService svc :
zone.getServices().values()) {
+ ret += svc.getResources().size();
+ }
+
+ return ret;
+ }
+
+ private List<String> generateStrings(String prefix, int maxLen, int count)
{
+ List<String> ret = new ArrayList<>(count);
+
+ for (int i = 0; i < count; i++) {
+ ret.add(generateResourceName(prefix, maxLen));
+ }
+
+ return ret;
+ }
+
+ private String generateResourceName(String prefix, int maxLen) {
+ return prefix.length() < maxLen ? (prefix +
RandomStringUtils.random(maxLen - prefix.length(), true, true)) : prefix;
+ }
}
diff --git
a/security-admin/src/main/java/org/apache/ranger/biz/SecurityZoneDBStore.java
b/security-admin/src/main/java/org/apache/ranger/biz/SecurityZoneDBStore.java
index 44bca7489..a2c4e30ca 100644
---
a/security-admin/src/main/java/org/apache/ranger/biz/SecurityZoneDBStore.java
+++
b/security-admin/src/main/java/org/apache/ranger/biz/SecurityZoneDBStore.java
@@ -17,6 +17,7 @@
package org.apache.ranger.biz;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -26,6 +27,7 @@ import javax.annotation.PostConstruct;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.ranger.authorization.utils.StringUtil;
import org.apache.ranger.common.MessageEnums;
import org.apache.ranger.common.RESTErrorUtil;
import org.apache.ranger.db.RangerDaoManager;
@@ -115,8 +117,16 @@ public class SecurityZoneDBStore implements
SecurityZoneStore {
throw restErrorUtil.createRESTException("security-zone with id: "
+ securityZone.getId() + " does not exist");
}
- Gson gsonBuilder = new
GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").create();
- RangerSecurityZone oldSecurityZone =
gsonBuilder.fromJson(xxSecurityZone.getJsonData(), RangerSecurityZone.class);
+ Gson gsonBuilder = new
GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").create();
+ String json = xxSecurityZone.getJsonData();
+
+ try {
+ json = StringUtil.decompressString(json);
+ } catch (IOException excp) {
+ LOG.error("updateSecurityZoneById(): json decompression failed.
Will treat as uncompressed json", excp);
+ }
+
+ RangerSecurityZone oldSecurityZone = gsonBuilder.fromJson(json,
RangerSecurityZone.class);
daoMgr.getXXGlobalState().onGlobalStateChange(RANGER_GLOBAL_STATE_NAME);
diff --git
a/security-admin/src/main/java/org/apache/ranger/service/RangerSecurityZoneServiceService.java
b/security-admin/src/main/java/org/apache/ranger/service/RangerSecurityZoneServiceService.java
index 476d1511b..439d9a6de 100644
---
a/security-admin/src/main/java/org/apache/ranger/service/RangerSecurityZoneServiceService.java
+++
b/security-admin/src/main/java/org/apache/ranger/service/RangerSecurityZoneServiceService.java
@@ -18,6 +18,7 @@
package org.apache.ranger.service;
+import java.io.IOException;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Collection;
@@ -29,6 +30,8 @@ import java.util.Set;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.ranger.authorization.hadoop.config.RangerAdminConfig;
+import org.apache.ranger.authorization.utils.StringUtil;
import org.apache.ranger.biz.ServiceDBStore;
import org.apache.ranger.common.AppConstants;
import org.apache.ranger.common.view.VTrxLogAttr;
@@ -49,16 +52,19 @@ import org.springframework.stereotype.Service;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
+import javax.annotation.PostConstruct;
+
@Service
@Scope("singleton")
public class RangerSecurityZoneServiceService extends
RangerSecurityZoneServiceBase<XXSecurityZone, RangerSecurityZone> {
-
@Autowired
RangerEnumUtil xaEnumUtil;
@Autowired
ServiceDBStore serviceDBStore;
+ boolean compressJsonData = false;
+
private static final Logger logger =
LoggerFactory.getLogger(RangerSecurityZoneServiceService.class);
private static final Gson gsonBuilder = new
GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").create();
@@ -82,6 +88,23 @@ public class RangerSecurityZoneServiceService extends
RangerSecurityZoneServiceB
super();
}
+ @PostConstruct
+ public void initService() {
+ if (logger.isDebugEnabled()) {
+ logger.debug("==> RangerSecurityZoneServiceService.initService()");
+ }
+
+ RangerAdminConfig config = RangerAdminConfig.getInstance();
+
+ compressJsonData =
config.getBoolean("ranger.admin.store.security.zone.compress.json_data", false);
+
+ logger.info("ranger.admin.store.security.zone.compress.json_data={}",
compressJsonData);
+
+ if (logger.isDebugEnabled()) {
+ logger.debug("<== RangerSecurityZoneServiceService.initService()");
+ }
+ }
+
@Override
protected void validateForCreate(RangerSecurityZone vObj) {
}
@@ -99,19 +122,36 @@ public class RangerSecurityZoneServiceService extends
RangerSecurityZoneServiceB
protected XXSecurityZone mapViewToEntityBean(RangerSecurityZone
securityZone, XXSecurityZone xxSecurityZone, int OPERATION_CONTEXT) {
XXSecurityZone ret = super.mapViewToEntityBean(securityZone,
xxSecurityZone, OPERATION_CONTEXT);
- ret.setJsonData(gsonBuilder.toJson(securityZone));
+ String json = gsonBuilder.toJson(securityZone);
+
+ if (StringUtils.isNotEmpty(json) && compressJsonData) {
+ try {
+ json = StringUtil.compressString(json);
+ } catch (IOException excp) {
+ logger.error("mapViewToEntityBean(): json compression failed
(length={}). Will save uncompressed json", json.length(), excp);
+ }
+ }
+
+ ret.setJsonData(json);
return ret;
}
@Override
protected RangerSecurityZone mapEntityToViewBean(RangerSecurityZone
securityZone, XXSecurityZone xxSecurityZone) {
- RangerSecurityZone ret = super.mapEntityToViewBean(securityZone,
xxSecurityZone);
+ RangerSecurityZone ret = super.mapEntityToViewBean(securityZone,
xxSecurityZone);
+ String json = xxSecurityZone.getJsonData();
+
+ if (StringUtils.isNotEmpty(json)) {
+ try {
+ json = StringUtil.decompressString(json);
+ } catch (IOException excp) {
+ logger.error("mapEntityToViewBean(): json decompression failed
(length={}). Will treat as uncompressed json", json.length(), excp);
+ }
- if (StringUtils.isNotEmpty(xxSecurityZone.getJsonData())) {
- RangerSecurityZone zoneFromJsonData =
gsonBuilder.fromJson(xxSecurityZone.getJsonData(), RangerSecurityZone.class);
+ RangerSecurityZone zoneFromJsonData = gsonBuilder.fromJson(json,
RangerSecurityZone.class);
if (zoneFromJsonData == null) {
- logger.info("Cannot read jsonData into RangerSecurityZone
object in [" + xxSecurityZone.getJsonData() + "]!!");
+ logger.info("Cannot read jsonData into RangerSecurityZone
object in [" + json + "]!!");
} else {
ret.setName(zoneFromJsonData.getName());
ret.setServices(zoneFromJsonData.getServices());
