Repository: incubator-ranger Updated Branches: refs/heads/master f46c5cd91 -> 6dbd2b8f1
RANGER-271 : Audit migration script from DB to Solr Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/6dbd2b8f Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/6dbd2b8f Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/6dbd2b8f Branch: refs/heads/master Commit: 6dbd2b8f14ded9f1cbcfe896b1d1c2b2141ce41d Parents: f46c5cd Author: Pradeep Agrawal <[email protected]> Authored: Sat Mar 5 17:30:27 2016 +0530 Committer: Gautam Borad <[email protected]> Committed: Sat Mar 5 20:09:41 2016 +0530 ---------------------------------------------------------------------- .../java/org/apache/ranger/common/DateUtil.java | 9 +- .../org/apache/ranger/db/XXAccessAuditDao.java | 34 ++- .../patch/cliutil/DbToSolrMigrationUtil.java | 222 +++++++++++++++++++ .../resources/META-INF/jpa_named_queries.xml | 6 + .../src/main/resources/META-INF/persistence.xml | 1 + 5 files changed, 270 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/6dbd2b8f/security-admin/src/main/java/org/apache/ranger/common/DateUtil.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/common/DateUtil.java b/security-admin/src/main/java/org/apache/ranger/common/DateUtil.java index b710020..92150ee 100644 --- a/security-admin/src/main/java/org/apache/ranger/common/DateUtil.java +++ b/security-admin/src/main/java/org/apache/ranger/common/DateUtil.java @@ -105,5 +105,12 @@ public class DateUtil { return null; } } - + public static Date getLocalDateForUTCDate(Date date) { + Calendar local = Calendar.getInstance(); + int offset = local.getTimeZone().getOffset(local.getTimeInMillis()); + GregorianCalendar utc = new GregorianCalendar(); + utc.setTimeInMillis(date.getTime()); + utc.add(Calendar.MILLISECOND, offset); + return utc.getTime(); + } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/6dbd2b8f/security-admin/src/main/java/org/apache/ranger/db/XXAccessAuditDao.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXAccessAuditDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXAccessAuditDao.java index cd1add6..432dfa9 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXAccessAuditDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXAccessAuditDao.java @@ -19,13 +19,45 @@ package org.apache.ranger.db; +import java.util.ArrayList; +import java.util.List; +import javax.persistence.NoResultException; +import org.apache.log4j.Logger; import org.apache.ranger.common.db.BaseDao; import org.apache.ranger.entity.XXAccessAudit; public class XXAccessAuditDao extends BaseDao<XXAccessAudit> { - + private static Logger logger = Logger.getLogger(XXAccessAuditDao.class); public XXAccessAuditDao( RangerDaoManagerBase daoManager ) { super(daoManager, "loggingPU"); } + public Long getMaxIdOfXXAccessAudit(){ + Long maxXXAccessAuditID=Long.valueOf(0L); + try { + maxXXAccessAuditID = (Long) getEntityManager() + .createNamedQuery("XXAccessAudit.getMaxIdOfXXAccessAudit", Long.class) + .getSingleResult(); + } catch (NoResultException e) { + logger.debug(e.getMessage()); + }finally{ + if(maxXXAccessAuditID==null){ + maxXXAccessAuditID=Long.valueOf(0L); + } + } + return maxXXAccessAuditID; + } + public List<XXAccessAudit> getByIdRange(long idFrom,long idTo){ + //idFrom and idTo both exclusive + List<XXAccessAudit> xXAccessAuditList = new ArrayList<XXAccessAudit>(); + try { + xXAccessAuditList= getEntityManager().createNamedQuery("XXAccessAudit.getByIdRange", tClass) + .setParameter("idFrom", idFrom) + .setParameter("idTo", idTo) + .getResultList(); + } catch (NoResultException e) { + logger.debug(e.getMessage()); + } + return xXAccessAuditList; + } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/6dbd2b8f/security-admin/src/main/java/org/apache/ranger/patch/cliutil/DbToSolrMigrationUtil.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/patch/cliutil/DbToSolrMigrationUtil.java b/security-admin/src/main/java/org/apache/ranger/patch/cliutil/DbToSolrMigrationUtil.java new file mode 100644 index 0000000..9ce1a12 --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/patch/cliutil/DbToSolrMigrationUtil.java @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.patch.cliutil; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import org.apache.log4j.Logger; +import org.apache.ranger.db.RangerDaoManager; +import org.apache.ranger.entity.XXAccessAudit; +import org.apache.ranger.patch.BaseLoader; +import org.apache.ranger.solr.SolrAccessAuditsService; +import org.apache.ranger.authorization.utils.StringUtil; +import org.apache.ranger.common.DateUtil; +import org.apache.ranger.common.PropertiesUtil; +import org.apache.ranger.util.CLIUtil; +import org.apache.solr.client.solrj.impl.HttpSolrServer; +import org.apache.solr.client.solrj.response.UpdateResponse; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.SolrInputField; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class DbToSolrMigrationUtil extends BaseLoader { + private static Logger logger = Logger.getLogger(DbToSolrMigrationUtil.class); + private HttpSolrServer solrServer=null; + private final static String CHECK_FILE_NAME = "migration_check_file.txt"; + private final static Charset ENCODING = StandardCharsets.UTF_8; + @Autowired + RangerDaoManager daoManager; + @Autowired + SolrAccessAuditsService solrAccessAuditsService; + + public static void main(String[] args) { + logger.info("main()"); + try { + DbToSolrMigrationUtil loader = (DbToSolrMigrationUtil) CLIUtil + .getBean(DbToSolrMigrationUtil.class); + + loader.init(); + while (loader.isMoreToProcess()) { + loader.load(); + } + logger.info("Load complete. Exiting!!!"); + System.exit(0); + } catch (Exception e) { + logger.error("Error loading", e); + System.exit(1); + } + } + + @Override + public void init() throws Exception { + String solrURL=PropertiesUtil.getProperty("ranger.audit.solr.urls"); + logger.info("solrURL:"+solrURL); + solrServer = new HttpSolrServer(solrURL); + } + + @Override + public void execLoad() { + logger.info("==> DbToSolrMigrationUtil.execLoad() Start."); + migrateAuditDbLogsToSolr(); + logger.info("<== DbToSolrMigrationUtil.execLoad() End."); + } + + public void migrateAuditDbLogsToSolr() { + long maxXXAccessAuditID = daoManager.getXXAccessAudit().getMaxIdOfXXAccessAudit(); + if(maxXXAccessAuditID==0){ + logger.info("Access Audit log does not exist."); + return; + } + long maxMigratedID=0; + try { + maxMigratedID = readMigrationStatusFile(CHECK_FILE_NAME); + } catch (IOException ex) { + logger.error("Failed to read migration status from file " + CHECK_FILE_NAME, ex); + } + logger.info("ID of the last available audit log: "+ maxXXAccessAuditID); + if(maxMigratedID > 0) { + logger.info("ID of the last migrated audit log: "+ maxMigratedID); + } + if(maxMigratedID>=maxXXAccessAuditID){ + logger.info("No more DB Audit logs to migrate. Last migrated audit log ID: " + maxMigratedID); + return; + } + long maxRowsPerBatch=10000; + //To ceil the actual division result i.e noOfBatches=maxXXAccessAuditID/maxRowsPerBatch + long noOfBatches=((maxXXAccessAuditID-maxMigratedID)+maxRowsPerBatch-1)/maxRowsPerBatch; + long rangeStart=maxMigratedID; + long rangeEnd=maxXXAccessAuditID-maxMigratedID<=maxRowsPerBatch ? maxXXAccessAuditID : rangeStart+maxRowsPerBatch; + long startTimeInMS=0; + long timeTaken=0; + List<XXAccessAudit> xXAccessAuditList=null; + for(long index=1;index<=noOfBatches;index++){ + logger.info("Batch "+ index+" of total "+noOfBatches); + startTimeInMS=System.currentTimeMillis(); + //rangeStart and rangeEnd both exclusive, if we add +1 in maxRange + xXAccessAuditList=daoManager.getXXAccessAudit().getByIdRange(rangeStart,rangeEnd+1); + for(XXAccessAudit xXAccessAudit:xXAccessAuditList){ + if(xXAccessAudit!=null){ + try { + send2solr(xXAccessAudit); + } catch (Throwable e) { + logger.error("Error while writing audit log id '"+xXAccessAudit.getId()+"' to Solr.", e); + writeMigrationStatusFile(xXAccessAudit.getId(),CHECK_FILE_NAME); + logger.info("Stopping migration process!"); + return; + } + } + } + timeTaken=(System.currentTimeMillis()-startTimeInMS); + logger.info("Batch #" + index + ": time taken:"+timeTaken+" ms"); + if(rangeEnd<maxXXAccessAuditID){ + writeMigrationStatusFile(rangeEnd,CHECK_FILE_NAME); + }else{ + writeMigrationStatusFile(maxXXAccessAuditID,CHECK_FILE_NAME); + } + rangeStart=rangeEnd; + rangeEnd=rangeEnd+maxRowsPerBatch; + } + + } + + public void send2solr(XXAccessAudit xXAccessAudit) throws Throwable { + boolean uidIsString = true; + SolrInputDocument document = new SolrInputDocument(); + document.addField("id", xXAccessAudit.getId()); + document.addField("access", xXAccessAudit.getAccessType()); + document.addField("enforcer", xXAccessAudit.getAclEnforcer()); + document.addField("agent", xXAccessAudit.getAgentId()); + document.addField("repo", xXAccessAudit.getRepoName()); + document.addField("sess", xXAccessAudit.getSessionId()); + document.addField("reqUser", xXAccessAudit.getRequestUser()); + document.addField("reqData", xXAccessAudit.getRequestData()); + document.addField("resource", xXAccessAudit.getResourcePath()); + document.addField("cliIP", xXAccessAudit.getClientIP()); + document.addField("logType", "RangerAudit"); + document.addField("result", xXAccessAudit.getAccessResult()); + document.addField("policy", xXAccessAudit.getPolicyId()); + document.addField("repoType", xXAccessAudit.getRepoType()); + document.addField("resType", xXAccessAudit.getResourceType()); + document.addField("reason", xXAccessAudit.getResultReason()); + document.addField("action", xXAccessAudit.getAction()); + document.addField("evtTime", DateUtil.getLocalDateForUTCDate(xXAccessAudit.getEventTime())); + document.addField("seq_num", xXAccessAudit.getSequenceNumber()); + document.addField("event_count", xXAccessAudit.getEventCount()); + document.addField("event_dur_ms", xXAccessAudit.getEventDuration()); + document.addField("tags", xXAccessAudit.getTags()); + //If ID is not set, then we should add it. + SolrInputField idField = document.getField("id"); + if( idField == null) { + Object uid = null; + if(uidIsString) { + uid = UUID.randomUUID().toString(); + } + document.setField("id", uid); + } + + UpdateResponse response = solrServer.add(document); + if (response.getStatus() != 0) { + logger.info("Response=" + response.toString() + ", status= " + + response.getStatus() + ", event=" + xXAccessAudit.toString()); + throw new Exception("Failed to send audit event ID=" + xXAccessAudit.getId()); + } + } + + private Long readMigrationStatusFile(String aFileName) throws IOException { + Long migratedDbID=0L; + Path path = Paths.get(aFileName); + if (Files.exists(path) && Files.isRegularFile(path)) { + List<String> fileContents=Files.readAllLines(path, ENCODING); + if(fileContents!=null && fileContents.size()>=1){ + String line=fileContents.get(fileContents.size()-1).trim(); + if(!StringUtil.isEmpty(line)){ + try{ + migratedDbID=Long.parseLong(line); + }catch(Exception ex){ + } + } + } + } + return migratedDbID; + } + + private void writeMigrationStatusFile(Long DbID, String aFileName) { + try{ + Path path = Paths.get(aFileName); + List<String> fileContents=new ArrayList<String>(); + fileContents.add(String.valueOf(DbID)); + Files.write(path, fileContents, ENCODING); + }catch(IOException ex){ + logger.error("Failed to update migration status to file " + CHECK_FILE_NAME, ex); + }catch(Exception ex){ + logger.error("Error while updating migration status to file " + CHECK_FILE_NAME, ex); + } + } + @Override + public void printStats() { + } +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/6dbd2b8f/security-admin/src/main/resources/META-INF/jpa_named_queries.xml ---------------------------------------------------------------------- diff --git a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml index 3c84b62..be9b69e 100644 --- a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml +++ b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml @@ -923,4 +923,10 @@ <query>select obj from XXPolicy obj, XXPolicyItem polItem,XXPolicyItemGroupPerm polItemGroupPerm where obj.id = polItem.policyId and polItem.id = polItemGroupPerm.policyItemId and polItemGroupPerm.groupId = :groupId</query> </named-query> + <named-query name="XXAccessAudit.getMaxIdOfXXAccessAudit"> + <query>select max(obj.id) from XXAccessAudit obj</query> + </named-query> + <named-query name="XXAccessAudit.getByIdRange"> + <query>select obj from XXAccessAudit obj WHERE obj.id > :idFrom AND obj.id < :idTo</query> + </named-query> </entity-mappings> http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/6dbd2b8f/security-admin/src/main/resources/META-INF/persistence.xml ---------------------------------------------------------------------- diff --git a/security-admin/src/main/resources/META-INF/persistence.xml b/security-admin/src/main/resources/META-INF/persistence.xml index 344e2e9..e34ac0a 100644 --- a/security-admin/src/main/resources/META-INF/persistence.xml +++ b/security-admin/src/main/resources/META-INF/persistence.xml @@ -41,6 +41,7 @@ </properties> </persistence-unit> <persistence-unit name="loggingPU"> + <mapping-file>META-INF/jpa_named_queries.xml</mapping-file> <class>org.apache.ranger.entity.XXAccessAudit</class> <shared-cache-mode>NONE</shared-cache-mode>
