HBase Naming that unify `Hbase` and `HBase` into `HBase` HBase Naming that unify `Hbase` and `HBase` into `HBase`
(https://issues.apache.org/jira/browse/EAGLE-992) Author: asdf2014 <[email protected]> Closes #905 from asdf2014/hbase_naming. Project: http://git-wip-us.apache.org/repos/asf/eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/eagle/commit/cc5d23ba Tree: http://git-wip-us.apache.org/repos/asf/eagle/tree/cc5d23ba Diff: http://git-wip-us.apache.org/repos/asf/eagle/diff/cc5d23ba Branch: refs/heads/master Commit: cc5d23baed2a30bafc31bba713cc1dbffed86971 Parents: 391c672 Author: asdf2014 <[email protected]> Authored: Tue Apr 18 01:05:26 2017 -0700 Committer: Jay <[email protected]> Committed: Tue Apr 18 01:05:26 2017 -0700 ---------------------------------------------------------------------- .../apache/eagle/common/config/EagleConfig.java | 2 +- .../eagle/common/config/EagleConfigFactory.java | 2 +- .../eagle/service/hbase/EmbeddedHBase.java | 141 ++++++++++++++ .../eagle/service/hbase/EmbeddedHbase.java | 141 -------------- .../eagle/service/hbase/EmbeddedHBaseTest.java | 28 +++ .../eagle/service/hbase/EmbeddedHbaseTest.java | 28 --- .../eagle/audit/listener/TestAuditSupport.java | 2 +- .../eagle/service/client/ClientTestBase.java | 6 +- .../log/entity/TestHBaseWritePerformance.java | 100 ++++++++++ .../log/entity/TestHbaseWritePerformance.java | 100 ---------- .../apache/eagle/query/TestHBaseLogReader2.java | 4 +- .../EagleServiceSelfCheckAPIEntity.java | 8 +- .../EagleServiceSelfCheckResource.java | 4 +- .../storage/hbase/HBaseEntitySchemaManager.java | 2 +- .../security/entity/HBaseResourceEntity.java | 105 ++++++++++ .../security/entity/HbaseResourceEntity.java | 105 ---------- .../service/HBaseSensitivityEntity.java | 4 +- .../security/service/InMemMetadataDaoImpl.java | 3 +- .../service/JDBCSecurityMetadataDAO.java | 4 +- .../audit/JDBCSecurityMetadataDAOTest.java | 2 +- .../audit/TestMetaDataAccessConfigRepo.java | 2 +- .../eagle-security-hbase-auditlog/README.md | 2 +- .../hbase/HBaseAuditLogApplication.java | 2 +- .../security/hbase/HBaseAuditLogObject.java | 28 +++ .../security/hbase/HBaseAuditLogParser.java | 148 +++++++++++++++ .../security/hbase/HBaseAuditLogParserBolt.java | 4 +- .../HBaseResourceSensitivityDataJoinBolt.java | 80 ++++++++ .../hbase/HBaseSensitivityDataEnrichLCM.java | 2 +- .../security/hbase/HbaseAuditLogObject.java | 28 --- .../security/hbase/HbaseAuditLogParser.java | 148 --------------- .../HbaseResourceSensitivityDataJoinBolt.java | 80 -------- ....security.hbase.HBaseAuditLogAppProvider.xml | 4 +- .../security/hbase/TestHBaseAuditLogParser.java | 44 +++++ .../security/hbase/TestHbaseAuditLogParser.java | 44 ----- .../hbase/HBaseMetadataBrowseWebResource.java | 189 ++++++++++++++++++ .../hbase/HBaseMetadataBrowseWebResponse.java | 45 +++++ .../hbase/HbaseMetadataBrowseWebResource.java | 190 ------------------- .../hbase/HbaseMetadataBrowseWebResponse.java | 45 ----- .../hbase/dao/HBaseMetadataDAOImpl.java | 95 ++++++++++ .../hbase/dao/HbaseMetadataDAOImpl.java | 95 ---------- .../hbase/resolver/HBaseActionResolver.java | 41 ++++ .../hbase/resolver/HBaseMetadataResolver.java | 130 +++++++++++++ .../hbase/resolver/HBaseRequestResolver.java | 67 +++++++ .../resolver/HBaseSensitivityTypeResolver.java | 58 ++++++ .../hbase/resolver/HbaseActionResolver.java | 41 ---- .../hbase/resolver/HbaseMetadataResolver.java | 130 ------------- .../hbase/resolver/HbaseRequestResolver.java | 67 ------- .../resolver/HbaseSensitivityTypeResolver.java | 58 ------ .../hbase/TestHBaseMetadataResolver.java | 41 ++++ .../hbase/TestHbaseMetadataResolver.java | 41 ---- .../extractor/TopologyExtractorFactory.java | 5 +- .../extractor/hbase/HBaseTopologyCrawler.java | 58 ++++++ .../hbase/HBaseTopologyEntityParser.java | 176 +++++++++++++++++ .../extractor/hbase/HbaseTopologyCrawler.java | 61 ------ .../hbase/HbaseTopologyEntityParser.java | 176 ----------------- ....eagle.topology.TopologyCheckAppProvider.xml | 10 +- .../topology/TestHBaseTopologyCrawler.java | 40 ++++ .../topology/TestHbaseTopologyCrawler.java | 40 ---- .../eagle/topology/TopologyConstants.java | 2 +- 59 files changed, 1651 insertions(+), 1657 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfig.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfig.java b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfig.java index 14e2451..008a7e6 100755 --- a/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfig.java +++ b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfig.java @@ -29,7 +29,7 @@ public interface EagleConfig { HTableInterface getHTable(String tableName); - Configuration getHbaseConf(); + Configuration getHBaseConf(); String getStorageType(); http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigFactory.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigFactory.java b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigFactory.java index e95e65c..6ad655a 100755 --- a/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigFactory.java +++ b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigFactory.java @@ -153,7 +153,7 @@ public class EagleConfigFactory implements EagleConfig { } @Override - public Configuration getHbaseConf() { + public Configuration getHBaseConf() { return hbaseConf; } http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-core/eagle-embed/eagle-embed-hbase/src/main/java/org/apache/eagle/service/hbase/EmbeddedHBase.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-embed/eagle-embed-hbase/src/main/java/org/apache/eagle/service/hbase/EmbeddedHBase.java b/eagle-core/eagle-embed/eagle-embed-hbase/src/main/java/org/apache/eagle/service/hbase/EmbeddedHBase.java new file mode 100644 index 0000000..e2942f6 --- /dev/null +++ b/eagle-core/eagle-embed/eagle-embed-hbase/src/main/java/org/apache/eagle/service/hbase/EmbeddedHBase.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.eagle.service.hbase; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; + +@Deprecated +public class EmbeddedHBase { + private HBaseTestingUtility util; + private MiniHBaseCluster hbaseCluster; + private static EmbeddedHBase hbase; + private int port; + private String znode; + private static int DEFAULT_PORT = 2181; + private static String DEFAULT_ZNODE = "/hbase-unsecure"; + private static final Logger LOG = LoggerFactory.getLogger(EmbeddedHBase.class); + + private EmbeddedHBase(int port, String znode) { + this.port = port; + this.znode = znode; + } + + private EmbeddedHBase(int port) { + this(port, DEFAULT_ZNODE); + } + + public static EmbeddedHBase getInstance(Configuration conf) { + if (hbase == null) { + synchronized (EmbeddedHBase.class) { + if (hbase == null) { + hbase = new EmbeddedHBase(); + hbase.start(conf); + } + } + } + return hbase; + } + + public static EmbeddedHBase getInstance() { + return getInstance(null); + } + + public EmbeddedHBase() { + this(DEFAULT_PORT, DEFAULT_ZNODE); + } + + public void start() { + start(null); + } + + public void start(Configuration confMap) { + try { + util = new HBaseTestingUtility(); + Configuration conf = util.getConfiguration(); + if (confMap != null) { + conf.addResource(confMap); + } + conf.setInt("test.hbase.zookeeper.property.clientPort", port); + conf.set("zookeeper.znode.parent", znode); + conf.setInt("hbase.zookeeper.property.maxClientCnxns", 200); + + conf.setInt("hbase.master.info.port", -1);//avoid port clobbering + // start mini hbase cluster + hbaseCluster = util.startMiniCluster(); + Configuration config = hbaseCluster.getConf(); + + config.set("zookeeper.session.timeout", "120000"); + config.set("hbase.zookeeper.property.tickTime", "6000"); + config.set(HConstants.HBASE_CLIENT_PAUSE, "3000"); + config.set(HConstants.HBASE_CLIENT_RETRIES_NUMBER, "1"); + config.set(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, "60000"); + + Runtime.getRuntime().addShutdownHook(new Thread() { + @Override + public void run() { + shutdown(); + } + }); + } catch (Throwable t) { + LOG.error("Got an exception: ", t); + } + } + + public void shutdown() { + try { + util.shutdownMiniCluster(); + } catch (Throwable t) { + LOG.info("Got an exception, " + t, t.getCause()); + try { + util.shutdownMiniCluster(); + } catch (Throwable t1) { + // ignored + } + } + } + + public void createTable(String tableName, String cf) { + try { + util.createTable(tableName, cf); + } catch (IOException ex) { + LOG.warn("Create table failed, probably table already existed, table name: " + tableName); + } + } + + public void deleteTable(String tableName) { + try { + util.deleteTable(tableName); + } catch (Exception ex) { + LOG.warn("Delete table failed, probably table not existed, table name: " + tableName); + } + } + + public static void main(String[] args) { + EmbeddedHBase hbase = new EmbeddedHBase(12181); + hbase.start(); + for (String table : new Tables().getTables()) { + hbase.createTable(table, "f"); + } + } +} http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-core/eagle-embed/eagle-embed-hbase/src/main/java/org/apache/eagle/service/hbase/EmbeddedHbase.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-embed/eagle-embed-hbase/src/main/java/org/apache/eagle/service/hbase/EmbeddedHbase.java b/eagle-core/eagle-embed/eagle-embed-hbase/src/main/java/org/apache/eagle/service/hbase/EmbeddedHbase.java deleted file mode 100644 index 0aeac2c..0000000 --- a/eagle-core/eagle-embed/eagle-embed-hbase/src/main/java/org/apache/eagle/service/hbase/EmbeddedHbase.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.eagle.service.hbase; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.MiniHBaseCluster; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; - -@Deprecated -public class EmbeddedHbase { - private HBaseTestingUtility util; - private MiniHBaseCluster hbaseCluster; - private static EmbeddedHbase hbase; - private int port; - private String znode; - private static int DEFAULT_PORT = 2181; - private static String DEFAULT_ZNODE = "/hbase-unsecure"; - private static final Logger LOG = LoggerFactory.getLogger(EmbeddedHbase.class); - - private EmbeddedHbase(int port, String znode) { - this.port = port; - this.znode = znode; - } - - private EmbeddedHbase(int port) { - this(port, DEFAULT_ZNODE); - } - - public static EmbeddedHbase getInstance(Configuration conf) { - if (hbase == null) { - synchronized (EmbeddedHbase.class) { - if (hbase == null) { - hbase = new EmbeddedHbase(); - hbase.start(conf); - } - } - } - return hbase; - } - - public static EmbeddedHbase getInstance() { - return getInstance(null); - } - - public EmbeddedHbase() { - this(DEFAULT_PORT, DEFAULT_ZNODE); - } - - public void start() { - start(null); - } - - public void start(Configuration confMap) { - try { - util = new HBaseTestingUtility(); - Configuration conf = util.getConfiguration(); - if (confMap != null) { - conf.addResource(confMap); - } - conf.setInt("test.hbase.zookeeper.property.clientPort", port); - conf.set("zookeeper.znode.parent", znode); - conf.setInt("hbase.zookeeper.property.maxClientCnxns", 200); - - conf.setInt("hbase.master.info.port", -1);//avoid port clobbering - // start mini hbase cluster - hbaseCluster = util.startMiniCluster(); - Configuration config = hbaseCluster.getConf(); - - config.set("zookeeper.session.timeout", "120000"); - config.set("hbase.zookeeper.property.tickTime", "6000"); - config.set(HConstants.HBASE_CLIENT_PAUSE, "3000"); - config.set(HConstants.HBASE_CLIENT_RETRIES_NUMBER, "1"); - config.set(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, "60000"); - - Runtime.getRuntime().addShutdownHook(new Thread() { - @Override - public void run() { - shutdown(); - } - }); - } catch (Throwable t) { - LOG.error("Got an exception: ", t); - } - } - - public void shutdown() { - try { - util.shutdownMiniCluster(); - } catch (Throwable t) { - LOG.info("Got an exception, " + t, t.getCause()); - try { - util.shutdownMiniCluster(); - } catch (Throwable t1) { - // ignored - } - } - } - - public void createTable(String tableName, String cf) { - try { - util.createTable(tableName, cf); - } catch (IOException ex) { - LOG.warn("Create table failed, probably table already existed, table name: " + tableName); - } - } - - public void deleteTable(String tableName) { - try { - util.deleteTable(tableName); - } catch (Exception ex) { - LOG.warn("Delete table failed, probably table not existed, table name: " + tableName); - } - } - - public static void main(String[] args) { - EmbeddedHbase hbase = new EmbeddedHbase(12181); - hbase.start(); - for (String table : new Tables().getTables()) { - hbase.createTable(table, "f"); - } - } -} http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-core/eagle-embed/eagle-embed-hbase/src/test/java/org/apache/eagle/service/hbase/EmbeddedHBaseTest.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-embed/eagle-embed-hbase/src/test/java/org/apache/eagle/service/hbase/EmbeddedHBaseTest.java b/eagle-core/eagle-embed/eagle-embed-hbase/src/test/java/org/apache/eagle/service/hbase/EmbeddedHBaseTest.java new file mode 100644 index 0000000..c0f4e4c --- /dev/null +++ b/eagle-core/eagle-embed/eagle-embed-hbase/src/test/java/org/apache/eagle/service/hbase/EmbeddedHBaseTest.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.eagle.service.hbase; + +import org.junit.Test; + +import java.io.IOException; + +public class EmbeddedHBaseTest extends TestHBaseBase { + @Test + public void testHBaseCreateTable() throws IOException { + hbase.createTable("test_hbase_table","f"); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-core/eagle-embed/eagle-embed-hbase/src/test/java/org/apache/eagle/service/hbase/EmbeddedHbaseTest.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-embed/eagle-embed-hbase/src/test/java/org/apache/eagle/service/hbase/EmbeddedHbaseTest.java b/eagle-core/eagle-embed/eagle-embed-hbase/src/test/java/org/apache/eagle/service/hbase/EmbeddedHbaseTest.java deleted file mode 100644 index ee9d32d..0000000 --- a/eagle-core/eagle-embed/eagle-embed-hbase/src/test/java/org/apache/eagle/service/hbase/EmbeddedHbaseTest.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.eagle.service.hbase; - -import org.junit.Test; - -import java.io.IOException; - -public class EmbeddedHbaseTest extends TestHBaseBase { - @Test - public void testHBaseCreateTable() throws IOException { - hbase.createTable("test_hbase_table","f"); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-core/eagle-query/eagle-audit-base/src/test/java/org/apache/eagle/audit/listener/TestAuditSupport.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-query/eagle-audit-base/src/test/java/org/apache/eagle/audit/listener/TestAuditSupport.java b/eagle-core/eagle-query/eagle-audit-base/src/test/java/org/apache/eagle/audit/listener/TestAuditSupport.java index 6e7b5cf..d219516 100644 --- a/eagle-core/eagle-query/eagle-audit-base/src/test/java/org/apache/eagle/audit/listener/TestAuditSupport.java +++ b/eagle-core/eagle-query/eagle-audit-base/src/test/java/org/apache/eagle/audit/listener/TestAuditSupport.java @@ -54,7 +54,7 @@ public class TestAuditSupport { } @Test - public void addAuditProxyHbase() throws Exception { + public void addAuditProxyHBase() throws Exception { AuditSupport auditSupport = new AuditSupport(this); Field mapField = AuditSupport.class.getDeclaredField("map"); mapField.setAccessible(true); http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-core/eagle-query/eagle-client-base/src/test/java/org/apache/eagle/service/client/ClientTestBase.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-query/eagle-client-base/src/test/java/org/apache/eagle/service/client/ClientTestBase.java b/eagle-core/eagle-query/eagle-client-base/src/test/java/org/apache/eagle/service/client/ClientTestBase.java index adfd2e2..77d7c3d 100644 --- a/eagle-core/eagle-query/eagle-client-base/src/test/java/org/apache/eagle/service/client/ClientTestBase.java +++ b/eagle-core/eagle-query/eagle-client-base/src/test/java/org/apache/eagle/service/client/ClientTestBase.java @@ -16,16 +16,16 @@ */ package org.apache.eagle.service.client; -import org.apache.eagle.service.hbase.EmbeddedHbase; +import org.apache.eagle.service.hbase.EmbeddedHBase; public class ClientTestBase { //protected static EmbeddedServer server; - protected static EmbeddedHbase hbase; + protected static EmbeddedHBase hbase; //@BeforeClass public static void startup() throws Exception { - //hbase = EmbeddedHbase.getInstance(); + //hbase = EmbeddedHBase.getInstance(); //String webappDirLocation = "../../../eagle-webservice/target/eagle-service"; //server = EmbeddedServer.getInstance(webappDirLocation); } http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-core/eagle-query/eagle-entity-base/src/test/java/org/apache/eagle/log/entity/TestHBaseWritePerformance.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-query/eagle-entity-base/src/test/java/org/apache/eagle/log/entity/TestHBaseWritePerformance.java b/eagle-core/eagle-query/eagle-entity-base/src/test/java/org/apache/eagle/log/entity/TestHBaseWritePerformance.java new file mode 100644 index 0000000..75b55ba --- /dev/null +++ b/eagle-core/eagle-query/eagle-entity-base/src/test/java/org/apache/eagle/log/entity/TestHBaseWritePerformance.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.eagle.log.entity; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.eagle.common.config.EagleConfigFactory; +import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.hadoop.hbase.client.Put; +import org.junit.Test; + +public class TestHBaseWritePerformance { + + public static void main(String[] args) throws IOException { + + HTableInterface tbl = EagleConfigFactory.load().getHTable("unittest"); + + int putSize = 1000; + List<Put> list = new ArrayList<Put>(putSize); + for (int i = 0; i < putSize; ++i) { + byte[] v = Integer.toString(i).getBytes(); + Put p = new Put(v); + p.add("f".getBytes(), "a".getBytes(), 100, v); + list.add(p); + } + + // Case 1 + System.out.println("Case 1: autoflush = true, individual put"); + tbl.setAutoFlush(true); + long startTime = System.currentTimeMillis(); + for (int i = 0; i < 1; ++i) { + for (Put p : list) { + tbl.put(p); + } + tbl.flushCommits(); + } + long endTime = System.currentTimeMillis(); + System.out.println("Case 1: " + (endTime - startTime) + " ms"); + + + // Case 2 + System.out.println("Case 2: autoflush = true, multi-put"); + tbl.setAutoFlush(true); + startTime = System.currentTimeMillis(); + for (int i = 0; i < 1; ++i) { + tbl.put(list); + tbl.flushCommits(); + } + endTime = System.currentTimeMillis(); + System.out.println("Case 2: " + (endTime - startTime) + " ms"); + + + // Case 3 + System.out.println("Case 3: autoflush = false, multi-put"); + tbl.setAutoFlush(false); + startTime = System.currentTimeMillis(); + for (int i = 0; i < 1; ++i) { + tbl.put(list); + tbl.flushCommits(); + } + endTime = System.currentTimeMillis(); + System.out.println("Case 3: " + (endTime - startTime) + " ms"); + + + // Case 4 + System.out.println("Case 4: autoflush = false, individual put"); + tbl.setAutoFlush(true); + startTime = System.currentTimeMillis(); + for (int i = 0; i < 1; ++i) { + for (Put p : list) { + tbl.put(p); + } + tbl.flushCommits(); + } + endTime = System.currentTimeMillis(); + System.out.println("Case 4: " + (endTime - startTime) + " ms"); + + } + + @Test + public void test() { + + } +} http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-core/eagle-query/eagle-entity-base/src/test/java/org/apache/eagle/log/entity/TestHbaseWritePerformance.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-query/eagle-entity-base/src/test/java/org/apache/eagle/log/entity/TestHbaseWritePerformance.java b/eagle-core/eagle-query/eagle-entity-base/src/test/java/org/apache/eagle/log/entity/TestHbaseWritePerformance.java deleted file mode 100644 index 81f1dce..0000000 --- a/eagle-core/eagle-query/eagle-entity-base/src/test/java/org/apache/eagle/log/entity/TestHbaseWritePerformance.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.eagle.log.entity; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.apache.eagle.common.config.EagleConfigFactory; -import org.apache.hadoop.hbase.client.HTableInterface; -import org.apache.hadoop.hbase.client.Put; -import org.junit.Test; - -public class TestHbaseWritePerformance { - - public static void main(String[] args) throws IOException { - - HTableInterface tbl = EagleConfigFactory.load().getHTable("unittest"); - - int putSize = 1000; - List<Put> list = new ArrayList<Put>(putSize); - for (int i = 0; i < putSize; ++i) { - byte[] v = Integer.toString(i).getBytes(); - Put p = new Put(v); - p.add("f".getBytes(), "a".getBytes(), 100, v); - list.add(p); - } - - // Case 1 - System.out.println("Case 1: autoflush = true, individual put"); - tbl.setAutoFlush(true); - long startTime = System.currentTimeMillis(); - for (int i = 0; i < 1; ++i) { - for (Put p : list) { - tbl.put(p); - } - tbl.flushCommits(); - } - long endTime = System.currentTimeMillis(); - System.out.println("Case 1: " + (endTime - startTime) + " ms"); - - - // Case 2 - System.out.println("Case 2: autoflush = true, multi-put"); - tbl.setAutoFlush(true); - startTime = System.currentTimeMillis(); - for (int i = 0; i < 1; ++i) { - tbl.put(list); - tbl.flushCommits(); - } - endTime = System.currentTimeMillis(); - System.out.println("Case 2: " + (endTime - startTime) + " ms"); - - - // Case 3 - System.out.println("Case 3: autoflush = false, multi-put"); - tbl.setAutoFlush(false); - startTime = System.currentTimeMillis(); - for (int i = 0; i < 1; ++i) { - tbl.put(list); - tbl.flushCommits(); - } - endTime = System.currentTimeMillis(); - System.out.println("Case 3: " + (endTime - startTime) + " ms"); - - - // Case 4 - System.out.println("Case 4: autoflush = false, individual put"); - tbl.setAutoFlush(true); - startTime = System.currentTimeMillis(); - for (int i = 0; i < 1; ++i) { - for (Put p : list) { - tbl.put(p); - } - tbl.flushCommits(); - } - endTime = System.currentTimeMillis(); - System.out.println("Case 4: " + (endTime - startTime) + " ms"); - - } - - @Test - public void test() { - - } -} http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-core/eagle-query/eagle-query-base/src/test/java/org/apache/eagle/query/TestHBaseLogReader2.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-query/eagle-query-base/src/test/java/org/apache/eagle/query/TestHBaseLogReader2.java b/eagle-core/eagle-query/eagle-query-base/src/test/java/org/apache/eagle/query/TestHBaseLogReader2.java index 6513452..6406b13 100755 --- a/eagle-core/eagle-query/eagle-query-base/src/test/java/org/apache/eagle/query/TestHBaseLogReader2.java +++ b/eagle-core/eagle-query/eagle-query-base/src/test/java/org/apache/eagle/query/TestHBaseLogReader2.java @@ -24,7 +24,7 @@ import org.apache.eagle.log.entity.SearchCondition; import org.apache.eagle.log.entity.meta.EntityDefinition; import org.apache.eagle.log.entity.meta.EntityDefinitionManager; import org.apache.eagle.log.entity.test.TestTimeSeriesAPIEntity; -import org.apache.eagle.service.hbase.EmbeddedHbase; +import org.apache.eagle.service.hbase.EmbeddedHBase; import org.apache.hadoop.hbase.util.Bytes; import org.junit.Assert; import org.junit.Test; @@ -35,7 +35,7 @@ import java.util.*; public class TestHBaseLogReader2 { private final static Logger LOG = LoggerFactory.getLogger(TestHBaseLogReader2.class); - private static EmbeddedHbase hbase = EmbeddedHbase.getInstance(); + private static EmbeddedHBase hbase = EmbeddedHBase.getInstance(); @SuppressWarnings("serial") @Test http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-core/eagle-query/eagle-service-base/src/main/java/org/apache/eagle/service/selfcheck/EagleServiceSelfCheckAPIEntity.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-query/eagle-service-base/src/main/java/org/apache/eagle/service/selfcheck/EagleServiceSelfCheckAPIEntity.java b/eagle-core/eagle-query/eagle-service-base/src/main/java/org/apache/eagle/service/selfcheck/EagleServiceSelfCheckAPIEntity.java index 0e52a2e..a66c057 100644 --- a/eagle-core/eagle-query/eagle-service-base/src/main/java/org/apache/eagle/service/selfcheck/EagleServiceSelfCheckAPIEntity.java +++ b/eagle-core/eagle-query/eagle-service-base/src/main/java/org/apache/eagle/service/selfcheck/EagleServiceSelfCheckAPIEntity.java @@ -32,16 +32,16 @@ public class EagleServiceSelfCheckAPIEntity { public void setEnv(String env) { this.env = env; } - public String getHbaseZookeeperQuorum() { + public String getHBaseZookeeperQuorum() { return hbaseZookeeperQuorum; } - public void setHbaseZookeeperQuorum(String hbaseZookeeperQuorum) { + public void setHBaseZookeeperQuorum(String hbaseZookeeperQuorum) { this.hbaseZookeeperQuorum = hbaseZookeeperQuorum; } - public String getHbaseZookeeperClientPort() { + public String getHBaseZookeeperClientPort() { return hbaseZookeeperClientPort; } - public void setHbaseZookeeperClientPort(String hbaseZookeeperClientPort) { + public void setHBaseZookeeperClientPort(String hbaseZookeeperClientPort) { this.hbaseZookeeperClientPort = hbaseZookeeperClientPort; } } http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-core/eagle-query/eagle-service-base/src/main/java/org/apache/eagle/service/selfcheck/EagleServiceSelfCheckResource.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-query/eagle-service-base/src/main/java/org/apache/eagle/service/selfcheck/EagleServiceSelfCheckResource.java b/eagle-core/eagle-query/eagle-service-base/src/main/java/org/apache/eagle/service/selfcheck/EagleServiceSelfCheckResource.java index 88ab613..0eed64c 100644 --- a/eagle-core/eagle-query/eagle-service-base/src/main/java/org/apache/eagle/service/selfcheck/EagleServiceSelfCheckResource.java +++ b/eagle-core/eagle-query/eagle-service-base/src/main/java/org/apache/eagle/service/selfcheck/EagleServiceSelfCheckResource.java @@ -30,8 +30,8 @@ public class EagleServiceSelfCheckResource { @Produces(MediaType.APPLICATION_JSON) public EagleServiceSelfCheckAPIEntity selfCheck(){ EagleServiceSelfCheckAPIEntity entity = new EagleServiceSelfCheckAPIEntity(); - entity.setHbaseZookeeperQuorum(EagleConfigFactory.load().getZKQuorum()); - entity.setHbaseZookeeperClientPort(EagleConfigFactory.load().getZKPort()); + entity.setHBaseZookeeperQuorum(EagleConfigFactory.load().getZKQuorum()); + entity.setHBaseZookeeperClientPort(EagleConfigFactory.load().getZKPort()); entity.setEnv(EagleConfigFactory.load().getEnv()); return entity; } http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-core/eagle-query/eagle-storage-hbase/src/main/java/org/apache/eagle/storage/hbase/HBaseEntitySchemaManager.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-query/eagle-storage-hbase/src/main/java/org/apache/eagle/storage/hbase/HBaseEntitySchemaManager.java b/eagle-core/eagle-query/eagle-storage-hbase/src/main/java/org/apache/eagle/storage/hbase/HBaseEntitySchemaManager.java index 8e4005c..eaa5d3c 100644 --- a/eagle-core/eagle-query/eagle-storage-hbase/src/main/java/org/apache/eagle/storage/hbase/HBaseEntitySchemaManager.java +++ b/eagle-core/eagle-query/eagle-storage-hbase/src/main/java/org/apache/eagle/storage/hbase/HBaseEntitySchemaManager.java @@ -60,7 +60,7 @@ public class HBaseEntitySchemaManager { LOG.debug("Auto create table disabled, skip creating table"); return; } - Configuration conf = EagleConfigFactory.load().getHbaseConf(); + Configuration conf = EagleConfigFactory.load().getHBaseConf(); try { admin = new HBaseAdmin(conf); http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/HBaseResourceEntity.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/HBaseResourceEntity.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/HBaseResourceEntity.java new file mode 100644 index 0000000..a9421e4 --- /dev/null +++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/HBaseResourceEntity.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.security.entity; + +import com.google.common.base.Objects; + +import java.io.Serializable; +import java.util.Set; + +public class HBaseResourceEntity implements Serializable { + private String resource; + private String namespace; + private String table; + private String columnFamily; + private String sensitiveType; + private Set<String> childSensitiveTypes; + + + + public HBaseResourceEntity(String resource, String ns, String table, String cf, String sensitiveType, Set<String> childSensitiveTypes) { + this.resource = resource; + this.namespace = ns; + this.table = table; + this.columnFamily = cf; + this.sensitiveType = sensitiveType; + this.childSensitiveTypes = childSensitiveTypes; + } + + public String getResource() { + return resource; + } + + public void setResource(String resource) { + this.resource = resource; + } + + public String getNamespace() { + return namespace; + } + + public void setNamespace(String namespace) { + this.namespace = namespace; + } + + public String getTable() { + return table; + } + + public void setTable(String table) { + this.table = table; + } + + public String getColumnFamily() { + return columnFamily; + } + + public void setColumnFamily(String columnFamily) { + this.columnFamily = columnFamily; + } + + public String getSensitiveType() { + return sensitiveType; + } + + public void setSensitiveType(String sensitiveType) { + this.sensitiveType = sensitiveType; + } + + public Set<String> getChildSensitiveTypes() { + return childSensitiveTypes; + } + + public void setChildSensitiveTypes(Set<String> childSensitiveTypes) { + this.childSensitiveTypes = childSensitiveTypes; + } + + @Override + public boolean equals(Object obj) { + if (obj == null || getClass() != obj.getClass()) { + return false; + } + final HBaseResourceEntity other = (HBaseResourceEntity) obj; + return Objects.equal(this.resource, other.resource) + && this.namespace.equals(other.namespace) + && this.table.equals(other.table) + && this.columnFamily.equals(other.columnFamily) + && this.sensitiveType.equals(other.sensitiveType) + && this.childSensitiveTypes.containsAll(other.childSensitiveTypes); + } +} http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/HbaseResourceEntity.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/HbaseResourceEntity.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/HbaseResourceEntity.java deleted file mode 100644 index deb8364..0000000 --- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/HbaseResourceEntity.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eagle.security.entity; - -import com.google.common.base.Objects; - -import java.io.Serializable; -import java.util.Set; - -public class HbaseResourceEntity implements Serializable { - private String resource; - private String namespace; - private String table; - private String columnFamily; - private String sensitiveType; - private Set<String> childSensitiveTypes; - - - - public HbaseResourceEntity(String resource, String ns, String table, String cf, String sensitiveType, Set<String> childSensitiveTypes) { - this.resource = resource; - this.namespace = ns; - this.table = table; - this.columnFamily = cf; - this.sensitiveType = sensitiveType; - this.childSensitiveTypes = childSensitiveTypes; - } - - public String getResource() { - return resource; - } - - public void setResource(String resource) { - this.resource = resource; - } - - public String getNamespace() { - return namespace; - } - - public void setNamespace(String namespace) { - this.namespace = namespace; - } - - public String getTable() { - return table; - } - - public void setTable(String table) { - this.table = table; - } - - public String getColumnFamily() { - return columnFamily; - } - - public void setColumnFamily(String columnFamily) { - this.columnFamily = columnFamily; - } - - public String getSensitiveType() { - return sensitiveType; - } - - public void setSensitiveType(String sensitiveType) { - this.sensitiveType = sensitiveType; - } - - public Set<String> getChildSensitiveTypes() { - return childSensitiveTypes; - } - - public void setChildSensitiveTypes(Set<String> childSensitiveTypes) { - this.childSensitiveTypes = childSensitiveTypes; - } - - @Override - public boolean equals(Object obj) { - if (obj == null || getClass() != obj.getClass()) { - return false; - } - final HbaseResourceEntity other = (HbaseResourceEntity) obj; - return Objects.equal(this.resource, other.resource) - && this.namespace.equals(other.namespace) - && this.table.equals(other.table) - && this.columnFamily.equals(other.columnFamily) - && this.sensitiveType.equals(other.sensitiveType) - && this.childSensitiveTypes.containsAll(other.childSensitiveTypes); - } -} http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/HBaseSensitivityEntity.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/HBaseSensitivityEntity.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/HBaseSensitivityEntity.java index d46ef13..eff5092 100644 --- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/HBaseSensitivityEntity.java +++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/HBaseSensitivityEntity.java @@ -32,11 +32,11 @@ public class HBaseSensitivityEntity { this.site = site; } - public String getHbaseResource() { + public String getHBaseResource() { return hbaseResource; } - public void setHbaseResource(String hbaseResource) { + public void setHBaseResource(String hbaseResource) { this.hbaseResource = hbaseResource; } http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/InMemMetadataDaoImpl.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/InMemMetadataDaoImpl.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/InMemMetadataDaoImpl.java index 359cbe6..898f2a9 100644 --- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/InMemMetadataDaoImpl.java +++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/InMemMetadataDaoImpl.java @@ -17,7 +17,6 @@ package org.apache.eagle.security.service; import com.google.inject.Inject; -import com.typesafe.config.Config; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.slf4j.Logger; @@ -54,7 +53,7 @@ public class InMemMetadataDaoImpl implements ISecurityMetadataDAO { @Override public synchronized OpResult addHBaseSensitivity(Collection<HBaseSensitivityEntity> h) { for (HBaseSensitivityEntity e : h) { - Pair p = new ImmutablePair<>(e.getSite(), e.getHbaseResource()); + Pair p = new ImmutablePair<>(e.getSite(), e.getHBaseResource()); hBaseSensitivityEntities.put(p, e); } return new OpResult(); http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/JDBCSecurityMetadataDAO.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/JDBCSecurityMetadataDAO.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/JDBCSecurityMetadataDAO.java index e089bf9..07523d2 100644 --- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/JDBCSecurityMetadataDAO.java +++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/JDBCSecurityMetadataDAO.java @@ -124,7 +124,7 @@ public class JDBCSecurityMetadataDAO implements ISecurityMetadataDAO { try { HBaseSensitivityEntity entity = new HBaseSensitivityEntity(); entity.setSite(rs.getString(1)); - entity.setHbaseResource(rs.getString(2)); + entity.setHBaseResource(rs.getString(2)); entity.setSensitivityType(rs.getString(3)); return entity; }catch(Exception ex){ throw new IllegalStateException(ex);} @@ -138,7 +138,7 @@ public class JDBCSecurityMetadataDAO implements ISecurityMetadataDAO { HBaseSensitivityEntity e = (HBaseSensitivityEntity)entity; try { statement.setString(1, e.getSite()); - statement.setString(2, e.getHbaseResource()); + statement.setString(2, e.getHBaseResource()); statement.setString(3, e.getSensitivityType()); }catch(Exception ex){ throw new IllegalStateException(ex); http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-common/src/test/java/org/apache/eagle/security/crawler/audit/JDBCSecurityMetadataDAOTest.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-common/src/test/java/org/apache/eagle/security/crawler/audit/JDBCSecurityMetadataDAOTest.java b/eagle-security/eagle-security-common/src/test/java/org/apache/eagle/security/crawler/audit/JDBCSecurityMetadataDAOTest.java index 03ae58c..18c89a3 100644 --- a/eagle-security/eagle-security-common/src/test/java/org/apache/eagle/security/crawler/audit/JDBCSecurityMetadataDAOTest.java +++ b/eagle-security/eagle-security-common/src/test/java/org/apache/eagle/security/crawler/audit/JDBCSecurityMetadataDAOTest.java @@ -55,7 +55,7 @@ public class JDBCSecurityMetadataDAOTest { public void testJDBCSecurityMetadataDAO(){ HBaseSensitivityEntity entity = new HBaseSensitivityEntity(); entity.setSite("test_site"); - entity.setHbaseResource("test_hbaseResource"); + entity.setHBaseResource("test_hbaseResource"); metadataDAO.addHBaseSensitivity(Collections.singletonList(entity)); Collection<HBaseSensitivityEntity> entities = metadataDAO.listHBaseSensitivities(); Assert.assertEquals(1,entities.size()); http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-common/src/test/java/org/apache/eagle/security/crawler/audit/TestMetaDataAccessConfigRepo.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-common/src/test/java/org/apache/eagle/security/crawler/audit/TestMetaDataAccessConfigRepo.java b/eagle-security/eagle-security-common/src/test/java/org/apache/eagle/security/crawler/audit/TestMetaDataAccessConfigRepo.java index 325663b..092ee73 100644 --- a/eagle-security/eagle-security-common/src/test/java/org/apache/eagle/security/crawler/audit/TestMetaDataAccessConfigRepo.java +++ b/eagle-security/eagle-security-common/src/test/java/org/apache/eagle/security/crawler/audit/TestMetaDataAccessConfigRepo.java @@ -51,7 +51,7 @@ public class TestMetaDataAccessConfigRepo { Assert.assertTrue(hbaseConfig.getString("hbase.zookeeper.property.clientPort").equals("2181")); } - String appConfigStr = "classification.hbase.zookeeper.property.clientPort=2181\nclassification.hbase.zookeeper.quorum=sandbox.hortonworks.com\n\napp.envContextConfig.env=storm\napp.envContextConfig.mode=cluster\napp.dataSourceConfig.topic=sandbox_hbase_security_log\napp.dataSourceConfig.zkConnection=127.0.0.1:2181\napp.dataSourceConfig.zkConnectionTimeoutMS=15000\napp.dataSourceConfig.brokerZkPath=/brokers\napp.dataSourceConfig.fetchSize=1048586\napp.dataSourceConfig.transactionZKServers=127.0.0.1\napp.dataSourceConfig.transactionZKPort=2181\napp.dataSourceConfig.transactionZKRoot=/consumers\napp.dataSourceConfig.consumerGroupId=eagle.hbasesecurity.consumer\napp.dataSourceConfig.transactionStateUpdateMS=2000\napp.dataSourceConfig.deserializerClass=org.apache.eagle.security.hbase.parse.HbaseAuditLogKafkaDeserializer\napp.eagleProps.site=sandbox\napp.eagleProps.application=hbaseSecurityLog\napp.eagleProps.dataJoinPollIntervalSec=30\napp.eagleProps.mailHost=mailHost.com\napp.ea gleProps.mailSmtpPort=25\napp.eagleProps.mailDebug=true\napp.eagleProps.eagleService.host=localhost\napp.eagleProps.eagleService.port=9099\napp.eagleProps.eagleService.username=admin\napp.eagleProps.eagleService.password=secret"; + String appConfigStr = "classification.hbase.zookeeper.property.clientPort=2181\nclassification.hbase.zookeeper.quorum=sandbox.hortonworks.com\n\napp.envContextConfig.env=storm\napp.envContextConfig.mode=cluster\napp.dataSourceConfig.topic=sandbox_hbase_security_log\napp.dataSourceConfig.zkConnection=127.0.0.1:2181\napp.dataSourceConfig.zkConnectionTimeoutMS=15000\napp.dataSourceConfig.brokerZkPath=/brokers\napp.dataSourceConfig.fetchSize=1048586\napp.dataSourceConfig.transactionZKServers=127.0.0.1\napp.dataSourceConfig.transactionZKPort=2181\napp.dataSourceConfig.transactionZKRoot=/consumers\napp.dataSourceConfig.consumerGroupId=eagle.hbasesecurity.consumer\napp.dataSourceConfig.transactionStateUpdateMS=2000\napp.dataSourceConfig.deserializerClass=org.apache.eagle.security.hbase.parse.HBaseAuditLogKafkaDeserializer\napp.eagleProps.site=sandbox\napp.eagleProps.application=hbaseSecurityLog\napp.eagleProps.dataJoinPollIntervalSec=30\napp.eagleProps.mailHost=mailHost.com\napp.ea gleProps.mailSmtpPort=25\napp.eagleProps.mailDebug=true\napp.eagleProps.eagleService.host=localhost\napp.eagleProps.eagleService.port=9099\napp.eagleProps.eagleService.username=admin\napp.eagleProps.eagleService.password=secret"; config = ConfigFactory.parseString(appConfigStr, options); Config appConfig = null; if(config.hasPath(EagleConfigConstants.APP_CONFIG)) { http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-auditlog/README.md ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hbase-auditlog/README.md b/eagle-security/eagle-security-hbase-auditlog/README.md index b0a6f90..d70c417 100644 --- a/eagle-security/eagle-security-hbase-auditlog/README.md +++ b/eagle-security/eagle-security-hbase-auditlog/README.md @@ -68,7 +68,7 @@ download logstash 2.3.x #### 2.3 run eagle webservice find eagle-webservice project, run it #### 2.4 run eagle-security-hdfs-authlog topology -find org.apache.eagle.security.hbase.HbaseAuditLogMonitoringMain, run it +find org.apache.eagle.security.hbase.HBaseAuditLogMonitoringMain, run it ### Step 3: alert engine: consume parsed log #### 3.1 run alert engine http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogApplication.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogApplication.java b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogApplication.java index 11a22e5..2008f0b 100644 --- a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogApplication.java +++ b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogApplication.java @@ -55,7 +55,7 @@ public class HBaseAuditLogApplication extends StormApplication { BoltDeclarer boltDeclarer = builder.setBolt("parserBolt", bolt, numOfParserTasks); boltDeclarer.fieldsGrouping("ingest", new Fields(StringScheme.STRING_SCHEME_KEY)); - HbaseResourceSensitivityDataJoinBolt joinBolt = new HbaseResourceSensitivityDataJoinBolt(config); + HBaseResourceSensitivityDataJoinBolt joinBolt = new HBaseResourceSensitivityDataJoinBolt(config); BoltDeclarer joinBoltDeclarer = builder.setBolt("joinBolt", joinBolt, numOfJoinTasks); joinBoltDeclarer.fieldsGrouping("parserBolt", new Fields("f1")); http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogObject.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogObject.java b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogObject.java new file mode 100644 index 0000000..c09073a --- /dev/null +++ b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogObject.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.eagle.security.hbase; + + +public class HBaseAuditLogObject { + public long timestamp; + public String user = ""; + public String scope = ""; + public String action = ""; + public String host = ""; + public String request = ""; + public String status = ""; +} http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogParser.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogParser.java b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogParser.java new file mode 100644 index 0000000..b033a82 --- /dev/null +++ b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogParser.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.eagle.security.hbase; + +import java.io.Serializable; +import java.util.LinkedList; +import java.util.Arrays; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.lang3.StringUtils; +import org.apache.eagle.common.DateTimeUtil; +import org.apache.eagle.security.util.LogParseUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +public class HBaseAuditLogParser implements Serializable { + private final static Logger LOG = LoggerFactory.getLogger(HBaseAuditLogParser.class); + + private final static int LOGDATE_INDEX = 1; + private final static int LOGLEVEL_INDEX = 2; + private final static int LOGATTRS_INDEX = 3; + private final static String ALLOWED = "allowed"; + private final static String DENIED = "denied"; + private final static Pattern loggerPattern = Pattern.compile("^([\\d\\s\\-:,]+)\\s+(\\w+)\\s+(.*)"); + private final static Pattern loggerContextPattern = Pattern.compile("\\w+:\\s*\\((.*)\\s*\\)"); + private final static Pattern allowedPattern = Pattern.compile(ALLOWED); + + + public HBaseAuditLogObject parse(String logLine) { + if(logLine == null || logLine.isEmpty()) return null; + + HBaseAuditLogObject ret = new HBaseAuditLogObject(); + String timestamp = ""; + String user = ""; + String scope = ""; + String action = ""; + String ip = ""; + String request = ""; + String family = ""; + String context = ""; + + Matcher loggerMatcher = loggerPattern.matcher(logLine); + if(loggerMatcher.find()) { + try { + timestamp = loggerMatcher.group(LOGDATE_INDEX); + String [] attrs = loggerMatcher.group(LOGATTRS_INDEX).split(";"); + ret.status = allowedPattern.matcher(attrs[0]).find() ? ALLOWED : DENIED; + try { + ip = attrs[2].split(":")[1].trim(); + } catch (Exception e) { + ip = ""; + } + try { + request = attrs[3].split(":")[1].trim(); + } catch (Exception e) { + request = ""; + } + try { + context = attrs[4].trim(); + } catch (Exception e) { + context = ""; + } + + Matcher contextMatcher = loggerContextPattern.matcher(context.replaceAll("\\s+","")); + if(contextMatcher.find()) { + boolean paramsOpen = false; + + List<String> kvs = new LinkedList<String>(Arrays.asList(contextMatcher.group(1).split(","))); + + while (!kvs.isEmpty()) { + String kv = kvs.get(0); + + if (kv.split("=").length < 2) { + kvs.remove(0); + continue; + } + + String k = kv.split("=")[0]; + String v = kv.split("=")[1]; + + if (paramsOpen && kv.substring(kv.length() - 1).equals("]")) { + paramsOpen = false; + v = v.substring(0, v.length() - 1); + } + + switch (k) { + case "user": + user = v; + break; + case "scope": + scope = v; + break; + case "family": + family = v; + break; + case "action": + action = v; + break; + case "params": + kvs.add(v.substring(1) + "=" + kv.split("=")[2]); + paramsOpen = true; + break; + default: break; + } + + kvs.remove(0); + } + } + + if(StringUtils.isNotEmpty(family)) { + if(!scope.contains(":")) scope = "default:" + scope; + scope = String.format("%s:%s", scope, family); + } + if(StringUtils.isNotEmpty(ip)) { + ret.host = ip.substring(1); + } + ret.timestamp = DateTimeUtil.humanDateToMilliseconds(timestamp); + ret.scope = scope; + ret.action = action; + ret.user = LogParseUtil.parseUserFromUGI(user); + ret.request = request; + return ret; + } catch(Exception e) { + LOG.error("Got exception when parsing audit log:" + logLine + ", exception:" + e.getMessage(), e); + } + } + return null; + } +} + http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogParserBolt.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogParserBolt.java b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogParserBolt.java index ffed0ef..36fe33a 100644 --- a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogParserBolt.java +++ b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogParserBolt.java @@ -34,7 +34,7 @@ import java.util.*; public class HBaseAuditLogParserBolt extends BaseRichBolt { private static Logger LOG = LoggerFactory.getLogger(HBaseAuditLogParserBolt.class); private OutputCollector collector; - private static final HbaseAuditLogParser parser = new HbaseAuditLogParser(); + private static final HBaseAuditLogParser parser = new HBaseAuditLogParser(); @Override public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { @@ -45,7 +45,7 @@ public class HBaseAuditLogParserBolt extends BaseRichBolt { public void execute(Tuple input) { String logLine = input.getString(0); try { - HbaseAuditLogObject entity = parser.parse(logLine); + HBaseAuditLogObject entity = parser.parse(logLine); Map<String, Object> map = new TreeMap<>(); map.put("action", entity.action); map.put("host", entity.host); http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseResourceSensitivityDataJoinBolt.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseResourceSensitivityDataJoinBolt.java b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseResourceSensitivityDataJoinBolt.java new file mode 100644 index 0000000..8b5461e --- /dev/null +++ b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseResourceSensitivityDataJoinBolt.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.eagle.security.hbase; + +import backtype.storm.topology.OutputFieldsDeclarer; +import backtype.storm.tuple.Fields; +import backtype.storm.tuple.Tuple; +import com.typesafe.config.Config; +import org.apache.eagle.security.service.HBaseSensitivityEntity; +import org.apache.eagle.security.enrich.AbstractDataEnrichBolt; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.Map; +import java.util.TreeMap; +import java.util.regex.Pattern; + +public class HBaseResourceSensitivityDataJoinBolt extends AbstractDataEnrichBolt<HBaseSensitivityEntity, String> { + private final static Logger LOG = LoggerFactory.getLogger(HBaseResourceSensitivityDataJoinBolt.class); + + public HBaseResourceSensitivityDataJoinBolt(Config config) { + super(config, new HBaseSensitivityDataEnrichLCM(config)); + } + + @Override + public void executeWithEnrich(Tuple input, Map<String, HBaseSensitivityEntity> map) { + try { + Map<String, Object> event = (Map<String, Object>) input.getValue(0); + LOG.info(">>>> event: " + event + " >>>> map: " + map); + + String resource = (String) event.get("scope"); + + HBaseSensitivityEntity sensitivityEntity = null; + + if (map != null && resource != "") { + for (String key : map.keySet()) { + Pattern pattern = Pattern.compile(key, Pattern.CASE_INSENSITIVE); + if (pattern.matcher(resource).find()) { + sensitivityEntity = map.get(key); + break; + } + } + } + Map<String, Object> newEvent = new TreeMap<String, Object>(event); + newEvent.put("sensitivityType", sensitivityEntity == null + ? "NA" : sensitivityEntity.getSensitivityType()); + newEvent.put("scope", resource); + if (LOG.isDebugEnabled()) { + LOG.debug("After hbase resource sensitivity lookup: " + newEvent); + } + LOG.info("After hbase resource sensitivity lookup: " + newEvent); + // push to Kafka sink + collector.emit(Arrays.asList(newEvent.get("user"), newEvent)); + } catch (Exception ex) { + LOG.error("error joining data, ignore it", ex); + } finally { + collector.ack(input); + } + } + + @Override + public void declareOutputFields(OutputFieldsDeclarer declarer) { + declarer.declare(new Fields("user", "message")); + } +} http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseSensitivityDataEnrichLCM.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseSensitivityDataEnrichLCM.java b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseSensitivityDataEnrichLCM.java index 2cb1dec..9745706 100644 --- a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseSensitivityDataEnrichLCM.java +++ b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseSensitivityDataEnrichLCM.java @@ -38,6 +38,6 @@ public class HBaseSensitivityDataEnrichLCM extends AbstractDataEnrichLCM<HBaseSe @Override public String getCacheKey(HBaseSensitivityEntity entity) { - return entity.getHbaseResource(); + return entity.getHBaseResource(); } } http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogObject.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogObject.java b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogObject.java deleted file mode 100644 index ed9367d..0000000 --- a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogObject.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.eagle.security.hbase; - - -public class HbaseAuditLogObject { - public long timestamp; - public String user = ""; - public String scope = ""; - public String action = ""; - public String host = ""; - public String request = ""; - public String status = ""; -} http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogParser.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogParser.java b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogParser.java deleted file mode 100644 index bff9c0b..0000000 --- a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogParser.java +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package org.apache.eagle.security.hbase; - -import java.io.Serializable; -import java.util.LinkedList; -import java.util.Arrays; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.apache.commons.lang3.StringUtils; -import org.apache.eagle.common.DateTimeUtil; -import org.apache.eagle.security.util.LogParseUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - -public class HbaseAuditLogParser implements Serializable { - private final static Logger LOG = LoggerFactory.getLogger(HbaseAuditLogParser.class); - - private final static int LOGDATE_INDEX = 1; - private final static int LOGLEVEL_INDEX = 2; - private final static int LOGATTRS_INDEX = 3; - private final static String ALLOWED = "allowed"; - private final static String DENIED = "denied"; - private final static Pattern loggerPattern = Pattern.compile("^([\\d\\s\\-:,]+)\\s+(\\w+)\\s+(.*)"); - private final static Pattern loggerContextPattern = Pattern.compile("\\w+:\\s*\\((.*)\\s*\\)"); - private final static Pattern allowedPattern = Pattern.compile(ALLOWED); - - - public HbaseAuditLogObject parse(String logLine) { - if(logLine == null || logLine.isEmpty()) return null; - - HbaseAuditLogObject ret = new HbaseAuditLogObject(); - String timestamp = ""; - String user = ""; - String scope = ""; - String action = ""; - String ip = ""; - String request = ""; - String family = ""; - String context = ""; - - Matcher loggerMatcher = loggerPattern.matcher(logLine); - if(loggerMatcher.find()) { - try { - timestamp = loggerMatcher.group(LOGDATE_INDEX); - String [] attrs = loggerMatcher.group(LOGATTRS_INDEX).split(";"); - ret.status = allowedPattern.matcher(attrs[0]).find() ? ALLOWED : DENIED; - try { - ip = attrs[2].split(":")[1].trim(); - } catch (Exception e) { - ip = ""; - } - try { - request = attrs[3].split(":")[1].trim(); - } catch (Exception e) { - request = ""; - } - try { - context = attrs[4].trim(); - } catch (Exception e) { - context = ""; - } - - Matcher contextMatcher = loggerContextPattern.matcher(context.replaceAll("\\s+","")); - if(contextMatcher.find()) { - boolean paramsOpen = false; - - List<String> kvs = new LinkedList<String>(Arrays.asList(contextMatcher.group(1).split(","))); - - while (!kvs.isEmpty()) { - String kv = kvs.get(0); - - if (kv.split("=").length < 2) { - kvs.remove(0); - continue; - } - - String k = kv.split("=")[0]; - String v = kv.split("=")[1]; - - if (paramsOpen && kv.substring(kv.length() - 1).equals("]")) { - paramsOpen = false; - v = v.substring(0, v.length() - 1); - } - - switch (k) { - case "user": - user = v; - break; - case "scope": - scope = v; - break; - case "family": - family = v; - break; - case "action": - action = v; - break; - case "params": - kvs.add(v.substring(1) + "=" + kv.split("=")[2]); - paramsOpen = true; - break; - default: break; - } - - kvs.remove(0); - } - } - - if(StringUtils.isNotEmpty(family)) { - if(!scope.contains(":")) scope = "default:" + scope; - scope = String.format("%s:%s", scope, family); - } - if(StringUtils.isNotEmpty(ip)) { - ret.host = ip.substring(1); - } - ret.timestamp = DateTimeUtil.humanDateToMilliseconds(timestamp); - ret.scope = scope; - ret.action = action; - ret.user = LogParseUtil.parseUserFromUGI(user); - ret.request = request; - return ret; - } catch(Exception e) { - LOG.error("Got exception when parsing audit log:" + logLine + ", exception:" + e.getMessage(), e); - } - } - return null; - } -} - http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityDataJoinBolt.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityDataJoinBolt.java b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityDataJoinBolt.java deleted file mode 100644 index a1545c2..0000000 --- a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityDataJoinBolt.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.eagle.security.hbase; - -import backtype.storm.topology.OutputFieldsDeclarer; -import backtype.storm.tuple.Fields; -import backtype.storm.tuple.Tuple; -import com.typesafe.config.Config; -import org.apache.eagle.security.service.HBaseSensitivityEntity; -import org.apache.eagle.security.enrich.AbstractDataEnrichBolt; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Arrays; -import java.util.Map; -import java.util.TreeMap; -import java.util.regex.Pattern; - -public class HbaseResourceSensitivityDataJoinBolt extends AbstractDataEnrichBolt<HBaseSensitivityEntity, String> { - private final static Logger LOG = LoggerFactory.getLogger(HbaseResourceSensitivityDataJoinBolt.class); - - public HbaseResourceSensitivityDataJoinBolt(Config config) { - super(config, new HBaseSensitivityDataEnrichLCM(config)); - } - - @Override - public void executeWithEnrich(Tuple input, Map<String, HBaseSensitivityEntity> map) { - try { - Map<String, Object> event = (Map<String, Object>) input.getValue(0); - LOG.info(">>>> event: " + event + " >>>> map: " + map); - - String resource = (String) event.get("scope"); - - HBaseSensitivityEntity sensitivityEntity = null; - - if (map != null && resource != "") { - for (String key : map.keySet()) { - Pattern pattern = Pattern.compile(key, Pattern.CASE_INSENSITIVE); - if (pattern.matcher(resource).find()) { - sensitivityEntity = map.get(key); - break; - } - } - } - Map<String, Object> newEvent = new TreeMap<String, Object>(event); - newEvent.put("sensitivityType", sensitivityEntity == null - ? "NA" : sensitivityEntity.getSensitivityType()); - newEvent.put("scope", resource); - if (LOG.isDebugEnabled()) { - LOG.debug("After hbase resource sensitivity lookup: " + newEvent); - } - LOG.info("After hbase resource sensitivity lookup: " + newEvent); - // push to Kafka sink - collector.emit(Arrays.asList(newEvent.get("user"), newEvent)); - } catch (Exception ex) { - LOG.error("error joining data, ignore it", ex); - } finally { - collector.ack(input); - } - } - - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declare(new Fields("user", "message")); - } -} http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.hbase.HBaseAuditLogAppProvider.xml ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.hbase.HBaseAuditLogAppProvider.xml b/eagle-security/eagle-security-hbase-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.hbase.HBaseAuditLogAppProvider.xml index 1fbf863..16651aa 100644 --- a/eagle-security/eagle-security-hbase-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.hbase.HBaseAuditLogAppProvider.xml +++ b/eagle-security/eagle-security-hbase-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.hbase.HBaseAuditLogAppProvider.xml @@ -18,7 +18,7 @@ <application> <type>HBASE_AUDIT_LOG_MONITOR</type> - <name>Hbase Audit Log Monitor</name> + <name>HBase Audit Log Monitor</name> <appClass>org.apache.eagle.security.hbase.HBaseAuditLogApplication</appClass> <configuration> <property> @@ -150,7 +150,7 @@ <streams> <stream> <streamId>hbase_audit_log_stream</streamId> - <description>Hbase Audit Log Stream</description> + <description>HBase Audit Log Stream</description> <validate>true</validate> <timeseries>true</timeseries> <columns> http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-auditlog/src/test/java/org/apache/eagle/security/hbase/TestHBaseAuditLogParser.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hbase-auditlog/src/test/java/org/apache/eagle/security/hbase/TestHBaseAuditLogParser.java b/eagle-security/eagle-security-hbase-auditlog/src/test/java/org/apache/eagle/security/hbase/TestHBaseAuditLogParser.java new file mode 100644 index 0000000..945ea50 --- /dev/null +++ b/eagle-security/eagle-security-hbase-auditlog/src/test/java/org/apache/eagle/security/hbase/TestHBaseAuditLogParser.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.eagle.security.hbase; + +import org.junit.Assert; +import org.junit.Test; + + +public class TestHBaseAuditLogParser { + HBaseAuditLogParser parser = new HBaseAuditLogParser(); + + @Test + public void test() throws Exception { + String log = "2015-08-11 13:31:03,729 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access allowed for user eagle; reason: Table permission granted; remote address: /127.0.0.1; request: get; context: (user=eagle,scope=hbase:namespace,family=info, action=READ)"; + HBaseAuditLogObject obj = parser.parse(log); + Assert.assertEquals(obj.action, "READ"); + Assert.assertEquals(obj.host, "127.0.0.1"); + Assert.assertEquals(obj.scope, "hbase:namespace:info"); + } + + @Test + public void test2() throws Exception { + String log = "2015-08-04 12:29:03,073 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access allowed for user eagle; reason: Global check allowed; remote address: ; request: preOpen; context: (user=eagle, scope=GLOBAL, family=, action=ADMIN)"; + HBaseAuditLogObject obj = parser.parse(log); + Assert.assertEquals(obj.action, "ADMIN"); + Assert.assertEquals(obj.host, ""); + Assert.assertEquals(obj.scope, "GLOBAL"); + } +}
