METRON-975: Normalize logging and switch to common idiom with SLF4J (mmiklavc) closes apache/metron#599
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/ec959d20 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/ec959d20 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/ec959d20 Branch: refs/heads/master Commit: ec959d20efa72e9868d87a02d9407bbaad34c4c8 Parents: 7b9f20c Author: mmiklavc <[email protected]> Authored: Tue Aug 1 09:33:20 2017 -0600 Committer: Michael Miklavcic <[email protected]> Committed: Tue Aug 1 09:33:20 2017 -0600 ---------------------------------------------------------------------- .../metron/maas/functions/MaaSFunctions.java | 28 +++---- .../profiler/client/stellar/GetProfile.java | 28 ++++--- .../metron/profiler/client/stellar/Util.java | 20 ++--- .../apache/metron/profiler/ProfileBuilder.java | 28 +++---- .../profiler/bolt/KafkaDestinationHandler.java | 6 +- .../profiler/bolt/ProfileBuilderBolt.java | 18 ++--- .../profiler/bolt/ProfileSplitterBolt.java | 22 +++--- .../apache/metron/api/ConfigurationManager.java | 24 +++--- .../pcapservice/PcapReceiverImplRestEasy.java | 44 ++++++----- .../metron/pcapservice/rest/PcapService.java | 13 ++- .../metron/common/bolt/ConfiguredBolt.java | 19 ++--- .../common/bolt/ConfiguredEnrichmentBolt.java | 10 ++- .../common/bolt/ConfiguredIndexingBolt.java | 12 +-- .../common/bolt/ConfiguredParserBolt.java | 11 ++- .../common/bolt/ConfiguredProfilerBolt.java | 15 ++-- .../common/configuration/Configurations.java | 10 +-- .../SensorEnrichmentUpdateConfig.java | 19 +++-- .../common/hadoop/SequenceFileIterable.java | 23 +++--- .../apache/metron/common/utils/ErrorUtils.java | 18 ++--- .../apache/metron/common/utils/SerDeUtils.java | 40 ++++++---- .../apache/metron/common/system/ClockTest.java | 9 +-- .../metron/dataloads/bulk/DataPruner.java | 9 ++- .../dataloads/bulk/ElasticsearchDataPruner.java | 25 +++--- .../bulk/ElasticsearchDataPrunerRunner.java | 32 ++++---- .../metron/dataloads/bulk/HDFSDataPruner.java | 37 ++++----- .../TransformFilterExtractorDecorator.java | 26 ++++-- .../flatfile/importer/MapReduceImporter.java | 19 ++--- .../dataloads/nonbulk/taxii/TaxiiHandler.java | 83 +++++++++++++------- .../writer/ElasticsearchWriter.java | 29 +++---- .../adapters/cif/CIFHbaseAdapter.java | 26 +++--- .../enrichment/adapters/geo/GeoAdapter.java | 12 +-- .../adapters/geo/GeoLiteDatabase.java | 18 ++--- .../enrichment/adapters/jdbc/JdbcAdapter.java | 2 +- .../simplehbase/SimpleHBaseAdapter.java | 26 +++--- .../adapters/stellar/StellarAdapter.java | 26 +++--- .../threatintel/ThreatIntelAdapter.java | 26 +++--- .../enrichment/bolt/EnrichmentJoinBolt.java | 4 +- .../enrichment/bolt/EnrichmentSplitterBolt.java | 26 +++--- .../enrichment/bolt/GenericEnrichmentBolt.java | 4 +- .../apache/metron/enrichment/bolt/JoinBolt.java | 14 ++-- .../enrichment/bolt/ThreatIntelJoinBolt.java | 25 +++--- .../bolt/ThreatIntelSplitterBolt.java | 4 +- .../accesstracker/PersistentAccessTracker.java | 11 +-- .../stellar/GeoEnrichmentFunctions.java | 17 ++-- .../stellar/SimpleHBaseEnrichmentFunctions.java | 37 ++++----- .../apache/metron/hbase/HTableConnector.java | 20 ++--- .../apache/metron/hbase/TupleTableConfig.java | 10 +-- .../apache/metron/hbase/bolt/BatchHelper.java | 8 +- .../org/apache/metron/hbase/bolt/HBaseBolt.java | 22 +++--- .../apache/metron/hbase/client/HBaseClient.java | 12 +-- .../components/FluxTopologyComponent.java | 38 ++++----- .../integration/components/KafkaComponent.java | 27 +++++-- .../management/ConfigurationFunctions.java | 30 ++++--- .../management/EnrichmentConfigFunctions.java | 23 +++--- .../metron/management/FileSystemFunctions.java | 30 ++++--- .../apache/metron/management/GrokFunctions.java | 24 +++--- .../management/IndexingConfigFunctions.java | 23 +++--- .../management/ParserConfigFunctions.java | 19 +++-- .../metron/management/ShellFunctions.java | 30 ++++--- .../management/ThreatTriageFunctions.java | 31 +++++--- .../org/apache/metron/parsers/BasicParser.java | 14 ++-- .../org/apache/metron/parsers/GrokParser.java | 69 ++++++---------- .../metron/parsers/asa/BasicAsaParser.java | 23 +++--- .../apache/metron/parsers/bolt/ParserBolt.java | 27 ++++--- .../metron/parsers/bro/BasicBroParser.java | 28 +++---- .../apache/metron/parsers/cef/CEFParser.java | 6 +- .../apache/metron/parsers/csv/CSVParser.java | 14 ++-- .../parsers/fireeye/BasicFireEyeParser.java | 2 +- .../metron/parsers/ise/BasicIseParser.java | 2 +- .../parsers/lancope/BasicLancopeParser.java | 17 ++-- .../paloalto/BasicPaloAltoFirewallParser.java | 4 +- .../metron/parsers/snort/BasicSnortParser.java | 18 ++--- .../sourcefire/BasicSourcefireParser.java | 6 +- .../components/ParserTopologyComponent.java | 21 ++--- .../org/apache/metron/pcap/query/PcapCli.java | 22 +++--- .../metron/spout/pcap/HDFSWriterCallback.java | 25 +++--- .../metron/spout/pcap/PartitionHDFSWriter.java | 20 ++--- .../pcap/deserializer/FromKeyDeserializer.java | 3 +- .../deserializer/FromPacketDeserializer.java | 3 +- .../java/org/apache/metron/pcap/PacketInfo.java | 12 ++- .../metron/pcap/PcapByteOutputStream.java | 10 +-- .../java/org/apache/metron/pcap/PcapHelper.java | 6 +- .../java/org/apache/metron/pcap/PcapMerger.java | 15 ++-- .../metron/pcap/PcapPacketComparator.java | 12 ++- .../java/org/apache/metron/pcap/mr/PcapJob.java | 5 +- .../metron/pcap/utils/FileFilterUtil.java | 5 +- .../storm/kafka/flux/StormKafkaSpout.java | 12 +-- .../metron/writer/BulkWriterComponent.java | 6 +- .../writer/hdfs/SourceAwareMoveAction.java | 11 +-- .../metron/writer/hdfs/SourceHandler.java | 23 +++--- .../common/utils/VFSClassloaderUtil.java | 19 +++-- .../resolver/BaseFunctionResolver.java | 25 +++--- .../resolver/ClasspathFunctionResolver.java | 31 +++++--- .../resolver/SimpleFunctionResolver.java | 8 +- 94 files changed, 969 insertions(+), 857 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/ec959d20/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java index 96d43c2..e205518 100644 --- a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java +++ b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java @@ -20,11 +20,15 @@ package org.apache.metron.maas.functions; import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; +import java.lang.invoke.MethodHandles; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; import org.apache.curator.framework.CuratorFramework; -import org.apache.metron.stellar.dsl.Context; -import org.apache.metron.stellar.dsl.ParseException; -import org.apache.metron.stellar.dsl.Stellar; -import org.apache.metron.stellar.dsl.StellarFunction; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.maas.config.Endpoint; import org.apache.metron.maas.config.MaaSConfig; @@ -32,19 +36,15 @@ import org.apache.metron.maas.config.ModelEndpoint; import org.apache.metron.maas.discovery.ServiceDiscoverer; import org.apache.metron.maas.util.ConfigUtil; import org.apache.metron.maas.util.RESTUtil; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.ParseException; +import org.apache.metron.stellar.dsl.Stellar; +import org.apache.metron.stellar.dsl.StellarFunction; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.TimeUnit; - public class MaaSFunctions { - protected static final Logger LOG = LoggerFactory.getLogger(MaaSFunctions.class); + protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static class ModelCacheKey { String name; String version; @@ -276,7 +276,7 @@ public class MaaSFunctions { return ep == null ? null : endpointToMap(ep.getName(), ep.getVersion(), ep.getEndpoint()); } catch(Exception ex) { - LOG.error("Unable to discover endpoint: " + ex.getMessage(), ex); + LOG.error("Unable to discover endpoint: {}", ex.getMessage(), ex); return null; } } http://git-wip-us.apache.org/repos/asf/metron/blob/ec959d20/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/GetProfile.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/GetProfile.java b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/GetProfile.java index 04b391e..03ac0a6 100644 --- a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/GetProfile.java +++ b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/GetProfile.java @@ -20,6 +20,23 @@ package org.apache.metron.profiler.client.stellar; +import static org.apache.metron.profiler.client.stellar.ProfilerConfig.PROFILER_COLUMN_FAMILY; +import static org.apache.metron.profiler.client.stellar.ProfilerConfig.PROFILER_HBASE_TABLE; +import static org.apache.metron.profiler.client.stellar.ProfilerConfig.PROFILER_HBASE_TABLE_PROVIDER; +import static org.apache.metron.profiler.client.stellar.ProfilerConfig.PROFILER_PERIOD; +import static org.apache.metron.profiler.client.stellar.ProfilerConfig.PROFILER_PERIOD_UNITS; +import static org.apache.metron.profiler.client.stellar.ProfilerConfig.PROFILER_SALT_DIVISOR; +import static org.apache.metron.profiler.client.stellar.Util.getArg; +import static org.apache.metron.profiler.client.stellar.Util.getEffectiveConfig; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.metron.stellar.dsl.Context; @@ -38,15 +55,6 @@ import org.apache.metron.profiler.hbase.ValueOnlyColumnBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.util.*; -import java.util.concurrent.TimeUnit; - -import static java.lang.String.format; -import static org.apache.metron.profiler.client.stellar.ProfilerConfig.*; -import static org.apache.metron.profiler.client.stellar.Util.getArg; -import static org.apache.metron.profiler.client.stellar.Util.getEffectiveConfig; - /** * A Stellar function that can retrieve data contained within a Profile. * @@ -105,7 +113,7 @@ public class GetProfile implements StellarFunction { */ private Map<String, Object> cachedConfigMap = new HashMap<String, Object>(6); - private static final Logger LOG = LoggerFactory.getLogger(GetProfile.class); + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); /** * Initialization. No longer need to do anything in initialization, http://git-wip-us.apache.org/repos/asf/metron/blob/ec959d20/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/Util.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/Util.java b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/Util.java index f30af74..279a60c 100644 --- a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/Util.java +++ b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/Util.java @@ -19,24 +19,24 @@ */ package org.apache.metron.profiler.client.stellar; -import org.apache.commons.lang.StringUtils; -import org.apache.metron.stellar.dsl.Context; -import org.apache.metron.stellar.dsl.ParseException; -import org.apache.metron.stellar.common.utils.ConversionUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static java.lang.String.format; +import static org.apache.metron.stellar.dsl.Context.Capabilities.GLOBAL_CONFIG; +import java.lang.invoke.MethodHandles; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; - -import static java.lang.String.format; -import static org.apache.metron.stellar.dsl.Context.Capabilities.GLOBAL_CONFIG; +import org.apache.commons.lang.StringUtils; +import org.apache.metron.stellar.common.utils.ConversionUtils; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.ParseException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class Util { - private static final Logger LOG = LoggerFactory.getLogger(Util.class); + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); /** * Ensure that the required capabilities are defined. http://git-wip-us.apache.org/repos/asf/metron/blob/ec959d20/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileBuilder.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileBuilder.java b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileBuilder.java index 4a9946d..da8db82 100644 --- a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileBuilder.java +++ b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileBuilder.java @@ -20,33 +20,33 @@ package org.apache.metron.profiler; +import static java.lang.String.format; + import com.google.common.collect.ImmutableMap; +import java.io.Serializable; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import org.apache.commons.collections4.ListUtils; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; import org.apache.curator.framework.CuratorFramework; import org.apache.metron.common.configuration.profiler.ProfileConfig; +import org.apache.metron.profiler.clock.Clock; +import org.apache.metron.profiler.clock.WallClock; +import org.apache.metron.stellar.common.DefaultStellarStatefulExecutor; import org.apache.metron.stellar.common.StellarStatefulExecutor; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.ParseException; import org.apache.metron.stellar.dsl.StellarFunctions; -import org.apache.metron.profiler.clock.Clock; -import org.apache.metron.profiler.clock.WallClock; -import org.apache.metron.stellar.common.DefaultStellarStatefulExecutor; import org.json.simple.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -import static java.lang.String.format; - /** * Responsible for building and maintaining a Profile. * @@ -59,7 +59,7 @@ import static java.lang.String.format; */ public class ProfileBuilder implements Serializable { - protected static final Logger LOG = LoggerFactory.getLogger(ProfileBuilder.class); + protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); /** * The name of the profile. http://git-wip-us.apache.org/repos/asf/metron/blob/ec959d20/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/KafkaDestinationHandler.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/KafkaDestinationHandler.java b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/KafkaDestinationHandler.java index 5d8c971..be82468 100644 --- a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/KafkaDestinationHandler.java +++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/KafkaDestinationHandler.java @@ -19,6 +19,8 @@ package org.apache.metron.profiler.bolt; +import java.io.Serializable; +import java.lang.invoke.MethodHandles; import org.apache.commons.lang3.ClassUtils; import org.apache.metron.profiler.ProfileMeasurement; import org.apache.storm.task.OutputCollector; @@ -29,15 +31,13 @@ import org.json.simple.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.Serializable; - /** * Handles emitting a ProfileMeasurement to the stream which writes * profile measurements to Kafka. */ public class KafkaDestinationHandler implements DestinationHandler, Serializable { - protected static final Logger LOG = LoggerFactory.getLogger(KafkaDestinationHandler.class); + protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); /** * The stream identifier used for this destination; http://git-wip-us.apache.org/repos/asf/metron/blob/ec959d20/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java index da33080..1323089 100644 --- a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java +++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileBuilderBolt.java @@ -20,8 +20,16 @@ package org.apache.metron.profiler.bolt; +import static java.lang.String.format; + import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; import org.apache.metron.common.bolt.ConfiguredProfilerBolt; import org.apache.metron.common.configuration.profiler.ProfileConfig; import org.apache.metron.stellar.common.utils.ConversionUtils; @@ -39,14 +47,6 @@ import org.json.simple.parser.JSONParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; - -import static java.lang.String.format; - /** * A bolt that is responsible for building a Profile. * @@ -57,7 +57,7 @@ import static java.lang.String.format; */ public class ProfileBuilderBolt extends ConfiguredProfilerBolt { - protected static final Logger LOG = LoggerFactory.getLogger(ProfileBuilderBolt.class); + protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private OutputCollector collector; http://git-wip-us.apache.org/repos/asf/metron/blob/ec959d20/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java index 31a4093..255069a 100644 --- a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java +++ b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java @@ -20,30 +20,28 @@ package org.apache.metron.profiler.bolt; +import java.io.UnsupportedEncodingException; +import java.lang.invoke.MethodHandles; +import java.util.Map; +import org.apache.metron.common.bolt.ConfiguredProfilerBolt; +import org.apache.metron.common.configuration.profiler.ProfileConfig; +import org.apache.metron.common.configuration.profiler.ProfilerConfig; import org.apache.metron.stellar.common.DefaultStellarStatefulExecutor; import org.apache.metron.stellar.common.StellarStatefulExecutor; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.StellarFunctions; import org.apache.storm.task.OutputCollector; import org.apache.storm.task.TopologyContext; import org.apache.storm.topology.OutputFieldsDeclarer; import org.apache.storm.tuple.Fields; import org.apache.storm.tuple.Tuple; import org.apache.storm.tuple.Values; -import org.apache.metron.common.bolt.ConfiguredProfilerBolt; -import org.apache.metron.common.configuration.profiler.ProfileConfig; -import org.apache.metron.common.configuration.profiler.ProfilerConfig; -import org.apache.metron.stellar.dsl.Context; -import org.apache.metron.stellar.dsl.StellarFunctions; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.UnsupportedEncodingException; -import java.util.Map; - -import static java.lang.String.format; - /** * The bolt responsible for filtering incoming messages and directing * each to the one or more bolts responsible for building a Profile. Each @@ -51,7 +49,7 @@ import static java.lang.String.format; */ public class ProfileSplitterBolt extends ConfiguredProfilerBolt { - protected static final Logger LOG = LoggerFactory.getLogger(ProfileSplitterBolt.class); + protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private OutputCollector collector; @@ -97,7 +95,7 @@ public class ProfileSplitterBolt extends ConfiguredProfilerBolt { doExecute(input); } catch (IllegalArgumentException | ParseException | UnsupportedEncodingException e) { - LOG.error(format("Unexpected failure: message='%s', tuple='%s'", e.getMessage(), input), e); + LOG.error("Unexpected failure: message='{}', tuple='{}'", e.getMessage(), input, e); collector.reportError(e); } finally { http://git-wip-us.apache.org/repos/asf/metron/blob/ec959d20/metron-platform/metron-api/src/main/java/org/apache/metron/api/ConfigurationManager.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-api/src/main/java/org/apache/metron/api/ConfigurationManager.java b/metron-platform/metron-api/src/main/java/org/apache/metron/api/ConfigurationManager.java index f9dbfc2..7297b7e 100644 --- a/metron-platform/metron-api/src/main/java/org/apache/metron/api/ConfigurationManager.java +++ b/metron-platform/metron-api/src/main/java/org/apache/metron/api/ConfigurationManager.java @@ -18,14 +18,18 @@ package org.apache.metron.api; - import java.io.File; +import java.lang.invoke.MethodHandles; import java.util.HashMap; import java.util.Map; - -import org.apache.commons.configuration.*; +import org.apache.commons.configuration.CombinedConfiguration; +import org.apache.commons.configuration.Configuration; +import org.apache.commons.configuration.ConfigurationException; +import org.apache.commons.configuration.ConfigurationRuntimeException; +import org.apache.commons.configuration.DefaultConfigurationBuilder; import org.apache.commons.lang.StringUtils; -import org.apache.log4j.Logger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Configuration manager class which loads all 'config-definition.xml' files and @@ -33,6 +37,7 @@ import org.apache.log4j.Logger; * configuration resource */ public class ConfigurationManager { + private static final Logger LOGGER = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); /** configuration definition file name. */ private static String DEFAULT_CONFIG_DEFINITION_FILE_NAME = "config-definition.xml"; @@ -40,9 +45,6 @@ public class ConfigurationManager { /** Stores a map with the configuration for each path specified. */ private static Map<String, Configuration> configurationsCache = new HashMap<String, Configuration>(); - /** The Constant LOGGER. */ - private static final Logger LOGGER = Logger - .getLogger(ConfigurationManager.class); /** * Common method to load content of all configuration resources defined in @@ -62,12 +64,12 @@ public class ConfigurationManager { return configurationsCache.get(configDefFilePath); } DefaultConfigurationBuilder builder = new DefaultConfigurationBuilder(); - String fielPath = getConfigDefFilePath(configDefFilePath); - LOGGER.info("loading from 'configDefFilePath' :" + fielPath); - builder.setFile(new File(fielPath)); + String filePath = getConfigDefFilePath(configDefFilePath); + LOGGER.info("loading from 'configDefFilePath' : {}", filePath); + builder.setFile(new File(filePath)); try { configuration = builder.getConfiguration(true); - configurationsCache.put(fielPath, configuration); + configurationsCache.put(filePath, configuration); } catch (ConfigurationException|ConfigurationRuntimeException e) { LOGGER.info("Exception in loading property files.", e); } http://git-wip-us.apache.org/repos/asf/metron/blob/ec959d20/metron-platform/metron-api/src/main/java/org/apache/metron/pcapservice/PcapReceiverImplRestEasy.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-api/src/main/java/org/apache/metron/pcapservice/PcapReceiverImplRestEasy.java b/metron-platform/metron-api/src/main/java/org/apache/metron/pcapservice/PcapReceiverImplRestEasy.java index 9c58813..56ec7b1 100644 --- a/metron-platform/metron-api/src/main/java/org/apache/metron/pcapservice/PcapReceiverImplRestEasy.java +++ b/metron-platform/metron-api/src/main/java/org/apache/metron/pcapservice/PcapReceiverImplRestEasy.java @@ -20,10 +20,25 @@ package org.apache.metron.pcapservice; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.collect.Lists; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.DefaultValue; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.QueryParam; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; -import org.apache.log4j.Logger; import org.apache.metron.common.Constants; import org.apache.metron.common.hadoop.SequenceFileIterable; import org.apache.metron.common.utils.timestamp.TimestampConverters; @@ -31,21 +46,12 @@ import org.apache.metron.pcap.PcapHelper; import org.apache.metron.pcap.filter.fixed.FixedPcapFilter; import org.apache.metron.pcap.filter.query.QueryPcapFilter; import org.apache.metron.pcap.mr.PcapJob; - -import javax.servlet.http.HttpServletResponse; -import javax.ws.rs.*; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import java.io.IOException; -import java.util.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @Path("/") public class PcapReceiverImplRestEasy { - - /** The Constant LOGGER. */ - private static final Logger LOGGER = Logger - .getLogger(PcapReceiverImplRestEasy.class); + private static final Logger LOGGER = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); /** The Constant HEADER_CONTENT_DISPOSITION_NAME. */ private static final String HEADER_CONTENT_DISPOSITION_NAME = "Content-Disposition"; @@ -135,9 +141,7 @@ public class PcapReceiverImplRestEasy { //convert to nanoseconds since the epoch startTime = TimestampConverters.MILLISECONDS.toNanoseconds(startTime); endTime = TimestampConverters.MILLISECONDS.toNanoseconds(endTime); - if(LOGGER.isDebugEnabled()) { - LOGGER.debug("Query received: " + query); - } + LOGGER.debug("Query received: {}", query); results = getQueryUtil().query(new org.apache.hadoop.fs.Path(ConfigurationUtil.getPcapOutputPath()) , new org.apache.hadoop.fs.Path(ConfigurationUtil.getTempQueryOutputPath()) , startTime @@ -151,8 +155,7 @@ public class PcapReceiverImplRestEasy { response.setPcaps(results != null ? Lists.newArrayList(results) : null); } catch (Exception e) { - LOGGER.error("Exception occurred while fetching Pcaps by identifiers :", - e); + LOGGER.error("Exception occurred while fetching Pcaps by identifiers :", e); throw new WebApplicationException("Unable to fetch Pcaps via MR job", e); } finally { if (null != results) { @@ -246,7 +249,7 @@ public class PcapReceiverImplRestEasy { } }}; if(LOGGER.isDebugEnabled()) { - LOGGER.debug("Query received: " + Joiner.on(",").join(query.entrySet())); + LOGGER.debug("Query received: {}", Joiner.on(",").join(query.entrySet())); } results = getQueryUtil().query(new org.apache.hadoop.fs.Path(ConfigurationUtil.getPcapOutputPath()) , new org.apache.hadoop.fs.Path(ConfigurationUtil.getTempQueryOutputPath()) @@ -261,8 +264,7 @@ public class PcapReceiverImplRestEasy { response.setPcaps(results != null ? Lists.newArrayList(results) : null); } catch (Exception e) { - LOGGER.error("Exception occurred while fetching Pcaps by identifiers :", - e); + LOGGER.error("Exception occurred while fetching Pcaps by identifiers :", e); throw new WebApplicationException("Unable to fetch Pcaps via MR job", e); } finally { if (null != results) { http://git-wip-us.apache.org/repos/asf/metron/blob/ec959d20/metron-platform/metron-api/src/main/java/org/apache/metron/pcapservice/rest/PcapService.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-api/src/main/java/org/apache/metron/pcapservice/rest/PcapService.java b/metron-platform/metron-api/src/main/java/org/apache/metron/pcapservice/rest/PcapService.java index 6773acb..d183d5d 100644 --- a/metron-platform/metron-api/src/main/java/org/apache/metron/pcapservice/rest/PcapService.java +++ b/metron-platform/metron-api/src/main/java/org/apache/metron/pcapservice/rest/PcapService.java @@ -18,28 +18,27 @@ package org.apache.metron.pcapservice.rest; import java.io.IOException; - -import org.apache.log4j.Logger; +import java.lang.invoke.MethodHandles; import org.apache.metron.api.helper.service.PcapServiceCli; import org.apache.metron.pcapservice.ConfigurationUtil; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; import org.jboss.resteasy.plugins.server.servlet.HttpServletDispatcher; - +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class PcapService { + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private static final Logger LOG = Logger.getLogger(PcapService.class); public static void main(String[] args) throws IOException { - PcapServiceCli cli = new PcapServiceCli(args); cli.parse(); ConfigurationUtil.setPcapOutputPath(cli.getPcapHdfsPath()); - LOG.info("Pcap location set to " + cli.getPcapHdfsPath()); + LOG.info("Pcap location set to {}", cli.getPcapHdfsPath()); ConfigurationUtil.setTempQueryOutputPath(cli.getQueryHdfsPath()); - LOG.info("Query temp location set to " + cli.getQueryHdfsPath()); + LOG.info("Query temp location set to {}", cli.getQueryHdfsPath()); Server server = new Server(cli.getPort()); ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS); context.setContextPath("/"); http://git-wip-us.apache.org/repos/asf/metron/blob/ec959d20/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredBolt.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredBolt.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredBolt.java index 9f78b4b..a97091a 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredBolt.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredBolt.java @@ -17,10 +17,9 @@ */ package org.apache.metron.common.bolt; -import org.apache.metron.common.configuration.ConfigurationsUtils; -import org.apache.storm.task.OutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.base.BaseRichBolt; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.Map; import org.apache.curator.RetryPolicy; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; @@ -28,17 +27,19 @@ import org.apache.curator.framework.recipes.cache.TreeCache; import org.apache.curator.framework.recipes.cache.TreeCacheEvent; import org.apache.curator.framework.recipes.cache.TreeCacheListener; import org.apache.curator.retry.ExponentialBackoffRetry; -import org.apache.log4j.Logger; import org.apache.metron.common.Constants; import org.apache.metron.common.configuration.ConfigurationType; import org.apache.metron.common.configuration.Configurations; - -import java.io.IOException; -import java.util.Map; +import org.apache.metron.common.configuration.ConfigurationsUtils; +import org.apache.storm.task.OutputCollector; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.topology.base.BaseRichBolt; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public abstract class ConfiguredBolt<CONFIG_T extends Configurations> extends BaseRichBolt { - private static final Logger LOG = Logger.getLogger(ConfiguredBolt.class); + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private String zookeeperUrl; http://git-wip-us.apache.org/repos/asf/metron/blob/ec959d20/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredEnrichmentBolt.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredEnrichmentBolt.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredEnrichmentBolt.java index 0f24cd9..9c3ee97 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredEnrichmentBolt.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredEnrichmentBolt.java @@ -17,16 +17,18 @@ */ package org.apache.metron.common.bolt; -import org.apache.log4j.Logger; +import java.io.IOException; +import java.lang.invoke.MethodHandles; import org.apache.metron.common.configuration.ConfigurationType; import org.apache.metron.common.configuration.ConfigurationsUtils; import org.apache.metron.common.configuration.EnrichmentConfigurations; - -import java.io.IOException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public abstract class ConfiguredEnrichmentBolt extends ConfiguredBolt<EnrichmentConfigurations> { - private static final Logger LOG = Logger.getLogger(ConfiguredEnrichmentBolt.class); + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + public ConfiguredEnrichmentBolt(String zookeeperUrl) { super(zookeeperUrl); http://git-wip-us.apache.org/repos/asf/metron/blob/ec959d20/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredIndexingBolt.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredIndexingBolt.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredIndexingBolt.java index 966739a..cddcada 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredIndexingBolt.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredIndexingBolt.java @@ -17,16 +17,16 @@ */ package org.apache.metron.common.bolt; -import org.apache.log4j.Logger; +import java.io.IOException; +import java.lang.invoke.MethodHandles; import org.apache.metron.common.configuration.ConfigurationType; import org.apache.metron.common.configuration.ConfigurationsUtils; import org.apache.metron.common.configuration.IndexingConfigurations; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import java.io.IOException; - -public abstract class ConfiguredIndexingBolt extends ConfiguredBolt<IndexingConfigurations>{ - - private static final Logger LOG = Logger.getLogger(ConfiguredIndexingBolt.class); +public abstract class ConfiguredIndexingBolt extends ConfiguredBolt<IndexingConfigurations> { + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); public ConfiguredIndexingBolt(String zookeeperUrl) { super(zookeeperUrl); http://git-wip-us.apache.org/repos/asf/metron/blob/ec959d20/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredParserBolt.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredParserBolt.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredParserBolt.java index 15d573f..99313fa 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredParserBolt.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredParserBolt.java @@ -17,19 +17,18 @@ */ package org.apache.metron.common.bolt; -import org.apache.log4j.Logger; -import org.apache.metron.common.Constants; +import java.io.IOException; +import java.lang.invoke.MethodHandles; import org.apache.metron.common.configuration.ConfigurationType; import org.apache.metron.common.configuration.ConfigurationsUtils; import org.apache.metron.common.configuration.ParserConfigurations; import org.apache.metron.common.configuration.SensorParserConfig; - -import java.io.IOException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public abstract class ConfiguredParserBolt extends ConfiguredBolt<ParserConfigurations> { - private static final Logger LOG = Logger.getLogger(ConfiguredParserBolt.class); - + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); protected final ParserConfigurations configurations = new ParserConfigurations(); private String sensorType; http://git-wip-us.apache.org/repos/asf/metron/blob/ec959d20/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredProfilerBolt.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredProfilerBolt.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredProfilerBolt.java index c934e6d..22ff3a9 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredProfilerBolt.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredProfilerBolt.java @@ -17,24 +17,25 @@ */ package org.apache.metron.common.bolt; +import static org.apache.metron.common.configuration.ConfigurationType.PROFILER; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.lang.invoke.MethodHandles; import org.apache.curator.framework.CuratorFramework; -import org.apache.log4j.Logger; import org.apache.metron.common.configuration.ConfigurationType; import org.apache.metron.common.configuration.profiler.ProfilerConfig; import org.apache.metron.common.configuration.profiler.ProfilerConfigurations; import org.apache.metron.common.utils.JSONUtils; - -import java.io.ByteArrayInputStream; -import java.io.IOException; - -import static org.apache.metron.common.configuration.ConfigurationType.PROFILER; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * A bolt used in the Profiler topology that is configured with values stored in Zookeeper. */ public abstract class ConfiguredProfilerBolt extends ConfiguredBolt<ProfilerConfigurations> { - private static final Logger LOG = Logger.getLogger(ConfiguredProfilerBolt.class); + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); public ConfiguredProfilerBolt(String zookeeperUrl) { super(zookeeperUrl); http://git-wip-us.apache.org/repos/asf/metron/blob/ec959d20/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/Configurations.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/Configurations.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/Configurations.java index 66ea530..2d03f30 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/Configurations.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/Configurations.java @@ -18,23 +18,23 @@ package org.apache.metron.common.configuration; import com.fasterxml.jackson.core.type.TypeReference; -import org.apache.log4j.Logger; -import org.apache.metron.common.utils.JSONUtils; - import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.io.Serializable; +import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import org.apache.metron.common.utils.JSONUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class Configurations implements Serializable { - - private static final Logger LOG = Logger.getLogger(Configurations.class); + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private List<FieldValidator> validations = new ArrayList<>(); protected ConcurrentMap<String, Object> configurations = new ConcurrentHashMap<>(); http://git-wip-us.apache.org/repos/asf/metron/blob/ec959d20/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/SensorEnrichmentUpdateConfig.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/SensorEnrichmentUpdateConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/SensorEnrichmentUpdateConfig.java index fd15d31..69aaaf4 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/SensorEnrichmentUpdateConfig.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/SensorEnrichmentUpdateConfig.java @@ -19,6 +19,12 @@ package org.apache.metron.common.configuration.enrichment; import com.google.common.base.Joiner; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; import org.apache.curator.framework.CuratorFramework; import org.apache.metron.common.Constants; import org.apache.metron.common.configuration.ConfigurationsUtils; @@ -26,11 +32,10 @@ import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; - public class SensorEnrichmentUpdateConfig { - protected static final Logger _LOG = LoggerFactory.getLogger(SensorEnrichmentUpdateConfig.class); + protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + public static class FieldList { Type type; Map<String, List<String>> fieldToEnrichmentTypes; @@ -152,7 +157,7 @@ public class SensorEnrichmentUpdateConfig { } } if(fieldToTypeMap == null || fieldMap == null) { - _LOG.debug("fieldToTypeMap is null or fieldMap is null, so skipping"); + LOG.debug("fieldToTypeMap is null or fieldMap is null, so skipping"); continue; } //Add the additional fields to the field list associated with the hbase adapter @@ -166,7 +171,7 @@ public class SensorEnrichmentUpdateConfig { } //adding only the ones that we don't already have to the field list if (additionalFields.size() > 0) { - _LOG.debug("Adding additional fields: " + Joiner.on(',').join(additionalFields)); + LOG.debug("Adding additional fields: {}", Joiner.on(',').join(additionalFields)); fieldList.addAll(additionalFields); sourceConfigsChanged.put(kv.getKey(), config); } @@ -202,8 +207,8 @@ public class SensorEnrichmentUpdateConfig { SensorEnrichmentConfig config = sourceConfigsChanged.get(key); if(config == null) { config = scHandler.readConfig(key); - if(_LOG.isDebugEnabled()) { - _LOG.debug(config.toJSON()); + if(LOG.isDebugEnabled()) { + LOG.debug(config.toJSON()); } } return config; http://git-wip-us.apache.org/repos/asf/metron/blob/ec959d20/metron-platform/metron-common/src/main/java/org/apache/metron/common/hadoop/SequenceFileIterable.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/hadoop/SequenceFileIterable.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/hadoop/SequenceFileIterable.java index a57cd35..1e4384b 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/hadoop/SequenceFileIterable.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/hadoop/SequenceFileIterable.java @@ -19,23 +19,22 @@ package org.apache.metron.common.hadoop; import com.google.common.collect.Iterators; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.SequenceFile; -import org.apache.log4j.Logger; - -import java.io.IOException; -import java.util.Iterator; -import java.util.List; -import java.util.NoSuchElementException; - -import static java.lang.String.format; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class SequenceFileIterable implements Iterable<byte[]> { - private static final Logger LOGGER = Logger.getLogger(SequenceFileIterable.class); + private static final Logger LOGGER = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private List<Path> files; private Configuration config; @@ -87,12 +86,12 @@ public class SequenceFileIterable implements Iterable<byte[]> { if (!finished && null == reader) { try { reader = new SequenceFile.Reader(config, SequenceFile.Reader.file(path)); - LOGGER.debug("Writing file: " + path.toString()); + LOGGER.debug("Writing file: {}", path.toString()); } catch (IOException e) { throw new RuntimeException("Failed to get reader", e); } } else { - LOGGER.debug(format("finished=%s, reader=%s, next=%s", finished, reader, next)); + LOGGER.debug("finished={}, reader={}, next={}", finished, reader, next); } try { //ensure hasnext is idempotent @@ -111,7 +110,7 @@ public class SequenceFileIterable implements Iterable<byte[]> { } private void close() { - LOGGER.debug("Closing file: " + path.toString()); + LOGGER.debug("Closing file: {}", path.toString()); finished = true; try { if (reader != null) { http://git-wip-us.apache.org/repos/asf/metron/blob/ec959d20/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/ErrorUtils.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/ErrorUtils.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/ErrorUtils.java index f4e3a8d..241b0fe 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/ErrorUtils.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/ErrorUtils.java @@ -17,6 +17,14 @@ */ package org.apache.metron.common.utils; +import static java.lang.String.format; + +import java.lang.invoke.MethodHandles; +import java.lang.management.ManagementFactory; +import java.lang.management.ThreadInfo; +import java.lang.management.ThreadMXBean; +import java.util.Optional; +import java.util.function.Function; import org.apache.commons.lang3.tuple.Pair; import org.apache.metron.common.Constants; import org.apache.metron.common.error.MetronError; @@ -25,16 +33,8 @@ import org.apache.storm.tuple.Values; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.lang.management.ManagementFactory; -import java.lang.management.ThreadInfo; -import java.lang.management.ThreadMXBean; -import java.util.Optional; -import java.util.function.Function; - -import static java.lang.String.format; - public class ErrorUtils { - private final static Logger LOGGER = LoggerFactory.getLogger(ErrorUtils.class); + private final static Logger LOGGER = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); public enum RuntimeErrors { ILLEGAL_ARG(t -> new IllegalArgumentException(formatReason(t), t.getRight().orElse(null))), http://git-wip-us.apache.org/repos/asf/metron/blob/ec959d20/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/SerDeUtils.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/SerDeUtils.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/SerDeUtils.java index f2921b9..4a89f97 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/SerDeUtils.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/SerDeUtils.java @@ -20,13 +20,25 @@ package org.apache.metron.common.utils; +import static com.esotericsoftware.kryo.util.Util.className; + import com.esotericsoftware.kryo.Kryo; import com.esotericsoftware.kryo.KryoException; import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; import com.esotericsoftware.kryo.util.Util; import com.esotericsoftware.reflectasm.ConstructorAccess; -import de.javakaffee.kryoserializers.*; +import de.javakaffee.kryoserializers.ArraysAsListSerializer; +import de.javakaffee.kryoserializers.CollectionsEmptyListSerializer; +import de.javakaffee.kryoserializers.CollectionsEmptyMapSerializer; +import de.javakaffee.kryoserializers.CollectionsEmptySetSerializer; +import de.javakaffee.kryoserializers.CollectionsSingletonListSerializer; +import de.javakaffee.kryoserializers.CollectionsSingletonMapSerializer; +import de.javakaffee.kryoserializers.CollectionsSingletonSetSerializer; +import de.javakaffee.kryoserializers.GregorianCalendarSerializer; +import de.javakaffee.kryoserializers.JdkProxySerializer; +import de.javakaffee.kryoserializers.SynchronizedCollectionsSerializer; +import de.javakaffee.kryoserializers.UnmodifiableCollectionsSerializer; import de.javakaffee.kryoserializers.cglib.CGLibProxySerializer; import de.javakaffee.kryoserializers.guava.ImmutableListSerializer; import de.javakaffee.kryoserializers.guava.ImmutableMapSerializer; @@ -34,30 +46,30 @@ import de.javakaffee.kryoserializers.guava.ImmutableMultimapSerializer; import de.javakaffee.kryoserializers.guava.ImmutableSetSerializer; import de.javakaffee.kryoserializers.jodatime.JodaLocalDateSerializer; import de.javakaffee.kryoserializers.jodatime.JodaLocalDateTimeSerializer; -import org.apache.commons.io.output.ByteArrayOutputStream; -import org.objenesis.instantiator.ObjectInstantiator; -import org.objenesis.strategy.InstantiatorStrategy; -import org.objenesis.strategy.StdInstantiatorStrategy; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.ByteArrayInputStream; +import java.lang.invoke.MethodHandles; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationHandler; import java.lang.reflect.Modifier; import java.time.LocalDate; import java.time.LocalDateTime; -import java.util.*; +import java.util.Arrays; +import java.util.Collections; +import java.util.GregorianCalendar; import java.util.function.Function; - -import static com.esotericsoftware.kryo.util.Util.className; +import org.apache.commons.io.output.ByteArrayOutputStream; +import org.objenesis.instantiator.ObjectInstantiator; +import org.objenesis.strategy.InstantiatorStrategy; +import org.objenesis.strategy.StdInstantiatorStrategy; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Provides basic functionality to serialize and deserialize the allowed * value types for a ProfileMeasurement. */ public class SerDeUtils { - protected static final Logger LOG = LoggerFactory.getLogger(SerDeUtils.class); + protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static ThreadLocal<Kryo> kryo = new ThreadLocal<Kryo>() { @Override protected Kryo initialValue() { @@ -229,7 +241,7 @@ public class SerDeUtils { return bos.toByteArray(); } catch(Throwable t) { - LOG.error("Unable to serialize: " + value + " because " + t.getMessage(), t); + LOG.error("Unable to serialize: {} because {}", value, t.getMessage(), t); throw new IllegalStateException("Unable to serialize " + value + " because " + t.getMessage(), t); } } @@ -249,7 +261,7 @@ public class SerDeUtils { return clazz.cast(kryo.get().readClassAndObject(input)); } catch(Throwable t) { - LOG.error("Unable to deserialize because " + t.getMessage(), t); + LOG.error("Unable to deserialize because {}", t.getMessage(), t); throw t; } } http://git-wip-us.apache.org/repos/asf/metron/blob/ec959d20/metron-platform/metron-common/src/test/java/org/apache/metron/common/system/ClockTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/system/ClockTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/system/ClockTest.java index 1efc356..d552255 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/system/ClockTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/system/ClockTest.java @@ -17,15 +17,14 @@ */ package org.apache.metron.common.system; -import org.junit.Test; -import org.mockito.Mockito; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.junit.Assert.assertThat; import java.text.SimpleDateFormat; import java.util.Date; import java.util.TimeZone; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.junit.Assert.assertThat; +import org.junit.Test; +import org.mockito.Mockito; public class ClockTest { http://git-wip-us.apache.org/repos/asf/metron/blob/ec959d20/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/DataPruner.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/DataPruner.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/DataPruner.java index 98e3b52..ff954dd 100644 --- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/DataPruner.java +++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/DataPruner.java @@ -17,17 +17,18 @@ */ package org.apache.metron.dataloads.bulk; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; +import java.lang.invoke.MethodHandles; import java.util.Calendar; import java.util.Date; import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public abstract class DataPruner { - protected static final Logger LOG = LoggerFactory.getLogger(DataPruner.class); + protected static final Logger LOG = LoggerFactory + .getLogger(MethodHandles.lookup().lookupClass()); protected long firstTimeMillis; protected long lastTimeMillis; protected String wildCard; http://git-wip-us.apache.org/repos/asf/metron/blob/ec959d20/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPruner.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPruner.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPruner.java index 7529a51..ce54345 100644 --- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPruner.java +++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPruner.java @@ -19,6 +19,13 @@ package org.apache.metron.dataloads.bulk; import com.google.common.base.Predicate; import com.google.common.collect.Iterables; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.Date; +import java.util.Iterator; import org.apache.commons.collections.IteratorUtils; import org.apache.metron.common.configuration.Configuration; import org.elasticsearch.client.AdminClient; @@ -28,25 +35,15 @@ import org.elasticsearch.common.collect.ImmutableOpenMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Arrays; -import java.util.Date; -import java.util.Iterator; - public class ElasticsearchDataPruner extends DataPruner { + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static final String defaultDateFormat = "yyyy.MM.dd.HH"; private String indexPattern; private SimpleDateFormat dateFormat; protected Client indexClient = null; protected Configuration configuration; - private static final Logger LOG = LoggerFactory.getLogger(ElasticsearchDataPruner.class); - private static final String defaultDateFormat = "yyyy.MM.dd.HH"; - - - private Predicate<String> filterWithRegex = new Predicate<String>() { @Override @@ -60,7 +57,7 @@ public class ElasticsearchDataPruner extends DataPruner { return true; } } catch (ParseException e) { - LOG.error("Unable to parse date from + " + str.substring(indexPattern.length()), e); + LOG.error("Unable to parse date from {}", str.substring(indexPattern.length()), e); } return false; @@ -89,9 +86,7 @@ public class ElasticsearchDataPruner extends DataPruner { } catch(Exception e) { - LOG.error("Unable to update configs",e); - } String dateString = configuration.getGlobalConfig().get("es.date.format").toString(); http://git-wip-us.apache.org/repos/asf/metron/blob/ec959d20/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerRunner.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerRunner.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerRunner.java index be8d9bd..5d2f0f1 100644 --- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerRunner.java +++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/ElasticsearchDataPrunerRunner.java @@ -17,7 +17,20 @@ */ package org.apache.metron.dataloads.bulk; -import org.apache.commons.cli.*; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.net.InetAddress; +import java.nio.file.Paths; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Map; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.cli.PosixParser; import org.apache.curator.RetryPolicy; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; @@ -30,16 +43,9 @@ import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.net.InetAddress; -import java.nio.file.Paths; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.Map; - public class ElasticsearchDataPrunerRunner { - private static final Logger LOG = LoggerFactory.getLogger(ElasticsearchDataPruner.class); + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); public static void main(String... argv) throws IOException, java.text.ParseException, ClassNotFoundException, InterruptedException { @@ -70,9 +76,7 @@ public class ElasticsearchDataPrunerRunner { Integer numDays = Integer.parseInt(cmd.getOptionValue("n")); String indexPrefix = cmd.getOptionValue("p"); - if(LOG.isDebugEnabled()) { - LOG.debug("Running prune with args: " + startDate + " " + numDays); - } + LOG.debug("Running prune with args: {} {}", startDate, numDays); Configuration configuration = null; @@ -107,9 +111,7 @@ public class ElasticsearchDataPrunerRunner { DataPruner pruner = new ElasticsearchDataPruner(startDate, numDays, configuration, client, indexPrefix); - LOG.info("Pruned " + pruner.prune() + " indices from " + globalConfiguration.get("es.ip") + ":" + globalConfiguration.get("es.port") + "/" + indexPrefix); - - + LOG.info("Pruned {} indices from {}:{}/{}", pruner.prune(), globalConfiguration.get("es.ip"), globalConfiguration.get("es.port"), indexPrefix); } catch (Exception e) { e.printStackTrace(); http://git-wip-us.apache.org/repos/asf/metron/blob/ec959d20/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/HDFSDataPruner.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/HDFSDataPruner.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/HDFSDataPruner.java index 915ba95..a13f279 100644 --- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/HDFSDataPruner.java +++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/bulk/HDFSDataPruner.java @@ -18,7 +18,17 @@ package org.apache.metron.dataloads.bulk; -import org.apache.commons.cli.*; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.text.SimpleDateFormat; +import java.util.Date; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.cli.PosixParser; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.FileStatus; @@ -28,16 +38,11 @@ import org.apache.hadoop.fs.PathFilter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.text.SimpleDateFormat; -import java.util.Date; - public class HDFSDataPruner extends DataPruner { - + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private Path globPath; protected FileSystem fileSystem; - protected static final Logger LOG = LoggerFactory.getLogger(HDFSDataPruner.class); HDFSDataPruner(Date startDate, Integer numDays, String fsUri, String globPath) throws IOException, StartDateException { @@ -122,14 +127,11 @@ public class HDFSDataPruner extends DataPruner { Integer numDays = Integer.parseInt(cmd.getOptionValue("n")); String globString = cmd.getOptionValue("g"); - if(LOG.isDebugEnabled()) { - LOG.debug("Running prune with args: " + startDate + " " + numDays + " " + fileSystemUri + " " + globString); - } + LOG.debug("Running prune with args: {} {} {} {}", startDate, numDays, fileSystemUri, globString); DataPruner pruner = new HDFSDataPruner(startDate, numDays, fileSystemUri, globString); - LOG.info("Pruned " + pruner.prune() + " files from " + fileSystemUri + globString); - + LOG.info("Pruned {} files from {}{}", pruner.prune(), fileSystemUri, globString); } catch (Exception e) { e.printStackTrace(); System.exit(-1); @@ -146,9 +148,7 @@ public class HDFSDataPruner extends DataPruner { for (FileStatus fileStatus : filesToDelete) { - if (LOG.isDebugEnabled()) { - LOG.debug("Deleting File: " + fileStatus.getPath()); - } + LOG.debug("Deleting File: {}", fileStatus.getPath()); fileSystem.delete(fileStatus.getPath(), false); @@ -177,17 +177,13 @@ public class HDFSDataPruner extends DataPruner { @Override public boolean accept(Path path) { try { - - if(LOG.isDebugEnabled()) { - LOG.debug("ACCEPT - working with file: " + path); - } + LOG.debug("ACCEPT - working with file: {}", path); if (pruner.fileSystem.isDirectory(path)) { return false; } } catch (IOException e) { - LOG.error("IOException", e); if (failOnError) { @@ -211,7 +207,6 @@ public class HDFSDataPruner extends DataPruner { return accept; } catch (IOException e) { - LOG.error("IOException", e); if (failOnError) { http://git-wip-us.apache.org/repos/asf/metron/blob/ec959d20/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/TransformFilterExtractorDecorator.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/TransformFilterExtractorDecorator.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/TransformFilterExtractorDecorator.java index 516c31e..2aa76b7 100644 --- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/TransformFilterExtractorDecorator.java +++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/TransformFilterExtractorDecorator.java @@ -17,10 +17,24 @@ */ package org.apache.metron.dataloads.extractor; +import static org.apache.metron.dataloads.extractor.TransformFilterExtractorDecorator.ExtractorOptions.INDICATOR; +import static org.apache.metron.dataloads.extractor.TransformFilterExtractorDecorator.ExtractorOptions.INDICATOR_FILTER; +import static org.apache.metron.dataloads.extractor.TransformFilterExtractorDecorator.ExtractorOptions.INDICATOR_TRANSFORM; +import static org.apache.metron.dataloads.extractor.TransformFilterExtractorDecorator.ExtractorOptions.VALUE_FILTER; +import static org.apache.metron.dataloads.extractor.TransformFilterExtractorDecorator.ExtractorOptions.VALUE_TRANSFORM; +import static org.apache.metron.dataloads.extractor.TransformFilterExtractorDecorator.ExtractorOptions.ZK_QUORUM; + import com.fasterxml.jackson.core.type.TypeReference; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; import org.apache.commons.lang.StringUtils; import org.apache.curator.framework.CuratorFramework; -import org.apache.log4j.Logger; import org.apache.metron.common.configuration.ConfigurationsUtils; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.MapVariableResolver; @@ -30,15 +44,11 @@ import org.apache.metron.stellar.common.StellarProcessor; import org.apache.metron.stellar.common.utils.ConversionUtils; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.enrichment.lookup.LookupKV; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.util.*; - -import static org.apache.metron.dataloads.extractor.TransformFilterExtractorDecorator.ExtractorOptions.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class TransformFilterExtractorDecorator extends ExtractorDecorator { - private static final Logger LOG = Logger.getLogger(TransformFilterExtractorDecorator.class); + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); protected enum ExtractorOptions { VALUE_TRANSFORM("value_transform"), http://git-wip-us.apache.org/repos/asf/metron/blob/ec959d20/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/flatfile/importer/MapReduceImporter.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/flatfile/importer/MapReduceImporter.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/flatfile/importer/MapReduceImporter.java index 63a84cb..401ace2 100644 --- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/flatfile/importer/MapReduceImporter.java +++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/flatfile/importer/MapReduceImporter.java @@ -17,6 +17,12 @@ */ package org.apache.metron.dataloads.nonbulk.flatfile.importer; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.EnumMap; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.client.Put; @@ -24,24 +30,19 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; import org.apache.hadoop.hbase.mapreduce.TableOutputFormat; import org.apache.hadoop.mapreduce.Job; -import org.apache.log4j.Logger; import org.apache.metron.dataloads.extractor.ExtractorHandler; import org.apache.metron.dataloads.hbase.mr.BulkLoadMapper; import org.apache.metron.dataloads.nonbulk.flatfile.LoadOptions; import org.apache.metron.enrichment.converter.EnrichmentConverter; - -import java.io.IOException; -import java.util.EnumMap; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public enum MapReduceImporter implements Importer{ INSTANCE ; - private static final Logger LOG = Logger.getLogger(MapReduceImporter.class); + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @Override public void importData(EnumMap<LoadOptions, Optional<Object>> config @@ -54,7 +55,7 @@ public enum MapReduceImporter implements Importer{ Job job = Job.getInstance(hadoopConfig); List<String> inputs = (List<String>) config.get(LoadOptions.INPUT).get(); job.setJobName("MapReduceImporter: " + inputs.stream().collect(Collectors.joining(",")) + " => " + table + ":" + cf); - LOG.info("Configuring " + job.getJobName()); + LOG.info("Configuring {}", job.getJobName()); job.setJarByClass(MapReduceImporter.class); job.setMapperClass(org.apache.metron.dataloads.hbase.mr.BulkLoadMapper.class); job.setOutputFormatClass(TableOutputFormat.class); http://git-wip-us.apache.org/repos/asf/metron/blob/ec959d20/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiHandler.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiHandler.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiHandler.java index b6994bb..8bbedf2 100644 --- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiHandler.java +++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiHandler.java @@ -18,6 +18,31 @@ package org.apache.metron.dataloads.nonbulk.taxii; +import java.io.IOException; +import java.io.StringWriter; +import java.lang.invoke.MethodHandles; +import java.net.URI; +import java.net.URL; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.GregorianCalendar; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TimerTask; +import javax.xml.XMLConstants; +import javax.xml.bind.JAXBException; +import javax.xml.datatype.DatatypeConfigurationException; +import javax.xml.datatype.DatatypeFactory; +import javax.xml.datatype.XMLGregorianCalendar; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.HTableInterface; @@ -41,7 +66,6 @@ import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; -import org.apache.log4j.Logger; import org.apache.metron.common.utils.ErrorUtils; import org.apache.metron.dataloads.extractor.Extractor; import org.apache.metron.enrichment.converter.EnrichmentConverter; @@ -49,29 +73,28 @@ import org.apache.metron.enrichment.converter.EnrichmentKey; import org.apache.metron.enrichment.converter.EnrichmentValue; import org.apache.metron.enrichment.lookup.LookupKV; import org.mitre.taxii.client.HttpClient; -import org.mitre.taxii.messages.xml11.*; +import org.mitre.taxii.messages.xml11.AnyMixedContentType; +import org.mitre.taxii.messages.xml11.CollectionInformationRequest; +import org.mitre.taxii.messages.xml11.CollectionInformationResponse; +import org.mitre.taxii.messages.xml11.CollectionRecordType; +import org.mitre.taxii.messages.xml11.ContentBlock; +import org.mitre.taxii.messages.xml11.DiscoveryRequest; +import org.mitre.taxii.messages.xml11.DiscoveryResponse; +import org.mitre.taxii.messages.xml11.MessageHelper; +import org.mitre.taxii.messages.xml11.ObjectFactory; +import org.mitre.taxii.messages.xml11.PollRequest; +import org.mitre.taxii.messages.xml11.PollResponse; +import org.mitre.taxii.messages.xml11.ServiceInstanceType; +import org.mitre.taxii.messages.xml11.ServiceTypeEnum; +import org.mitre.taxii.messages.xml11.TaxiiXml; +import org.mitre.taxii.messages.xml11.TaxiiXmlFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.w3c.dom.Document; import org.w3c.dom.Element; -import javax.xml.XMLConstants; -import javax.xml.bind.JAXBException; -import javax.xml.datatype.DatatypeConfigurationException; -import javax.xml.datatype.DatatypeFactory; -import javax.xml.datatype.XMLGregorianCalendar; -import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerException; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; -import java.io.IOException; -import java.io.StringWriter; -import java.net.URI; -import java.net.URL; -import java.text.SimpleDateFormat; -import java.util.*; - public class TaxiiHandler extends TimerTask { - private static final Logger LOG = Logger.getLogger(TaxiiHandler.class); + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static ThreadLocal<TaxiiXmlFactory> xmlFactory = new ThreadLocal<TaxiiXmlFactory>() { @Override @@ -108,7 +131,7 @@ public class TaxiiHandler extends TimerTask { , Configuration config ) throws Exception { - LOG.info("Loading configuration: " + connectionConfig); + LOG.info("Loading configuration: {}", connectionConfig); this.allowedIndicatorTypes = connectionConfig.getAllowedIndicatorTypes(); this.extractor = extractor; this.collection = connectionConfig.getCollection(); @@ -121,7 +144,7 @@ public class TaxiiHandler extends TimerTask { this.username = connectionConfig.getUsername(); this.password = connectionConfig.getPassword(); initializeClient(connectionConfig); - LOG.info("Configured, starting polling " + endpoint + " for " + collection); + LOG.info("Configured, starting polling {} for {}", endpoint, collection); } protected synchronized HTableInterface getTable(String table) throws IOException { @@ -145,7 +168,7 @@ public class TaxiiHandler extends TimerTask { return; } Date ts = new Date(); - LOG.info("Polling..." + new SimpleDateFormat().format(ts)); + LOG.info("Polling...{}", new SimpleDateFormat().format(ts)); try { inProgress = true; // Prepare the message to send. @@ -168,13 +191,13 @@ public class TaxiiHandler extends TimerTask { ErrorUtils.RuntimeErrors.ILLEGAL_STATE.throwRuntime("Unable to set the begin time due to", e); } gTime.setFractionalSecond(null); - LOG.info("Begin Time: " + gTime); + LOG.info("Begin Time: {}", gTime); request.setExclusiveBeginTimestamp(gTime); } try { PollResponse response = call(request, PollResponse.class); - LOG.info("Got Poll Response with " + response.getContentBlocks().size() + " blocks"); + LOG.info("Got Poll Response with {} blocks", response.getContentBlocks().size()); int numProcessed = 0; long avgTimeMS = 0; long timeStartedBlock = System.currentTimeMillis(); @@ -188,7 +211,7 @@ public class TaxiiHandler extends TimerTask { Element element = (Element) o; xml = getStringFromDocument(element.getOwnerDocument()); if(LOG.isDebugEnabled() && Math.random() < 0.01) { - LOG.debug("Random Stix doc: " + xml); + LOG.debug("Random Stix doc: {}", xml); } for (LookupKV<EnrichmentKey, EnrichmentValue> kv : extractor.extract(xml)) { if(allowedIndicatorTypes.isEmpty() @@ -201,14 +224,14 @@ public class TaxiiHandler extends TimerTask { Put p = converter.toPut(columnFamily, kv.getKey(), kv.getValue()); HTableInterface table = getTable(hbaseTable); table.put(p); - LOG.info("Found Threat Intel: " + kv.getKey() + " => " + kv.getValue()); + LOG.info("Found Threat Intel: {} => ", kv.getKey(), kv.getValue()); } } } avgTimeMS += System.currentTimeMillis() - timeS; } if( (numProcessed + 1) % 100 == 0) { - LOG.info("Processed " + numProcessed + " in " + (System.currentTimeMillis() - timeStartedBlock) + " ms, avg time: " + avgTimeMS / content.getContent().size()); + LOG.info("Processed {} in {} ms, avg time: {}", numProcessed, System.currentTimeMillis() - timeStartedBlock, avgTimeMS / content.getContent().size()); timeStartedBlock = System.currentTimeMillis(); avgTimeMS = 0; numProcessed = 0; @@ -258,7 +281,7 @@ public class TaxiiHandler extends TimerTask { LOG.info("Discovering endpoint"); endpoint = discoverPollingClient(config.getProxy(), endpoint, config.getUsername(), config.getPassword(), context, collection).pollEndpoint; this.endpoint = endpoint; - LOG.info("Discovered endpoint as " + endpoint); + LOG.info("Discovered endpoint as {}", endpoint); } } @@ -339,7 +362,7 @@ public class TaxiiHandler extends TimerTask { , Class<RESPONSE_T> responseClazz ) throws JAXBException, IOException { Object responseObj = taxiiClient.callTaxiiService(endpoint, request, context); - LOG.info("Request made : " + request.getClass().getCanonicalName() + " => " + responseObj.getClass().getCanonicalName() + " (expected " + responseClazz.getCanonicalName() + ")"); + LOG.info("Request made : {} => {} (expected {})", request.getClass().getCanonicalName(), responseObj.getClass().getCanonicalName(), responseClazz.getCanonicalName()); try { return responseClazz.cast(responseObj); } http://git-wip-us.apache.org/repos/asf/metron/blob/ec959d20/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java index cba2be6..39cdda9 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/writer/ElasticsearchWriter.java @@ -17,33 +17,28 @@ */ package org.apache.metron.elasticsearch.writer; -import org.apache.metron.elasticsearch.utils.ElasticsearchUtils; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.tuple.Tuple; -import com.google.common.base.Splitter; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; +import java.io.Serializable; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.Map; import org.apache.metron.common.configuration.writer.WriterConfiguration; +import org.apache.metron.common.interfaces.FieldNameConverter; import org.apache.metron.common.writer.BulkMessageWriter; import org.apache.metron.common.writer.BulkWriterResponse; -import org.apache.metron.common.interfaces.FieldNameConverter; +import org.apache.metron.elasticsearch.utils.ElasticsearchUtils; +import org.apache.storm.task.TopologyContext; +import org.apache.storm.tuple.Tuple; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.json.simple.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.Serializable; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.text.SimpleDateFormat; -import java.util.*; - public class ElasticsearchWriter implements BulkMessageWriter<JSONObject>, Serializable { private Map<String, String> optionalSettings; @@ -148,9 +143,7 @@ public class ElasticsearchWriter implements BulkMessageWriter<JSONObject>, Seria if(field.contains(".")){ - if(LOG.isDebugEnabled()){ - LOG.debug("Dotted field: " + field); - } + LOG.debug("Dotted field: {}", field); } String newkey = fieldNameConverter.convert(field);
