METRON-877 Extract core implementation and UDF support, create metron-stellar module (mattf-horton) closes apache/metron#616
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/a5b13777 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/a5b13777 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/a5b13777 Branch: refs/heads/master Commit: a5b13777a92418155da61864e98fc24cde866501 Parents: df94ed4 Author: mattf-horton <[email protected]> Authored: Sun Jul 2 15:38:14 2017 -0700 Committer: mattf <[email protected]> Committed: Sun Jul 2 15:38:14 2017 -0700 ---------------------------------------------------------------------- .gitignore | 1 + dependencies_with_url.csv | 1 + metron-analytics/metron-maas-service/pom.xml | 10 + .../metron/maas/functions/MaaSFunctions.java | 324 +++ .../metron/maas/service/MockDGAModel.java | 83 + .../service/StellarMaaSIntegrationTest.java | 181 ++ .../profiler/client/stellar/FixedLookback.java | 8 +- .../profiler/client/stellar/GetProfile.java | 8 +- .../profiler/client/stellar/ProfilerConfig.java | 2 +- .../metron/profiler/client/stellar/Util.java | 9 +- .../profiler/client/stellar/WindowLookback.java | 10 +- .../profiler/client/window/WindowProcessor.java | 10 +- .../metron/profiler/client/GetProfileTest.java | 21 +- .../client/HBaseProfilerClientTest.java | 8 +- .../client/stellar/WindowLookbackTest.java | 10 +- .../client/window/WindowProcessorTest.java | 10 +- .../apache/metron/profiler/ProfileBuilder.java | 16 +- .../stellar/DefaultStellarExecutor.java | 163 -- .../profiler/stellar/StellarExecutor.java | 81 - .../metron/profiler/ProfileBuilderTest.java | 3 +- .../stellar/DefaultStellarExecutorTest.java | 192 -- metron-analytics/metron-profiler/pom.xml | 2 +- .../profiler/bolt/ProfileBuilderBolt.java | 2 +- .../profiler/bolt/ProfileSplitterBolt.java | 17 +- .../profiler/bolt/ProfileBuilderBoltTest.java | 3 +- .../profiler/bolt/ProfileHBaseMapperTest.java | 4 - .../profiler/bolt/ProfileSplitterBoltTest.java | 11 +- metron-analytics/metron-statistics/pom.xml | 6 +- .../apache/metron/statistics/BinFunctions.java | 71 + .../apache/metron/statistics/MathFunctions.java | 104 - .../statistics/StellarStatisticsFunctions.java | 11 +- .../approximation/HyperLogLogPlusFunctions.java | 6 +- .../MedianAbsoluteDeviationFunctions.java | 8 +- .../metron/statistics/BinFunctionsTest.java | 53 + .../metron/statistics/MathFunctionsTest.java | 61 - .../StellarStatisticsFunctionsTest.java | 7 +- ...HyperLogLogPlusFunctionsIntegrationTest.java | 2 +- .../outlier/MedianAbsoluteDeviationTest.java | 6 +- metron-deployment/README.md | 1 + .../rest/service/impl/StellarServiceImpl.java | 11 +- .../metron-common/3rdPartyStellar.md | 160 - metron-platform/metron-common/README.md | 987 +------ metron-platform/metron-common/pom.xml | 19 +- .../metron/common/stellar/generated/Stellar.g4 | 273 -- .../metron-common/src/main/java/Stellar.tokens | 67 - .../src/main/java/StellarLexer.tokens | 67 - .../metron/common/aggregator/Aggregators.java | 2 +- .../metron/common/bolt/ConfiguredBolt.java | 1 - .../common/bolt/ConfiguredEnrichmentBolt.java | 2 - .../common/configuration/Configurations.java | 1 - .../configuration/ConfigurationsUtils.java | 5 +- .../common/configuration/FieldTransformer.java | 2 +- .../common/configuration/FieldValidator.java | 2 +- .../configuration/IndexingConfigurations.java | 2 +- .../configuration/SensorParserConfig.java | 1 - .../enrichment/handler/StellarConfig.java | 5 +- .../threatintel/ThreatTriageConfig.java | 4 +- .../writer/ParserWriterConfiguration.java | 2 +- .../metron/common/dsl/BaseStellarFunction.java | 42 - .../org/apache/metron/common/dsl/Context.java | 115 - .../apache/metron/common/dsl/ErrorListener.java | 51 - .../metron/common/dsl/FunctionMarker.java | 21 - .../apache/metron/common/dsl/GrammarUtils.java | 133 - .../metron/common/dsl/MapVariableResolver.java | 49 - .../metron/common/dsl/ParseException.java | 28 - .../common/dsl/Predicate2StellarFunction.java | 33 - .../org/apache/metron/common/dsl/Stellar.java | 35 - .../metron/common/dsl/StellarFunction.java | 26 - .../metron/common/dsl/StellarFunctionInfo.java | 117 - .../metron/common/dsl/StellarFunctions.java | 33 - .../org/apache/metron/common/dsl/Token.java | 72 - .../metron/common/dsl/VariableResolver.java | 23 - .../dsl/functions/ConversionFunctions.java | 83 - .../dsl/functions/DataStructureFunctions.java | 220 -- .../common/dsl/functions/DateFunctions.java | 368 --- .../dsl/functions/FunctionalFunctions.java | 121 - .../common/dsl/functions/MaaSFunctions.java | 325 --- .../common/dsl/functions/MapFunctions.java | 85 - .../common/dsl/functions/NetworkFunctions.java | 293 -- .../common/dsl/functions/StringFunctions.java | 476 --- .../common/dsl/functions/SystemFunctions.java | 87 - .../resolver/BaseFunctionResolver.java | 202 -- .../resolver/ClasspathFunctionResolver.java | 264 -- .../functions/resolver/FunctionResolver.java | 46 - .../resolver/SimpleFunctionResolver.java | 59 - .../resolver/SingletonFunctionResolver.java | 34 - .../transformation/FieldTransformation.java | 2 +- .../IPProtocolTransformation.java | 10 +- .../transformation/RemoveTransformation.java | 6 +- .../SimpleFieldTransformation.java | 2 +- .../transformation/StellarTransformation.java | 10 +- .../field/validation/FieldValidation.java | 2 +- .../field/validation/QueryValidation.java | 8 +- .../field/validation/SimpleValidation.java | 2 +- .../validation/network/DomainValidation.java | 6 +- .../validation/network/EmailValidation.java | 5 +- .../field/validation/network/IPValidation.java | 6 +- .../field/validation/network/URLValidation.java | 4 +- .../validation/primitive/DateValidation.java | 6 +- .../validation/primitive/IntegerValidation.java | 5 +- .../validation/primitive/RegexValidation.java | 2 +- .../metron/common/message/MessageGetters.java | 2 +- .../common/stellar/BaseStellarProcessor.java | 239 -- .../apache/metron/common/stellar/BooleanOp.java | 23 - .../metron/common/stellar/FrameContext.java | 45 - .../metron/common/stellar/LambdaExpression.java | 68 - .../common/stellar/StellarAssignment.java | 135 - .../metron/common/stellar/StellarCompiler.java | 719 ----- .../stellar/StellarPredicateProcessor.java | 66 - .../metron/common/stellar/StellarProcessor.java | 51 - .../stellar/benchmark/Microbenchmark.java | 68 - .../benchmark/StellarMicrobenchmark.java | 268 -- .../stellar/evaluators/ArithmeticEvaluator.java | 103 - .../ComparisonExpressionEvaluator.java | 42 - ...mparisonExpressionWithOperatorEvaluator.java | 96 - .../ComparisonOperatorsEvaluator.java | 185 -- .../evaluators/DoubleLiteralEvaluator.java | 34 - .../evaluators/EqualityOperatorsEvaluator.java | 75 - .../evaluators/FloatLiteralEvaluator.java | 34 - .../stellar/evaluators/IntLiteralEvaluator.java | 34 - .../evaluators/LongLiteralEvaluator.java | 42 - .../stellar/evaluators/NumberEvaluator.java | 27 - .../evaluators/NumberLiteralEvaluator.java | 71 - .../stellar/generated/StellarBaseListener.java | 780 ----- .../common/stellar/generated/StellarLexer.java | 306 -- .../stellar/generated/StellarListener.java | 719 ----- .../common/stellar/generated/StellarParser.java | 2739 ------------------ .../common/stellar/shell/PausableInput.java | 372 --- .../common/stellar/shell/StellarExecutor.java | 322 -- .../common/stellar/shell/StellarShell.java | 443 --- .../apache/metron/common/utils/BloomFilter.java | 86 - .../metron/common/utils/ConversionUtils.java | 58 - .../apache/metron/common/utils/KafkaUtils.java | 2 - .../metron/common/utils/VFSClassloaderUtil.java | 168 -- .../metron-common/src/main/scripts/stellar | 2 +- .../dsl/functions/ConversionFunctionsTest.java | 44 - .../functions/DataStructureFunctionsTest.java | 121 - .../dsl/functions/FunctionalFunctionsTest.java | 291 -- .../dsl/functions/StringFunctionsTest.java | 431 --- .../dsl/functions/SystemFunctionsTest.java | 78 - ...lasspathFunctionResolverIntegrationTest.java | 87 - .../resolver/ClasspathFunctionResolverTest.java | 135 - .../resolver/SimpleFunctionResolverTest.java | 122 - .../transformation/FieldTransformationTest.java | 2 +- .../RemoveTransformationTest.java | 2 +- .../StellarTransformationTest.java | 2 +- .../field/validation/BaseValidationTest.java | 2 +- .../network/DomainValidationTest.java | 2 +- .../validation/network/EmailValidationTest.java | 2 +- .../validation/network/IPValidationTest.java | 2 +- .../validation/network/URLValidationTest.java | 2 +- .../primitive/DateValidationTest.java | 2 +- .../primitive/IntegerValidationTest.java | 2 +- .../stellar/BaseStellarProcessorTest.java | 116 - .../metron/common/stellar/BloomFilterTest.java | 96 - .../common/stellar/DateFunctionsTest.java | 231 -- .../common/stellar/StellarArithmeticTest.java | 332 --- .../common/stellar/StellarAssignmentTest.java | 63 - ...larComparisonExpressionWithOperatorTest.java | 293 -- .../common/stellar/StellarInterpreterTest.java | 172 -- .../stellar/StellarPredicateProcessorTest.java | 36 - .../metron/common/stellar/StellarTest.java | 735 ----- .../evaluators/ArithmeticEvaluatorTest.java | 412 --- ...isonExpressionWithOperatorEvaluatorTest.java | 126 - .../ComparisonOperatorsEvaluatorTest.java | 446 --- .../evaluators/DoubleLiteralEvaluatorTest.java | 70 - .../EqualityOperatorsEvaluatorTest.java | 186 -- .../evaluators/FloatLiteralEvaluatorTest.java | 74 - .../evaluators/IntLiteralEvaluatorTest.java | 74 - .../evaluators/LongLiteralEvaluatorTest.java | 75 - .../evaluators/NumberLiteralEvaluatorTest.java | 107 - .../common/stellar/maas/MockDGAModel.java | 83 - .../maas/StellarMaaSIntegrationTest.java | 181 -- .../stellar/network/NetworkFunctionsTest.java | 156 - .../common/utils/ConversionUtilsTest.java | 34 - .../metron/common/utils/SerDeUtilsTest.java | 1 + .../common/utils/StellarProcessorUtils.java | 159 - .../TransformFilterExtractorDecorator.java | 12 +- .../dataloads/nonbulk/flatfile/LoadOptions.java | 2 +- .../adapters/stellar/StellarAdapter.java | 12 +- .../apache/metron/enrichment/bolt/CacheKey.java | 2 +- .../enrichment/bolt/GenericEnrichmentBolt.java | 5 +- .../enrichment/bolt/ThreatIntelJoinBolt.java | 8 +- .../accesstracker/BloomAccessTracker.java | 2 +- .../PersistentBloomTrackerCreator.java | 2 +- .../stellar/GeoEnrichmentFunctions.java | 8 +- .../stellar/SimpleHBaseEnrichmentFunctions.java | 9 +- .../writer/SimpleHbaseEnrichmentWriter.java | 2 +- .../triage/ThreatTriageProcessor.java | 14 +- .../adapters/geo/GeoLiteDatabaseTest.java | 2 +- .../adapters/stellar/StellarAdapterTest.java | 9 +- .../integration/EnrichmentIntegrationTest.java | 4 - .../stellar/DocumentationGenerator.java | 4 +- .../stellar/GeoEnrichmentFunctionsTest.java | 8 +- .../SimpleHBaseEnrichmentFunctionsTest.java | 7 +- .../threatintel/triage/ThreatTriageTest.java | 4 +- metron-platform/metron-integration-test/pom.xml | 5 + .../StellarClasspathFunctionResolver.java | 83 + metron-platform/metron-management/pom.xml | 12 +- .../management/ConfigurationFunctions.java | 13 +- .../management/EnrichmentConfigFunctions.java | 12 +- .../metron/management/FileSystemFunctions.java | 13 +- .../apache/metron/management/GrokFunctions.java | 8 +- .../management/IndexingConfigFunctions.java | 10 +- .../metron/management/KafkaFunctions.java | 16 +- .../management/ParserConfigFunctions.java | 11 +- .../metron/management/ShellFunctions.java | 15 +- .../management/ThreatTriageFunctions.java | 14 +- .../management/ConfigurationFunctionsTest.java | 6 +- .../EnrichmentConfigFunctionsTest.java | 11 +- .../management/FileSystemFunctionsTest.java | 2 +- .../metron/management/GrokFunctionsTest.java | 4 +- .../management/IndexingConfigFunctionsTest.java | 9 +- .../KafkaFunctionsIntegrationTest.java | 8 +- .../management/ParserConfigFunctionsTest.java | 6 +- .../metron/management/ShellFunctionsTest.java | 8 +- .../management/ThreatTriageFunctionsTest.java | 9 +- .../apache/metron/parsers/bolt/ParserBolt.java | 7 +- .../apache/metron/parsers/csv/CSVParser.java | 2 +- .../parsers/filters/BroMessageFilter.java | 4 +- .../metron/parsers/filters/StellarFilter.java | 9 +- .../parsers/interfaces/MessageFilter.java | 5 +- .../parsers/topology/ParserTopologyCLI.java | 2 - .../org/apache/metron/filters/FiltersTest.java | 2 +- .../metron/parsers/bolt/ParserBoltTest.java | 6 +- .../integration/WriterBoltIntegrationTest.java | 3 +- metron-platform/metron-pcap/pom.xml | 7 - .../metron/pcap/filter/PcapFieldResolver.java | 4 +- .../pcap/filter/fixed/FixedPcapFilter.java | 6 +- .../pcap/filter/query/QueryPcapFilter.java | 13 +- .../pcap/pattern/ByteArrayMatcherFunction.java | 8 +- .../pcap/pattern/ByteArrayMatchingUtilTest.java | 5 +- metron-platform/metron-storm-kafka/pom.xml | 5 + .../storm/kafka/flux/SpoutConfiguration.java | 3 +- .../org/apache/metron/writer/NoopWriter.java | 2 +- .../apache/metron/writer/hdfs/HdfsWriter.java | 13 +- .../apache/metron/writer/kafka/KafkaWriter.java | 2 +- metron-stellar/pom.xml | 88 + .../stellar-common/3rdPartyStellar.md | 160 + metron-stellar/stellar-common/README.md | 1012 +++++++ metron-stellar/stellar-common/pom.xml | 340 +++ .../metron/stellar/common/generated/Stellar.g4 | 273 ++ .../src/main/assembly/assembly.xml | 42 + .../stellar-common/src/main/java/Stellar.tokens | 67 + .../src/main/java/StellarLexer.tokens | 67 + .../stellar/common/BaseStellarProcessor.java | 236 ++ .../apache/metron/stellar/common/BooleanOp.java | 23 + .../apache/metron/stellar/common/Constants.java | 116 + .../common/DefaultStellarStatefulExecutor.java | 162 ++ .../metron/stellar/common/FrameContext.java | 45 + .../metron/stellar/common/LambdaExpression.java | 68 + .../stellar/common/StellarAssignment.java | 135 + .../metron/stellar/common/StellarCompiler.java | 719 +++++ .../common/StellarPredicateProcessor.java | 66 + .../metron/stellar/common/StellarProcessor.java | 49 + .../stellar/common/StellarStatefulExecutor.java | 81 + .../common/benchmark/Microbenchmark.java | 67 + .../common/benchmark/StellarMicrobenchmark.java | 268 ++ .../common/configuration/ConfigurationType.java | 72 + .../configuration/ConfigurationsUtils.java | 208 ++ .../common/evaluators/ArithmeticEvaluator.java | 103 + .../ComparisonExpressionEvaluator.java | 42 + ...mparisonExpressionWithOperatorEvaluator.java | 96 + .../ComparisonOperatorsEvaluator.java | 185 ++ .../evaluators/DoubleLiteralEvaluator.java | 34 + .../evaluators/EqualityOperatorsEvaluator.java | 75 + .../evaluators/FloatLiteralEvaluator.java | 34 + .../common/evaluators/IntLiteralEvaluator.java | 34 + .../common/evaluators/LongLiteralEvaluator.java | 42 + .../common/evaluators/NumberEvaluator.java | 27 + .../evaluators/NumberLiteralEvaluator.java | 71 + .../common/generated/StellarBaseListener.java | 780 +++++ .../stellar/common/generated/StellarLexer.java | 306 ++ .../common/generated/StellarListener.java | 719 +++++ .../stellar/common/generated/StellarParser.java | 2739 ++++++++++++++++++ .../stellar/common/shell/PausableInput.java | 372 +++ .../stellar/common/shell/StellarExecutor.java | 322 ++ .../stellar/common/shell/StellarShell.java | 440 +++ .../metron/stellar/common/system/Clock.java | 37 + .../stellar/common/system/Environment.java | 27 + .../stellar/common/utils/BloomFilter.java | 86 + .../stellar/common/utils/ConversionUtils.java | 58 + .../metron/stellar/common/utils/JSONUtils.java | 86 + .../metron/stellar/common/utils/SerDeUtils.java | 256 ++ .../common/utils/StellarProcessorUtils.java | 163 ++ .../common/utils/VFSClassloaderUtil.java | 168 ++ .../stellar/common/utils/cli/OptionHandler.java | 31 + .../metron/stellar/dsl/BaseStellarFunction.java | 42 + .../org/apache/metron/stellar/dsl/Context.java | 113 + .../metron/stellar/dsl/ErrorListener.java | 50 + .../metron/stellar/dsl/FunctionMarker.java | 21 + .../apache/metron/stellar/dsl/GrammarUtils.java | 133 + .../metron/stellar/dsl/MapVariableResolver.java | 49 + .../metron/stellar/dsl/ParseException.java | 28 + .../stellar/dsl/Predicate2StellarFunction.java | 33 + .../org/apache/metron/stellar/dsl/Stellar.java | 35 + .../metron/stellar/dsl/StellarFunction.java | 26 + .../metron/stellar/dsl/StellarFunctionInfo.java | 116 + .../metron/stellar/dsl/StellarFunctions.java | 33 + .../org/apache/metron/stellar/dsl/Token.java | 72 + .../metron/stellar/dsl/VariableResolver.java | 23 + .../dsl/functions/ConversionFunctions.java | 83 + .../dsl/functions/DataStructureFunctions.java | 220 ++ .../stellar/dsl/functions/DateFunctions.java | 368 +++ .../dsl/functions/FunctionalFunctions.java | 121 + .../stellar/dsl/functions/MapFunctions.java | 84 + .../stellar/dsl/functions/MathFunctions.java | 63 + .../stellar/dsl/functions/NetworkFunctions.java | 293 ++ .../stellar/dsl/functions/StringFunctions.java | 476 +++ .../stellar/dsl/functions/SystemFunctions.java | 87 + .../resolver/BaseFunctionResolver.java | 202 ++ .../resolver/ClasspathFunctionResolver.java | 262 ++ .../functions/resolver/FunctionResolver.java | 46 + .../resolver/SimpleFunctionResolver.java | 59 + .../resolver/SingletonFunctionResolver.java | 34 + .../src/main/resources/META-INF/LICENSE | 605 ++++ .../src/main/resources/META-INF/NOTICE | 28 + .../stellar-common/src/main/scripts/stellar | 34 + .../classpath-resources/custom-1.0-SNAPSHOT.jar | Bin 0 -> 3570 bytes .../common/BaseStellarProcessorTest.java | 116 + .../DefaultStellarStatefulExecutorTest.java | 189 ++ .../stellar/common/StellarArithmeticTest.java | 332 +++ .../stellar/common/StellarAssignmentTest.java | 64 + ...larComparisonExpressionWithOperatorTest.java | 293 ++ .../stellar/common/StellarInterpreterTest.java | 169 ++ .../common/StellarPredicateProcessorTest.java | 36 + .../evaluators/ArithmeticEvaluatorTest.java | 412 +++ ...isonExpressionWithOperatorEvaluatorTest.java | 126 + .../ComparisonOperatorsEvaluatorTest.java | 446 +++ .../evaluators/DoubleLiteralEvaluatorTest.java | 70 + .../EqualityOperatorsEvaluatorTest.java | 186 ++ .../evaluators/FloatLiteralEvaluatorTest.java | 74 + .../evaluators/IntLiteralEvaluatorTest.java | 74 + .../evaluators/LongLiteralEvaluatorTest.java | 75 + .../evaluators/NumberLiteralEvaluatorTest.java | 107 + .../common/network/NetworkFunctionsTest.java | 156 + .../metron/stellar/common/system/ClockTest.java | 54 + .../stellar/common/utils/BloomFilterTest.java | 95 + .../common/utils/ConversionUtilsTest.java | 35 + .../stellar/common/utils/JSONUtilsTest.java | 102 + .../stellar/common/utils/SerDeUtilsTest.java | 216 ++ .../stellar/common/utils/UnitTestHelper.java | 244 ++ .../stellar/dsl/functions/BasicStellarTest.java | 739 +++++ .../dsl/functions/ConversionFunctionsTest.java | 44 + .../functions/DataStructureFunctionsTest.java | 121 + .../dsl/functions/DateFunctionsTest.java | 232 ++ .../dsl/functions/FunctionalFunctionsTest.java | 291 ++ .../dsl/functions/MathFunctionsTest.java | 47 + .../dsl/functions/StringFunctionsTest.java | 430 +++ .../dsl/functions/SystemFunctionsTest.java | 78 + .../resolver/ClasspathFunctionResolverTest.java | 139 + .../resolver/SimpleFunctionResolverTest.java | 122 + .../src/test/resources/config/global.json | 3 + .../src/test/resources/log4j.properties | 24 + pom.xml | 15 +- site-book/pom.xml | 3 +- 356 files changed, 22640 insertions(+), 20021 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore index 6789e7a..d505c9c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *~ target/ +*/target/ *dependency-reduced-pom.xml .idea *.iml http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/dependencies_with_url.csv ---------------------------------------------------------------------- diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv index 3752366..7c3bf1f 100644 --- a/dependencies_with_url.csv +++ b/dependencies_with_url.csv @@ -19,6 +19,7 @@ com.esotericsoftware.minlog:minlog:jar:1.3.0:compile,New BSD License,http://code com.esotericsoftware:minlog:jar:1.3.0:compile,New BSD License,http://code.google.com/p/minlog/ com.esotericsoftware:reflectasm:jar:1.10.1:compile,New BSD License,http://code.google.com/p/minlog/ com.google.protobuf:protobuf-java:jar:2.5.0:compile,New BSD license,http://code.google.com/p/protobuf +com.google.protobuf:protobuf-java:jar:2.6.1:compile,New BSD license,http://code.google.com/p/protobuf com.jcraft:jsch:jar:0.1.42:compile,BSD,http://www.jcraft.com/jsch/ com.maxmind.db:maxmind-db:jar:1.2.1:compile,CC-BY-SA 3.0,https://github.com/maxmind/MaxMind-DB com.maxmind.geoip2:geoip2:jar:2.8.0:compile,Apache v2,https://github.com/maxmind/GeoIP2-java http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-maas-service/pom.xml ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-maas-service/pom.xml b/metron-analytics/metron-maas-service/pom.xml index 32a9e5d..0f8abb1 100644 --- a/metron-analytics/metron-maas-service/pom.xml +++ b/metron-analytics/metron-maas-service/pom.xml @@ -38,6 +38,16 @@ <version>${project.parent.version}</version> </dependency> <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>metron-common</artifactId> + <version>${project.parent.version}</version> + </dependency> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>stellar-common</artifactId> + <version>${project.parent.version}</version> + </dependency> + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-server-common</artifactId> <version>${hadoop.version}</version> http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java new file mode 100644 index 0000000..96d43c2 --- /dev/null +++ b/metron-analytics/metron-maas-service/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java @@ -0,0 +1,324 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.maas.functions; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import org.apache.curator.framework.CuratorFramework; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.ParseException; +import org.apache.metron.stellar.dsl.Stellar; +import org.apache.metron.stellar.dsl.StellarFunction; +import org.apache.metron.common.utils.JSONUtils; +import org.apache.metron.maas.config.Endpoint; +import org.apache.metron.maas.config.MaaSConfig; +import org.apache.metron.maas.config.ModelEndpoint; +import org.apache.metron.maas.discovery.ServiceDiscoverer; +import org.apache.metron.maas.util.ConfigUtil; +import org.apache.metron.maas.util.RESTUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.MalformedURLException; +import java.net.URL; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +public class MaaSFunctions { + protected static final Logger LOG = LoggerFactory.getLogger(MaaSFunctions.class); + private static class ModelCacheKey { + String name; + String version; + String method; + Map<String, String> args; + public ModelCacheKey(String name, String version, String method, Map<String, String> args) { + this.name = name; + this.version = version; + this.method = method; + this.args = args; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + ModelCacheKey that = (ModelCacheKey) o; + + if (name != null ? !name.equals(that.name) : that.name != null) return false; + if (version != null ? !version.equals(that.version) : that.version != null) return false; + if (method != null ? !method.equals(that.method) : that.method != null) return false; + return args != null ? args.equals(that.args) : that.args == null; + + } + + @Override + public int hashCode() { + int result = name != null ? name.hashCode() : 0; + result = 31 * result + (version != null ? version.hashCode() : 0); + result = 31 * result + (method != null ? method.hashCode() : 0); + result = 31 * result + (args != null ? args.hashCode() : 0); + return result; + } + } + + @Stellar(name="MODEL_APPLY" + , namespace="MAAS" + , description = "Returns the output of a model deployed via Model as a Service. NOTE: Results are cached locally for 10 minutes." + , params = { "endpoint - A map containing the name, version, and url for the REST endpoint" + , "function - The optional endpoint path; default is 'apply'" + , "model_args - A Dictionary of arguments for the model (these become request params)" + } + , returns = "The output of the model deployed as a REST endpoint in Map form. Assumes REST endpoint returns a JSON Map." + ) + public static class ModelApply implements StellarFunction { + private boolean isInitialized = false; + private ServiceDiscoverer discoverer; + private Cache<ModelCacheKey, Map<String, Object> > resultCache; + public ModelApply() { + resultCache = CacheBuilder.newBuilder() + .concurrencyLevel(4) + .weakKeys() + .maximumSize(100000) + .expireAfterWrite(10, TimeUnit.MINUTES) + .build(); + } + + @Override + public Object apply(List<Object> args, Context context) throws ParseException { + if(args.size() < 2) { + throw new ParseException("Unable to execute model_apply. " + + "Expected arguments: endpoint_map:map, " + + " [endpoint method:string], model_args:map" + ); + } + if(!isInitialized) { + return null; + } + int i = 0; + if(args.size() == 0) { + return null; + } + Object endpointObj = args.get(i++); + Map endpoint = null; + String modelName; + String modelVersion; + String modelUrl; + if(endpointObj instanceof Map) { + endpoint = (Map)endpointObj; + modelName = endpoint.get("name") + ""; + modelVersion = endpoint.get("version") + ""; + modelUrl = endpoint.get("url") + ""; + } + else { + return null; + } + String modelFunction = "apply"; + Map<String, String> modelArgs = new HashMap<>(); + if(args.get(i) instanceof String) { + String func = (String)args.get(i); + if(endpoint.containsKey("endpoint:" + func)) { + modelFunction = "" + endpoint.get("endpoint:" + func); + } + else { + modelFunction = func; + } + i++; + } + + if(args.get(i) instanceof Map) { + if(endpoint.containsKey("endpoint:apply")) { + modelFunction = "" + endpoint.get("endpoint:apply"); + } + modelArgs = (Map)args.get(i); + } + if( modelName == null + || modelVersion == null + || modelFunction == null + ) { + return null; + } + ModelCacheKey cacheKey = new ModelCacheKey(modelName, modelVersion, modelFunction, modelArgs); + Map<String, Object> ret = resultCache.getIfPresent(cacheKey); + if(ret != null) { + return ret; + } + else { + String url = modelUrl; + if (url.endsWith("/")) { + url = url.substring(0, url.length() - 1); + } + if (modelFunction.startsWith("/")) { + modelFunction = modelFunction.substring(1); + } + try { + URL u = new URL(url + "/" + modelFunction); + + String results = RESTUtil.INSTANCE.getRESTJSONResults(u, modelArgs); + ret = JSONUtils.INSTANCE.load(results, new TypeReference<Map<String, Object>>() { + }); + resultCache.put(cacheKey, ret); + return ret; + } catch (Exception e) { + LOG.error(e.getMessage(), e); + if (discoverer != null) { + try { + URL u = new URL(modelUrl); + discoverer.blacklist(u); + } catch (MalformedURLException e1) { + } + } + } + } + return null; + } + + @Override + public synchronized void initialize(Context context) { + + try { + Optional<ServiceDiscoverer> discovererOpt = (Optional) (context.getCapability(Context.Capabilities.SERVICE_DISCOVERER)); + if (discovererOpt.isPresent()) { + discoverer = discovererOpt.get(); + } + else { + Optional<Object> clientOptional = context.getCapability(Context.Capabilities.ZOOKEEPER_CLIENT); + CuratorFramework client = null; + if (clientOptional.isPresent() && clientOptional.get() instanceof CuratorFramework) { + client = (CuratorFramework) clientOptional.get(); + } else { + throw new IllegalStateException("Unable to initialize function: Cannot find zookeeper client."); + } + discoverer = createDiscoverer(client); + } + } + catch(Exception ex) { + LOG.error(ex.getMessage(), ex); + } + finally { + //We always want to set initialize to true because we don't want to keep trying to initialize over and over + isInitialized = true; + } + } + + @Override + public boolean isInitialized() { + return isInitialized; + } + } + + private static ServiceDiscoverer createDiscoverer(CuratorFramework client) throws Exception { + MaaSConfig config = ConfigUtil.INSTANCE.read(client, "/metron/maas/config", new MaaSConfig(), MaaSConfig.class); + ServiceDiscoverer discoverer = new ServiceDiscoverer(client, config.getServiceRoot()); + discoverer.start(); + return discoverer; + } + + @Stellar(name="GET_ENDPOINT" + , namespace="MAAS" + , description="Inspects ZooKeeper and returns a map containing the name, version and url for the model referred to by the input parameters." + , params = { + "model_name - The name of the model" + ,"model_version - The optional version of the model. If the model version is not specified, the most current version is used." + } + , returns = "A map containing the name, version, and url for the REST endpoint (fields named name, version and url). " + + "Note that the output of this function is suitable for input into the first argument of MAAS_MODEL_APPLY." + ) + public static class GetEndpoint implements StellarFunction { + ServiceDiscoverer discoverer; + private boolean isInitialized = false; + private boolean isValidState = false; + + @Override + public Object apply(List<Object> args, Context context) throws ParseException { + if(!isValidState) { + LOG.error("Invalid state: Unable to find ServiceDiscoverer service."); + return null; + } + String modelName = null; + String modelVersion = null; + if(args.size() >= 1) { + modelName = args.get(0).toString(); + } + if(args.size() >= 2) + { + modelVersion = args.get(1).toString(); + } + if(modelName == null) { + return null; + } + try { + ModelEndpoint ep = null; + if (modelVersion == null) { + ep = discoverer.getEndpoint(modelName); + } else { + ep = discoverer.getEndpoint(modelName, modelVersion); + } + return ep == null ? null : endpointToMap(ep.getName(), ep.getVersion(), ep.getEndpoint()); + } + catch(Exception ex) { + LOG.error("Unable to discover endpoint: " + ex.getMessage(), ex); + return null; + } + } + + public static Map<String, String> endpointToMap(String name, String version, Endpoint ep) { + Map<String, String> ret = new HashMap<>(); + ret.put("url", ep.getUrl()); + ret.put("name", name); + ret.put("version", version); + for(Map.Entry<String, String> kv : ep.getFunctions().entrySet()) { + ret.put("endpoint:" + kv.getKey(), kv.getValue()); + } + return ret; + } + + @Override + public synchronized void initialize(Context context) { + try { + Optional<Object> clientOptional = context.getCapability(Context.Capabilities.ZOOKEEPER_CLIENT); + CuratorFramework client = null; + if (clientOptional.isPresent() && clientOptional.get() instanceof CuratorFramework) { + client = (CuratorFramework) clientOptional.get(); + } else { + throw new IllegalStateException("Unable to initialize function: Cannot find zookeeper client."); + } + try { + discoverer = createDiscoverer(client); + context.addCapability(Context.Capabilities.SERVICE_DISCOVERER, () -> discoverer); + isValidState = true; + } catch (Exception e) { + LOG.error(e.getMessage(), e); + throw new IllegalStateException("Unable to initialize MAAS_GET_ENDPOINT", e); + } + } + finally { + isInitialized = true; + } + } + + @Override + public boolean isInitialized() { + return isInitialized; + } + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-maas-service/src/test/java/org/apache/metron/maas/service/MockDGAModel.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-maas-service/src/test/java/org/apache/metron/maas/service/MockDGAModel.java b/metron-analytics/metron-maas-service/src/test/java/org/apache/metron/maas/service/MockDGAModel.java new file mode 100644 index 0000000..18604b4 --- /dev/null +++ b/metron-analytics/metron-maas-service/src/test/java/org/apache/metron/maas/service/MockDGAModel.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.maas.service; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.collect.ImmutableMap; +import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpServer; +import org.apache.metron.common.utils.JSONUtils; + +import javax.ws.rs.*; +import javax.ws.rs.core.Application; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.UriBuilder; +import javax.ws.rs.ext.RuntimeDelegate; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.URI; +import java.util.*; + +@Path("/") +public class MockDGAModel { + private static HttpServer server; + private Map<String, Boolean> isMalicious = ImmutableMap.of( "badguy.com", true + ); + + @GET + @Path("/apply") + @Produces("application/json") + public Response apply(@QueryParam("host") String host ) throws JsonProcessingException { + Boolean b = isMalicious.get(host); + boolean isMalicious = b != null && b; + Map<String, Boolean> ret = new HashMap<String, Boolean>(); + ret.put("is_malicious", isMalicious ); + String resp = JSONUtils.INSTANCE.toJSON(ret, true); + return Response.ok(resp, MediaType.APPLICATION_JSON_TYPE).build(); + } + + @ApplicationPath("rs") + public static class ApplicationConfig extends Application { + private final Set<Class<?>> classes; + public ApplicationConfig() { + HashSet<Class<?>> c = new HashSet<>(); + c.add(MockDGAModel.class); + classes = Collections.unmodifiableSet(c); + } + @Override + public Set<Class<?>> getClasses() { + return classes; + } + } + + public static void start(int port) throws IOException { + // Create an HTTP server listening at port + URI uri = UriBuilder.fromUri("http://localhost/";).port(port).build(); + server = HttpServer.create(new InetSocketAddress(uri.getPort()), 0); + HttpHandler handler = RuntimeDelegate.getInstance().createEndpoint(new ApplicationConfig(), HttpHandler.class); + server.createContext(uri.getPath(), handler); + server.start(); + } + + public static void shutdown() { + if(server != null) { + server.stop(0); + } + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-maas-service/src/test/java/org/apache/metron/maas/service/StellarMaaSIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-maas-service/src/test/java/org/apache/metron/maas/service/StellarMaaSIntegrationTest.java b/metron-analytics/metron-maas-service/src/test/java/org/apache/metron/maas/service/StellarMaaSIntegrationTest.java new file mode 100644 index 0000000..4b95af5 --- /dev/null +++ b/metron-analytics/metron-maas-service/src/test/java/org/apache/metron/maas/service/StellarMaaSIntegrationTest.java @@ -0,0 +1,181 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.maas.service; + +import com.google.common.collect.ImmutableMap; +import com.sun.jersey.server.impl.application.WebApplicationImpl; +import org.apache.curator.RetryPolicy; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.ExponentialBackoffRetry; +import org.apache.curator.test.TestingServer; +import org.apache.curator.utils.CloseableUtils; +import org.apache.curator.x.discovery.ServiceInstance; +import org.apache.curator.x.discovery.ServiceInstanceBuilder; +import org.apache.curator.x.discovery.ServiceType; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.maas.config.Endpoint; +import org.apache.metron.maas.config.MaaSConfig; +import org.apache.metron.maas.config.ModelEndpoint; +import org.apache.metron.maas.discovery.ServiceDiscoverer; +import org.apache.metron.maas.util.ConfigUtil; +import org.apache.metron.test.utils.UnitTestHelper; +import org.junit.*; + +import java.net.URL; +import java.util.HashMap; +import java.util.Map; +import java.util.logging.Level; + +import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run; + +public class StellarMaaSIntegrationTest { + private static Context context; + private static TestingServer testZkServer; + private static String zookeeperUrl; + private static CuratorFramework client; + private static ServiceDiscoverer discoverer; + private static URL endpointUrl; + + @BeforeClass + public static void setup() throws Exception { + UnitTestHelper.setJavaLoggingLevel(WebApplicationImpl.class, Level.WARNING); + MockDGAModel.start(8282); + testZkServer = new TestingServer(true); + zookeeperUrl = testZkServer.getConnectString(); + RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3); + client = CuratorFrameworkFactory.newClient(zookeeperUrl, retryPolicy); + client.start(); + context = new Context.Builder() + .with(Context.Capabilities.ZOOKEEPER_CLIENT, () -> client) + .build(); + MaaSConfig config = ConfigUtil.INSTANCE.read(client, "/metron/maas/config", new MaaSConfig(), MaaSConfig.class); + discoverer = new ServiceDiscoverer(client, config.getServiceRoot()); + discoverer.start(); + endpointUrl = new URL("http://localhost:8282";); + ModelEndpoint endpoint = new ModelEndpoint(); + { + endpoint.setName("dga"); + endpoint.setContainerId("0"); + Endpoint ep = new Endpoint(); + ep.setUrl(endpointUrl.toString()); + endpoint.setEndpoint(ep); + endpoint.setVersion("1.0"); + } + ; + + ServiceInstanceBuilder<ModelEndpoint> builder = ServiceInstance.<ModelEndpoint>builder() + .address(endpointUrl.getHost()) + .id("0") + .name("dga") + .port(endpointUrl.getPort()) + .registrationTimeUTC(System.currentTimeMillis()) + .serviceType(ServiceType.STATIC) + .payload(endpoint); + final ServiceInstance<ModelEndpoint> instance = builder.build(); + discoverer.getServiceDiscovery().registerService(instance); + //wait til the endpoint is installed... + for(int i = 0;i < 10;++i) { + try { + Object o = discoverer.getEndpoint("dga"); + if(o != null) { + break; + } + } + catch(Exception e) { + + } + Thread.sleep(1000); + } + } + + @Test + public void testGetEndpointWithoutVersion() throws Exception { + String stellar = "MAAS_GET_ENDPOINT('dga')"; + Object result = run(stellar, new HashMap<>(), context); + Assert.assertTrue(result instanceof Map); + Map<String, String> resMap = (Map<String, String>)result; + Assert.assertEquals(resMap.get("url"), "http://localhost:8282";); + Assert.assertEquals(resMap.get("name"), "dga"); + Assert.assertEquals(resMap.get("version"), "1.0"); + Assert.assertEquals(resMap.get("endpoint:apply"), "apply"); + + } + + @Test + public void testGetEndpointWithVersion() throws Exception { + String stellar = "MAAS_GET_ENDPOINT('dga', '1.0')"; + Object result = run(stellar, new HashMap<>(), context); + Assert.assertTrue(result instanceof Map); + Map<String, String> resMap = (Map<String, String>)result; + Assert.assertEquals(resMap.get("url"), "http://localhost:8282";); + Assert.assertEquals(resMap.get("name"), "dga"); + Assert.assertEquals(resMap.get("version"), "1.0"); + Assert.assertEquals(resMap.get("endpoint:apply"), "apply"); + } + + @Test + public void testGetEndpointWithWrongVersion() throws Exception { + String stellar = "MAAS_GET_ENDPOINT('dga', '2.0')"; + Object result = run(stellar, new HashMap<>(), context); + Assert.assertNull(result); + } + + @Test + public void testModelApply() throws Exception { + { + String stellar = "MAP_GET('is_malicious', MAAS_MODEL_APPLY(MAAS_GET_ENDPOINT('dga'), {'host': host}))"; + Object result = run(stellar, ImmutableMap.of("host", "badguy.com"), context); + Assert.assertTrue((Boolean) result); + } + { + String stellar = "MAP_GET('is_malicious', MAAS_MODEL_APPLY(MAAS_GET_ENDPOINT('dga'), {'host': host}))"; + Object result = run(stellar, ImmutableMap.of("host", "youtube.com"), context); + Assert.assertFalse((Boolean) result); + } + { + String stellar = "MAP_GET('is_malicious', MAAS_MODEL_APPLY(MAAS_GET_ENDPOINT('dga'), 'apply', {'host': host}))"; + Object result = run(stellar, ImmutableMap.of("host", "youtube.com"), context); + Assert.assertFalse((Boolean) result); + } + + } + + @Test + public void testModelApplyNegative() { + { + String stellar = "MAP_GET('is_malicious', MAAS_MODEL_APPLY(MAAS_GET_ENDPOINT('dga', '2.0'), {'host': host}))"; + Object result = run(stellar, ImmutableMap.of("host", "youtube.com"), context); + Assert.assertNull( result); + } + } + + @AfterClass + public static void teardown() { + MockDGAModel.shutdown(); + if(discoverer != null) { + CloseableUtils.closeQuietly(discoverer); + } + if(client != null) { + CloseableUtils.closeQuietly(client); + } + if(testZkServer != null) { + CloseableUtils.closeQuietly(testZkServer); + } + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/FixedLookback.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/FixedLookback.java b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/FixedLookback.java index c4ed582..02b4a4c 100644 --- a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/FixedLookback.java +++ b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/FixedLookback.java @@ -19,10 +19,10 @@ */ package org.apache.metron.profiler.client.stellar; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.ParseException; -import org.apache.metron.common.dsl.Stellar; -import org.apache.metron.common.dsl.StellarFunction; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.ParseException; +import org.apache.metron.stellar.dsl.Stellar; +import org.apache.metron.stellar.dsl.StellarFunction; import org.apache.metron.profiler.ProfilePeriod; import java.util.List; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/GetProfile.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/GetProfile.java b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/GetProfile.java index 87232aa..04b391e 100644 --- a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/GetProfile.java +++ b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/GetProfile.java @@ -22,10 +22,10 @@ package org.apache.metron.profiler.client.stellar; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.client.HTableInterface; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.ParseException; -import org.apache.metron.common.dsl.Stellar; -import org.apache.metron.common.dsl.StellarFunction; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.ParseException; +import org.apache.metron.stellar.dsl.Stellar; +import org.apache.metron.stellar.dsl.StellarFunction; import org.apache.metron.hbase.HTableProvider; import org.apache.metron.hbase.TableProvider; import org.apache.metron.profiler.ProfilePeriod; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerConfig.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerConfig.java b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerConfig.java index f409ca8..e2ec275 100644 --- a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerConfig.java +++ b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/ProfilerConfig.java @@ -20,7 +20,7 @@ package org.apache.metron.profiler.client.stellar; -import org.apache.metron.common.utils.ConversionUtils; +import org.apache.metron.stellar.common.utils.ConversionUtils; import org.apache.metron.hbase.HTableProvider; import java.util.Map; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/Util.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/Util.java b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/Util.java index ab22967..f30af74 100644 --- a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/Util.java +++ b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/Util.java @@ -20,13 +20,12 @@ package org.apache.metron.profiler.client.stellar; import org.apache.commons.lang.StringUtils; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.ParseException; -import org.apache.metron.common.utils.ConversionUtils; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.ParseException; +import org.apache.metron.stellar.common.utils.ConversionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -34,7 +33,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import static java.lang.String.format; -import static org.apache.metron.common.dsl.Context.Capabilities.GLOBAL_CONFIG; +import static org.apache.metron.stellar.dsl.Context.Capabilities.GLOBAL_CONFIG; public class Util { private static final Logger LOG = LoggerFactory.getLogger(Util.class); http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/WindowLookback.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/WindowLookback.java b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/WindowLookback.java index c7390fa..9e420e5 100644 --- a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/WindowLookback.java +++ b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/WindowLookback.java @@ -21,11 +21,11 @@ package org.apache.metron.profiler.client.stellar; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.ParseException; -import org.apache.metron.common.dsl.Stellar; -import org.apache.metron.common.dsl.StellarFunction; -import org.apache.metron.common.utils.ConversionUtils; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.ParseException; +import org.apache.metron.stellar.dsl.Stellar; +import org.apache.metron.stellar.dsl.StellarFunction; +import org.apache.metron.stellar.common.utils.ConversionUtils; import org.apache.metron.profiler.ProfilePeriod; import org.apache.metron.profiler.client.window.Window; import org.apache.metron.profiler.client.window.WindowProcessor; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/WindowProcessor.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/WindowProcessor.java b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/WindowProcessor.java index 1ae4aaf..80dd649 100644 --- a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/WindowProcessor.java +++ b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/WindowProcessor.java @@ -21,11 +21,11 @@ package org.apache.metron.profiler.client.window; import org.antlr.v4.runtime.*; import org.antlr.v4.runtime.tree.ParseTree; -import org.apache.metron.common.dsl.ErrorListener; -import org.apache.metron.common.dsl.GrammarUtils; -import org.apache.metron.common.dsl.ParseException; -import org.apache.metron.common.dsl.Token; -import org.apache.metron.common.utils.ConversionUtils; +import org.apache.metron.stellar.dsl.ErrorListener; +import org.apache.metron.stellar.dsl.GrammarUtils; +import org.apache.metron.stellar.dsl.ParseException; +import org.apache.metron.stellar.dsl.Token; +import org.apache.metron.stellar.common.utils.ConversionUtils; import org.apache.metron.profiler.client.window.generated.WindowBaseListener; import org.apache.metron.profiler.client.window.generated.WindowLexer; import org.apache.metron.profiler.client.window.generated.WindowParser; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/GetProfileTest.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/GetProfileTest.java b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/GetProfileTest.java index 4bc3167..917a5ca 100644 --- a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/GetProfileTest.java +++ b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/GetProfileTest.java @@ -22,10 +22,9 @@ package org.apache.metron.profiler.client; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.HTableInterface; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.functions.resolver.SimpleFunctionResolver; -import org.apache.metron.common.dsl.functions.resolver.SingletonFunctionResolver; -import org.apache.metron.common.dsl.ParseException; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.functions.resolver.SimpleFunctionResolver; +import org.apache.metron.stellar.dsl.functions.resolver.SingletonFunctionResolver; import org.apache.metron.hbase.TableProvider; import org.apache.metron.profiler.ProfileMeasurement; import org.apache.metron.profiler.client.stellar.FixedLookback; @@ -34,8 +33,8 @@ import org.apache.metron.profiler.hbase.ColumnBuilder; import org.apache.metron.profiler.hbase.RowKeyBuilder; import org.apache.metron.profiler.hbase.SaltyRowKeyBuilder; import org.apache.metron.profiler.hbase.ValueOnlyColumnBuilder; -import org.apache.metron.profiler.stellar.DefaultStellarExecutor; -import org.apache.metron.profiler.stellar.StellarExecutor; +import org.apache.metron.stellar.common.DefaultStellarStatefulExecutor; +import org.apache.metron.stellar.common.StellarStatefulExecutor; import org.apache.metron.test.mock.MockHTable; import org.junit.Assert; import org.junit.Before; @@ -62,7 +61,7 @@ public class GetProfileTest { private static final int saltDivisor = 1000; private static final String tableName = "profiler"; private static final String columnFamily = "P"; - private StellarExecutor executor; + private StellarStatefulExecutor executor; private Map<String, Object> state; private ProfileWriter profileWriter; // different values of period and salt divisor, used to test config_overrides feature @@ -119,7 +118,7 @@ public class GetProfileTest { }}; // create the stellar execution environment - executor = new DefaultStellarExecutor( + executor = new DefaultStellarStatefulExecutor( new SimpleFunctionResolver() .withClass(GetProfile.class) .withClass(FixedLookback.class), @@ -140,9 +139,9 @@ public class GetProfileTest { * original context values in the PROFILE_GET config_overrides argument gets all expected results. * * @return context2 - The profiler client configuration context created by this method. - * The context2 values are also set in the configuration of the StellarExecutor + * The context2 values are also set in the configuration of the StellarStatefulExecutor * stored in the global variable 'executor'. However, there is no API for querying the - * context values from a StellarExecutor, so we output the context2 Context object itself, + * context values from a StellarStatefulExecutor, so we output the context2 Context object itself, * for validation purposes (so that its values can be validated as being significantly * different from the setup() settings). */ @@ -165,7 +164,7 @@ public class GetProfileTest { .build(); // create the stellar execution environment - executor = new DefaultStellarExecutor( + executor = new DefaultStellarStatefulExecutor( new SimpleFunctionResolver() .withClass(GetProfile.class) .withClass(FixedLookback.class), http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/HBaseProfilerClientTest.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/HBaseProfilerClientTest.java b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/HBaseProfilerClientTest.java index d3a0fe5..960e4d2 100644 --- a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/HBaseProfilerClientTest.java +++ b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/HBaseProfilerClientTest.java @@ -25,8 +25,8 @@ import org.apache.metron.profiler.hbase.ColumnBuilder; import org.apache.metron.profiler.hbase.RowKeyBuilder; import org.apache.metron.profiler.hbase.SaltyRowKeyBuilder; import org.apache.metron.profiler.hbase.ValueOnlyColumnBuilder; -import org.apache.metron.profiler.stellar.DefaultStellarExecutor; -import org.apache.metron.profiler.stellar.StellarExecutor; +import org.apache.metron.stellar.common.DefaultStellarStatefulExecutor; +import org.apache.metron.stellar.common.StellarStatefulExecutor; import org.apache.metron.test.mock.MockHTable; import org.junit.After; import org.junit.AfterClass; @@ -58,15 +58,15 @@ public class HBaseProfilerClientTest { private static final int periodsPerHour = 4; private HBaseProfilerClient client; + private StellarStatefulExecutor executor; private MockHTable table; - private StellarExecutor executor; private ProfileWriter profileWriter; @Before public void setup() throws Exception { table = new MockHTable(tableName, columnFamily); - executor = new DefaultStellarExecutor(); + executor = new DefaultStellarStatefulExecutor(); // used to write values to be read during testing RowKeyBuilder rowKeyBuilder = new SaltyRowKeyBuilder(); http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/WindowLookbackTest.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/WindowLookbackTest.java b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/WindowLookbackTest.java index aafbe5c..41d2074 100644 --- a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/WindowLookbackTest.java +++ b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/stellar/WindowLookbackTest.java @@ -21,11 +21,11 @@ package org.apache.metron.profiler.client.stellar; import com.google.common.collect.ImmutableMap; import org.apache.commons.lang3.Range; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.ParseException; -import org.apache.metron.common.dsl.functions.resolver.FunctionResolver; -import org.apache.metron.common.dsl.functions.resolver.SimpleFunctionResolver; -import org.apache.metron.common.stellar.StellarProcessor; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.ParseException; +import org.apache.metron.stellar.dsl.functions.resolver.FunctionResolver; +import org.apache.metron.stellar.dsl.functions.resolver.SimpleFunctionResolver; +import org.apache.metron.stellar.common.StellarProcessor; import org.apache.metron.profiler.ProfilePeriod; import org.apache.metron.profiler.client.window.WindowProcessor; import org.junit.Assert; http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/window/WindowProcessorTest.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/window/WindowProcessorTest.java b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/window/WindowProcessorTest.java index 79212e2..1b8d55d 100644 --- a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/window/WindowProcessorTest.java +++ b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/window/WindowProcessorTest.java @@ -293,27 +293,27 @@ public class WindowProcessorTest { } } - @Test(expected=org.apache.metron.common.dsl.ParseException.class) + @Test(expected=org.apache.metron.stellar.dsl.ParseException.class) public void testWithInvalidDaySpecifier() throws ParseException { WindowProcessor.process("30 minute window every 24 hours from 14 days ago excluding hoolidays:us"); } - @Test(expected=org.apache.metron.common.dsl.ParseException.class) + @Test(expected=org.apache.metron.stellar.dsl.ParseException.class) public void testWithInvalidTimeUnit() throws ParseException { WindowProcessor.process("30 minute window every 24 months from 14 days ago"); } - @Test(expected=org.apache.metron.common.dsl.ParseException.class) + @Test(expected=org.apache.metron.stellar.dsl.ParseException.class) public void testWithInvalidWindowUnit() throws ParseException { WindowProcessor.process("30 minuete window every 24 hours from 14 days ago"); } - @Test(expected=org.apache.metron.common.dsl.ParseException.class) + @Test(expected=org.apache.metron.stellar.dsl.ParseException.class) public void testWithInvalidTimeNumber() throws ParseException { WindowProcessor.process("30p minute window every 24 hours from 14 days ago"); } - @Test(expected=org.apache.metron.common.dsl.ParseException.class) + @Test(expected=org.apache.metron.stellar.dsl.ParseException.class) public void testInvalidDaySpecifier() throws ParseException { WindowProcessor.process("30 minute window every 14 hours from 14 days ago including date"); } http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileBuilder.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileBuilder.java b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileBuilder.java index 88d6e1b..4a9946d 100644 --- a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileBuilder.java +++ b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileBuilder.java @@ -26,14 +26,13 @@ import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; import org.apache.curator.framework.CuratorFramework; import org.apache.metron.common.configuration.profiler.ProfileConfig; -import org.apache.metron.common.configuration.profiler.ProfileResult; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.ParseException; -import org.apache.metron.common.dsl.StellarFunctions; +import org.apache.metron.stellar.common.StellarStatefulExecutor; +import org.apache.metron.stellar.dsl.Context; +import org.apache.metron.stellar.dsl.ParseException; +import org.apache.metron.stellar.dsl.StellarFunctions; import org.apache.metron.profiler.clock.Clock; import org.apache.metron.profiler.clock.WallClock; -import org.apache.metron.profiler.stellar.DefaultStellarExecutor; -import org.apache.metron.profiler.stellar.StellarExecutor; +import org.apache.metron.stellar.common.DefaultStellarStatefulExecutor; import org.json.simple.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,7 +40,6 @@ import org.slf4j.LoggerFactory; import java.io.Serializable; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -81,7 +79,7 @@ public class ProfileBuilder implements Serializable { /** * Executes Stellar code and maintains state across multiple invocations. */ - private StellarExecutor executor; + private StellarStatefulExecutor executor; /** * Has the profile been initialized? @@ -114,7 +112,7 @@ public class ProfileBuilder implements Serializable { this.entity = entity; this.clock = clock; this.periodDurationMillis = periodDurationMillis; - this.executor = new DefaultStellarExecutor(); + this.executor = new DefaultStellarStatefulExecutor(); Context context = new Context.Builder() .with(Context.Capabilities.ZOOKEEPER_CLIENT, () -> client) .with(Context.Capabilities.GLOBAL_CONFIG, () -> global) http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/stellar/DefaultStellarExecutor.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/stellar/DefaultStellarExecutor.java b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/stellar/DefaultStellarExecutor.java deleted file mode 100644 index 62bab85..0000000 --- a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/stellar/DefaultStellarExecutor.java +++ /dev/null @@ -1,163 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.metron.profiler.stellar; - -import com.google.common.collect.ImmutableMap; -import org.apache.commons.lang.ClassUtils; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.functions.resolver.FunctionResolver; -import org.apache.metron.common.dsl.MapVariableResolver; -import org.apache.metron.common.dsl.StellarFunctions; -import org.apache.metron.common.dsl.VariableResolver; -import org.apache.metron.common.stellar.StellarProcessor; -import org.apache.metron.common.utils.ConversionUtils; - -import java.io.Serializable; -import java.util.HashMap; -import java.util.Map; - -/** - * The default implementation of a StellarExecutor. - */ -public class DefaultStellarExecutor implements StellarExecutor, Serializable { - - /** - * The current state of the Stellar execution environment. - */ - private Map<String, Object> state; - - /** - * Provides additional context for initializing certain Stellar functions. For - * example, references to Zookeeper or HBase. - */ - private Context context; - - /** - * Responsible for function resolution. - */ - private FunctionResolver functionResolver; - - public DefaultStellarExecutor() { - this(StellarFunctions.FUNCTION_RESOLVER(), Context.EMPTY_CONTEXT()); - } - - public DefaultStellarExecutor(FunctionResolver functionResolver, Context context) { - clearState(); - this.context = context; - this.functionResolver = functionResolver; - } - - /** - * @param initialState Initial state loaded into the execution environment. - */ - public DefaultStellarExecutor(Map<String, Object> initialState) { - this(); - this.state = new HashMap<>(initialState); - } - - /** - * The current state of the Stellar execution environment. - */ - @Override - public Map<String, Object> getState() { - return ImmutableMap.copyOf(state); - } - - /** - * Execute an expression and assign the result to a variable. The variable is maintained - * in the context of this executor and is available to all subsequent expressions. - * - * @param variable The name of the variable to assign to. - * @param expression The expression to execute. - * @param transientState Additional state available to the expression. This most often represents - * the values available to the expression from an individual message. The state - * maps a variable name to a variable's value. - */ - @Override - public void assign(String variable, String expression, Map<String, Object> transientState) { - Object result = execute(expression, transientState); - if(result == null || variable == null) { - return; - } - state.put(variable, result); - } - - @Override - public void assign(String variable, Object value) { - if(value == null || variable == null) { - return; - } - state.put(variable, value); - } - - /** - * Execute a Stellar expression and return the result. The internal state of the executor - * is not modified. - * - * @param expression The expression to execute. - * @param state Additional state available to the expression. This most often represents - * the values available to the expression from an individual message. The state - * maps a variable name to a variable's value. - * @param clazz The expected type of the expression's result. - * @param <T> The expected type of the expression's result. - */ - @Override - public <T> T execute(String expression, Map<String, Object> state, Class<T> clazz) { - Object resultObject = execute(expression, state); - - // perform type conversion, if necessary - T result = ConversionUtils.convert(resultObject, clazz); - if (result == null) { - throw new IllegalArgumentException(String.format("Unexpected type: expected=%s, actual=%s, expression=%s", - clazz.getSimpleName(), ClassUtils.getShortClassName(resultObject,"null"), expression)); - } - - return result; - } - - @Override - public void clearState() { - this.state = new HashMap<>(); - } - - @Override - public void setContext(Context context) { - this.context = context; - } - - public void setFunctionResolver(FunctionResolver functionResolver) { - this.functionResolver = functionResolver; - } - - /** - * Execute a Stellar expression. - * - * @param expression The expression to execute. - * @param transientState Additional state available to the expression. This most often represents - * the values available to the expression from an individual message. The state - * maps a variable name to a variable's value. - */ - private Object execute(String expression, Map<String, Object> transientState) { - VariableResolver variableResolver = new MapVariableResolver(state, transientState); - StellarProcessor processor = new StellarProcessor(); - return processor.parse(expression, variableResolver, functionResolver, context); - } -} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/stellar/StellarExecutor.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/stellar/StellarExecutor.java b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/stellar/StellarExecutor.java deleted file mode 100644 index 2342eb7..0000000 --- a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/stellar/StellarExecutor.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.metron.profiler.stellar; - -import org.apache.metron.common.dsl.Context; - -import java.util.Map; - -/** - * Executes Stellar expressions and maintains state across multiple invocations. - */ -public interface StellarExecutor { - - /** - * Assign a variable a specific value. - * @param variable The variable name. - * @param value The value to assign to the variable. - */ - void assign(String variable, Object value); - - /** - * Execute an expression and assign the result to a variable. The variable is maintained - * in the context of this executor and is available to all subsequent expressions. - * - * @param variable The name of the variable to assign to. - * @param expression The expression to execute. - * @param state Additional state available to the expression. This most often represents - * the values available to the expression from an individual message. The state - * maps a variable name to a variable's value. - */ - void assign(String variable, String expression, Map<String, Object> state); - - /** - * Execute a Stellar expression and return the result. The internal state of the executor - * is not modified. - * - * @param expression The expression to execute. - * @param state Additional state available to the expression. This most often represents - * the values available to the expression from an individual message. The state - * maps a variable name to a variable's value. - * @param clazz The expected type of the expression's result. - * @param <T> The expected type of the expression's result. - */ - <T> T execute(String expression, Map<String, Object> state, Class<T> clazz); - - /** - * The current state of the Stellar execution environment. - */ - Map<String, Object> getState(); - - /** - * Removes all state from the execution environment. - */ - void clearState(); - - /** - * Sets the Context for the Stellar execution environment. This provides global data used - * to initialize Stellar functions. - * - * @param context The Stellar context. - */ - void setContext(Context context); -} http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/ProfileBuilderTest.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/ProfileBuilderTest.java b/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/ProfileBuilderTest.java index 794fde4..aa632e4 100644 --- a/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/ProfileBuilderTest.java +++ b/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/ProfileBuilderTest.java @@ -23,7 +23,6 @@ package org.apache.metron.profiler; import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.common.configuration.profiler.ProfileConfig; import org.apache.metron.common.utils.JSONUtils; -import org.apache.metron.profiler.clock.Clock; import org.apache.metron.profiler.clock.FixedClock; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; @@ -32,7 +31,7 @@ import org.junit.Test; import java.util.concurrent.TimeUnit; -import static org.apache.metron.common.utils.ConversionUtils.convert; +import static org.apache.metron.stellar.common.utils.ConversionUtils.convert; import static org.junit.Assert.assertEquals; /** http://git-wip-us.apache.org/repos/asf/metron/blob/a5b13777/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/stellar/DefaultStellarExecutorTest.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/stellar/DefaultStellarExecutorTest.java b/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/stellar/DefaultStellarExecutorTest.java deleted file mode 100644 index d569536..0000000 --- a/metron-analytics/metron-profiler-common/src/test/java/org/apache/metron/profiler/stellar/DefaultStellarExecutorTest.java +++ /dev/null @@ -1,192 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.metron.profiler.stellar; - -import org.adrianwalker.multilinestring.Multiline; -import org.apache.metron.common.dsl.Context; -import org.apache.metron.common.dsl.functions.StringFunctions; -import org.apache.metron.common.dsl.functions.resolver.ClasspathFunctionResolver; -import org.apache.metron.common.dsl.functions.resolver.FunctionResolver; -import org.apache.metron.common.dsl.functions.resolver.SimpleFunctionResolver; -import org.apache.metron.common.field.validation.primitive.IntegerValidation; -import org.apache.metron.profiler.stellar.DefaultStellarExecutor; -import org.json.simple.JSONObject; -import org.json.simple.parser.JSONParser; -import org.json.simple.parser.ParseException; -import org.junit.Before; -import org.junit.Test; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.instanceOf; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; - -/** - * Tests the DefaultStellarExecutor. - */ -@SuppressWarnings("unchecked") -// This test class passes raw JSONObject to the the executor, which gives unchecked cast warnings. -// Suppressing on the class level, given that every test is a typical example of use pattern. -public class DefaultStellarExecutorTest { - - /** - * { - * "ip_src_addr": "10.0.0.1", - * "ip_dst_addr": "10.0.0.20" - * } - */ - @Multiline - private String input; - - private JSONObject message; - private DefaultStellarExecutor executor; - - @Before - public void setup() throws ParseException { - - // parse the input message - JSONParser parser = new JSONParser(); - message = (JSONObject) parser.parse(input); - - // create the executor to test - executor = new DefaultStellarExecutor(); - executor.setContext(Context.EMPTY_CONTEXT()); - - ClasspathFunctionResolver resolver = new ClasspathFunctionResolver(); - executor.setFunctionResolver(resolver); - } - - /** - * Ensure that a value can be assigned to a variable. - */ - @Test - public void testAssign() { - executor.assign("foo", "2", message); - - // verify - Object var = executor.getState().get("foo"); - assertThat(var, instanceOf(Integer.class)); - assertThat(var, equalTo(2)); - } - - /** - * Ensure that a variable can be resolved from a message field. - */ - @Test - public void testAssignWithVariableResolution() { - executor.assign("foo", "ip_src_addr", message); - - // verify - Object var = executor.getState().get("foo"); - assertThat(var, instanceOf(String.class)); - assertThat(var, equalTo("10.0.0.1")); - } - - /** - * Ensure that state is maintained correctly in the execution environment. - */ - @Test - public void testState() { - executor.assign("two", "2", message); - executor.assign("four", "4", message); - executor.assign("sum", "two + four", message); - - // verify - Object var = executor.getState().get("sum"); - assertEquals(6, var); - } - - /** - * Ensure that state is maintained correctly in the execution environment. - */ - @Test - public void testClearState() { - executor.assign("two", "2", message); - executor.clearState(); - - // verify - assertThat(executor.getState().containsKey("two"), equalTo(false)); - } - - /** - * Ensure that a Transformation function can be executed. - * - * There are two sets of functions in Stellar currently. One can be executed with - * a PredicateProcessor and the other a TransformationProcessor. The StellarExecutor - * abstracts away that complication. - */ - @Test - public void testExecuteTransformation() { - String actual = executor.execute("TO_UPPER('lowercase')", message, String.class); - assertThat(actual, equalTo("LOWERCASE")); - } - - /** - * Ensure that a Predicate function can be executed. - * - * There are two sets of functions in Stellar currently. One can be executed with - * a PredicateProcessor and the other a TransformationProcessor. The StellarExecutor - * abstracts away that complication. - */ - @Test - public void testExecutePredicate() { - boolean actual = executor.execute("IS_INTEGER(2)", message, Boolean.class); - assertThat(actual, equalTo(true)); - } - - /** - * An exception is expected if an expression results in an unexpected type. - */ - @Test(expected = RuntimeException.class) - public void testExecuteWithWrongType() { - executor.execute("2 + 2", message, Boolean.class); - } - - /** - * A best effort should be made to do sensible type conversions. - */ - @Test - public void testExecuteWithTypeConversion() { - executor.execute("2", message, Double.class); - executor.execute("2", message, Float.class); - executor.execute("2", message, Short.class); - executor.execute("2", message, Long.class); - } - - /** - * The executor must be serializable. - */ - @Test - public void testSerializable() throws Exception { - - // serialize - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - new ObjectOutputStream(bytes).writeObject(executor); - - // deserialize - success when no exceptions - new ObjectInputStream(new ByteArrayInputStream(bytes.toByteArray())).readObject(); - } -}
