Initial import. Migrating code from https://github.com/yahoo/samoa/
Project: http://git-wip-us.apache.org/repos/asf/incubator-samoa/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-samoa/commit/787864b6 Tree: http://git-wip-us.apache.org/repos/asf/incubator-samoa/tree/787864b6 Diff: http://git-wip-us.apache.org/repos/asf/incubator-samoa/diff/787864b6 Branch: refs/heads/master Commit: 787864b6a0983ba650ea56d5a56a3ed56f14b2d5 Parents: Author: Gianmarco De Francisci Morales <[email protected]> Authored: Sun Jan 18 17:27:34 2015 +0100 Committer: Gianmarco De Francisci Morales <[email protected]> Committed: Sun Jan 18 17:44:58 2015 +0100 ---------------------------------------------------------------------- .gitignore | 14 + .travis.yml | 26 + CLA.md | 32 + CLA.pdf | Bin 0 -> 33236 bytes CONTRIBUTING.md | 18 + LICENSE.txt | 201 ++ NOTICE.txt | 22 + README.md | 97 + RELEASE.txt | 33 + bin/run-class.sh | 79 + bin/run-container.sh | 39 + bin/s4-build/README.md | 3 + bin/s4-build/gradle-wrapper-1.4.jar | Bin 0 -> 49875 bytes bin/s4-build/gradle-wrapper-1.4.properties | 25 + bin/s4-build/gradlew | 164 ++ bin/samoa | 318 +++ bin/samoa-s4.properties | 46 + bin/samoa-samza.properties | 67 + bin/samoa-storm.properties | 35 + bin/samza-dist/run-am.sh | 39 + bin/samza-dist/run-class.sh | 79 + bin/samza-dist/run-container.sh | 39 + bin/samza-dist/run-job.sh | 39 + bin/samza-kryo | 6 + pom.xml | 286 +++ samoa-api/pom.xml | 127 ++ .../com/yahoo/labs/samoa/core/ContentEvent.java | 43 + .../com/yahoo/labs/samoa/core/DoubleVector.java | 119 ++ .../labs/samoa/core/EntranceProcessor.java | 59 + .../java/com/yahoo/labs/samoa/core/Globals.java | 59 + .../com/yahoo/labs/samoa/core/Processor.java | 61 + .../labs/samoa/core/SerializableInstance.java | 67 + ...BasicClassificationPerformanceEvaluator.java | 157 ++ .../BasicRegressionPerformanceEvaluator.java | 134 ++ .../ClassificationPerformanceEvaluator.java | 24 + .../ClusteringEvaluationContentEvent.java | 82 + .../ClusteringEvaluatorProcessor.java | 319 +++ .../ClusteringResultContentEvent.java | 73 + .../samoa/evaluation/EvaluatorProcessor.java | 234 +++ .../samoa/evaluation/PerformanceEvaluator.java | 62 + .../RegressionPerformanceEvaluator.java | 25 + ...indowClassificationPerformanceEvaluator.java | 220 ++ .../labs/samoa/evaluation/measures/CMM.java | 509 +++++ .../evaluation/measures/CMM_GTAnalysis.java | 843 ++++++++ .../evaluation/measures/EntropyCollection.java | 174 ++ .../labs/samoa/evaluation/measures/F1.java | 115 ++ .../labs/samoa/evaluation/measures/General.java | 191 ++ .../labs/samoa/evaluation/measures/SSQ.java | 96 + .../samoa/evaluation/measures/Separation.java | 118 ++ .../measures/SilhouetteCoefficient.java | 137 ++ .../measures/StatisticalCollection.java | 186 ++ .../samoa/examples/HelloWorldContentEvent.java | 69 + .../HelloWorldDestinationProcessor.java | 49 + .../examples/HelloWorldSourceProcessor.java | 75 + .../labs/samoa/examples/HelloWorldTask.java | 98 + .../labs/samoa/learners/AdaptiveLearner.java | 53 + .../samoa/learners/ClassificationLearner.java | 27 + .../samoa/learners/InstanceContentEvent.java | 207 ++ .../samoa/learners/InstancesContentEvent.java | 193 ++ .../com/yahoo/labs/samoa/learners/Learner.java | 62 + .../labs/samoa/learners/RegressionLearner.java | 27 + .../labs/samoa/learners/ResultContentEvent.java | 212 ++ .../learners/classifiers/LocalLearner.java | 78 + .../classifiers/LocalLearnerProcessor.java | 217 ++ .../samoa/learners/classifiers/NaiveBayes.java | 269 +++ .../classifiers/SimpleClassifierAdapter.java | 150 ++ .../learners/classifiers/SingleClassifier.java | 109 + .../classifiers/ensemble/AdaptiveBagging.java | 149 ++ .../learners/classifiers/ensemble/Bagging.java | 138 ++ .../ensemble/BaggingDistributorProcessor.java | 201 ++ .../learners/classifiers/ensemble/Boosting.java | 142 ++ .../ensemble/BoostingDistributorProcessor.java | 36 + .../BoostingPredictionCombinerProcessor.java | 176 ++ .../ensemble/PredictionCombinerProcessor.java | 184 ++ .../classifiers/rules/AMRulesRegressor.java | 176 ++ .../rules/HorizontalAMRulesRegressor.java | 240 +++ .../rules/VerticalAMRulesRegressor.java | 200 ++ .../centralized/AMRulesRegressorProcessor.java | 509 +++++ .../classifiers/rules/common/ActiveRule.java | 226 +++ .../classifiers/rules/common/LearningRule.java | 122 ++ .../rules/common/NonLearningRule.java | 51 + .../classifiers/rules/common/PassiveRule.java | 71 + .../classifiers/rules/common/Perceptron.java | 487 +++++ .../learners/classifiers/rules/common/Rule.java | 111 + .../rules/common/RuleActiveLearningNode.java | 34 + .../rules/common/RuleActiveRegressionNode.java | 318 +++ .../rules/common/RulePassiveLearningNode.java | 33 + .../rules/common/RulePassiveRegressionNode.java | 76 + .../rules/common/RuleRegressionNode.java | 292 +++ .../classifiers/rules/common/RuleSplitNode.java | 66 + .../classifiers/rules/common/TargetMean.java | 220 ++ .../distributed/AMRDefaultRuleProcessor.java | 334 +++ .../rules/distributed/AMRLearnerProcessor.java | 259 +++ .../rules/distributed/AMRRuleSetProcessor.java | 362 ++++ .../distributed/AMRulesAggregatorProcessor.java | 525 +++++ .../distributed/AMRulesStatisticsProcessor.java | 220 ++ .../distributed/AssignmentContentEvent.java | 74 + .../distributed/PredicateContentEvent.java | 84 + .../rules/distributed/RuleContentEvent.java | 82 + .../classifiers/trees/ActiveLearningNode.java | 207 ++ .../trees/AttributeBatchContentEvent.java | 134 ++ .../trees/AttributeContentEvent.java | 222 ++ .../classifiers/trees/ComputeContentEvent.java | 142 ++ .../classifiers/trees/ControlContentEvent.java | 71 + .../classifiers/trees/DeleteContentEvent.java | 45 + .../classifiers/trees/FilterProcessor.java | 191 ++ .../learners/classifiers/trees/FoundNode.java | 77 + .../classifiers/trees/InactiveLearningNode.java | 56 + .../classifiers/trees/LearningNode.java | 55 + .../trees/LocalResultContentEvent.java | 92 + .../trees/LocalStatisticsProcessor.java | 246 +++ .../trees/ModelAggregatorProcessor.java | 731 +++++++ .../samoa/learners/classifiers/trees/Node.java | 94 + .../learners/classifiers/trees/SplitNode.java | 108 + .../trees/VerticalHoeffdingTree.java | 185 ++ .../clusterers/ClusteringContentEvent.java | 89 + .../clusterers/ClustreamClustererAdapter.java | 166 ++ .../clusterers/LocalClustererAdapter.java | 82 + .../clusterers/LocalClustererProcessor.java | 191 ++ .../learners/clusterers/SingleLearner.java | 97 + .../simple/ClusteringDistributorProcessor.java | 98 + .../clusterers/simple/DistributedClusterer.java | 118 ++ .../yahoo/labs/samoa/moa/AbstractMOAObject.java | 80 + .../com/yahoo/labs/samoa/moa/MOAObject.java | 58 + .../moa/classifiers/AbstractClassifier.java | 378 ++++ .../labs/samoa/moa/classifiers/Classifier.java | 77 + .../labs/samoa/moa/classifiers/Regressor.java | 31 + .../core/AttributeSplitSuggestion.java | 68 + .../AttributeClassObserver.java | 73 + ...BinaryTreeNumericAttributeClassObserver.java | 183 ++ ...NumericAttributeClassObserverRegression.java | 148 ++ .../DiscreteAttributeClassObserver.java | 34 + .../FIMTDDNumericAttributeClassObserver.java | 240 +++ .../GaussianNumericAttributeClassObserver.java | 182 ++ ...waldKhannaNumericAttributeClassObserver.java | 126 ++ .../NominalAttributeClassObserver.java | 178 ++ .../NullAttributeClassObserver.java | 80 + .../NumericAttributeClassObserver.java | 34 + .../VFMLNumericAttributeClassObserver.java | 223 ++ .../InstanceConditionalBinaryTest.java | 35 + .../InstanceConditionalTest.java | 76 + .../NominalAttributeBinaryTest.java | 73 + .../NominalAttributeMultiwayTest.java | 71 + .../NumericAttributeBinaryTest.java | 92 + .../classifiers/core/driftdetection/ADWIN.java | 604 ++++++ .../driftdetection/ADWINChangeDetector.java | 73 + .../driftdetection/AbstractChangeDetector.java | 143 ++ .../core/driftdetection/ChangeDetector.java | 102 + .../core/driftdetection/CusumDM.java | 110 + .../classifiers/core/driftdetection/DDM.java | 117 ++ .../classifiers/core/driftdetection/EDDM.java | 138 ++ .../core/driftdetection/EWMAChartDM.java | 122 ++ .../GeometricMovingAverageDM.java | 108 + .../core/driftdetection/PageHinkleyDM.java | 114 ++ .../core/splitcriteria/GiniSplitCriterion.java | 86 + .../splitcriteria/InfoGainSplitCriterion.java | 118 ++ .../InfoGainSplitCriterionMultilabel.java | 54 + .../core/splitcriteria/SDRSplitCriterion.java | 33 + .../core/splitcriteria/SplitCriterion.java | 56 + .../VarianceReductionSplitCriterion.java | 99 + .../classifiers/functions/MajorityClass.java | 84 + .../moa/classifiers/rules/core/Predicate.java | 33 + ...IMTDDNumericAttributeClassLimitObserver.java | 126 ++ .../NumericAttributeBinaryRulePredicate.java | 180 ++ .../splitcriteria/SDRSplitCriterionAMRules.java | 105 + .../core/voting/AbstractErrorWeightedVote.java | 103 + .../rules/core/voting/ErrorWeightedVote.java | 81 + .../core/voting/InverseErrorWeightedVote.java | 99 + .../rules/core/voting/UniformWeightedVote.java | 73 + .../rules/driftdetection/PageHinkleyFading.java | 83 + .../rules/driftdetection/PageHinkleyTest.java | 96 + .../yahoo/labs/samoa/moa/cluster/CFCluster.java | 173 ++ .../yahoo/labs/samoa/moa/cluster/Cluster.java | 172 ++ .../labs/samoa/moa/cluster/Clustering.java | 276 +++ .../yahoo/labs/samoa/moa/cluster/Miniball.java | 84 + .../labs/samoa/moa/cluster/SphereCluster.java | 373 ++++ .../samoa/moa/clusterers/AbstractClusterer.java | 298 +++ .../samoa/moa/clusterers/ClusterGenerator.java | 368 ++++ .../labs/samoa/moa/clusterers/Clusterer.java | 64 + .../yahoo/labs/samoa/moa/clusterers/KMeans.java | 202 ++ .../moa/clusterers/clustream/Clustream.java | 337 +++ .../clusterers/clustream/ClustreamKernel.java | 269 +++ .../moa/clusterers/clustream/WithKmeans.java | 468 +++++ .../labs/samoa/moa/core/AutoClassDiscovery.java | 211 ++ .../labs/samoa/moa/core/AutoExpandVector.java | 133 ++ .../yahoo/labs/samoa/moa/core/DataPoint.java | 134 ++ .../yahoo/labs/samoa/moa/core/DoubleVector.java | 195 ++ .../com/yahoo/labs/samoa/moa/core/Example.java | 31 + .../yahoo/labs/samoa/moa/core/FastVector.java | 67 + .../labs/samoa/moa/core/GaussianEstimator.java | 123 ++ .../core/GreenwaldKhannaQuantileSummary.java | 281 +++ .../moa/core/InputStreamProgressMonitor.java | 131 ++ .../labs/samoa/moa/core/InstanceExample.java | 50 + .../yahoo/labs/samoa/moa/core/Measurement.java | 115 ++ .../yahoo/labs/samoa/moa/core/MiscUtils.java | 97 + .../labs/samoa/moa/core/ObjectRepository.java | 32 + .../labs/samoa/moa/core/SerializeUtils.java | 112 + .../yahoo/labs/samoa/moa/core/Statistics.java | 1046 ++++++++++ .../yahoo/labs/samoa/moa/core/StringUtils.java | 96 + .../com/yahoo/labs/samoa/moa/core/Utils.java | 1911 ++++++++++++++++++ .../samoa/moa/evaluation/LearningCurve.java | 132 ++ .../moa/evaluation/LearningEvaluation.java | 63 + .../LearningPerformanceEvaluator.java | 61 + .../samoa/moa/evaluation/MeasureCollection.java | 259 +++ .../samoa/moa/evaluation/MembershipMatrix.java | 153 ++ .../yahoo/labs/samoa/moa/learners/Learner.java | 136 ++ .../samoa/moa/options/AbstractClassOption.java | 236 +++ .../moa/options/AbstractOptionHandler.java | 193 ++ .../labs/samoa/moa/options/ClassOption.java | 175 ++ .../labs/samoa/moa/options/OptionHandler.java | 79 + .../labs/samoa/moa/options/OptionsHandler.java | 202 ++ .../labs/samoa/moa/streams/ArffFileStream.java | 199 ++ .../labs/samoa/moa/streams/ExampleStream.java | 79 + .../labs/samoa/moa/streams/InstanceStream.java | 37 + .../moa/streams/clustering/ClusterEvent.java | 50 + .../clustering/ClusterEventListener.java | 31 + .../streams/clustering/ClusteringStream.java | 55 + .../clustering/RandomRBFGeneratorEvents.java | 975 +++++++++ .../streams/generators/HyperplaneGenerator.java | 178 ++ .../streams/generators/RandomTreeGenerator.java | 265 +++ .../yahoo/labs/samoa/moa/tasks/NullMonitor.java | 102 + .../samoa/moa/tasks/ResultPreviewListener.java | 40 + .../com/yahoo/labs/samoa/moa/tasks/Task.java | 61 + .../yahoo/labs/samoa/moa/tasks/TaskMonitor.java | 140 ++ .../labs/samoa/streams/ArffFileStream.java | 119 ++ .../streams/ClusteringEntranceProcessor.java | 241 +++ .../yahoo/labs/samoa/streams/FileStream.java | 174 ++ .../streams/PrequentialSourceProcessor.java | 229 +++ .../yahoo/labs/samoa/streams/StreamSource.java | 90 + .../samoa/streams/StreamSourceProcessor.java | 185 ++ .../labs/samoa/streams/fs/FileStreamSource.java | 67 + .../samoa/streams/fs/HDFSFileStreamSource.java | 150 ++ .../samoa/streams/fs/LocalFileStreamSource.java | 131 ++ .../labs/samoa/tasks/ClusteringEvaluation.java | 174 ++ .../labs/samoa/tasks/PrequentialEvaluation.java | 206 ++ .../java/com/yahoo/labs/samoa/tasks/Task.java | 61 + .../AbstractEntranceProcessingItem.java | 108 + .../samoa/topology/AbstractProcessingItem.java | 161 ++ .../labs/samoa/topology/AbstractStream.java | 115 ++ .../labs/samoa/topology/AbstractTopology.java | 133 ++ .../labs/samoa/topology/ComponentFactory.java | 78 + .../samoa/topology/EntranceProcessingItem.java | 46 + .../labs/samoa/topology/IProcessingItem.java | 47 + .../yahoo/labs/samoa/topology/ISubmitter.java | 46 + .../topology/LocalEntranceProcessingItem.java | 85 + .../labs/samoa/topology/ProcessingItem.java | 70 + .../com/yahoo/labs/samoa/topology/Stream.java | 62 + .../com/yahoo/labs/samoa/topology/Topology.java | 85 + .../labs/samoa/topology/TopologyBuilder.java | 223 ++ .../labs/samoa/utils/PartitioningScheme.java | 32 + .../labs/samoa/utils/StreamDestination.java | 65 + .../java/com/yahoo/labs/samoa/utils/Utils.java | 182 ++ .../yahoo/labs/samoa/core/DoubleVectorTest.java | 97 + .../streams/fs/HDFSFileStreamSourceTest.java | 303 +++ .../streams/fs/LocalFileStreamSourceTest.java | 276 +++ samoa-instances/pom.xml | 35 + .../yahoo/labs/samoa/instances/ArffLoader.java | 364 ++++ .../yahoo/labs/samoa/instances/Attribute.java | 205 ++ .../labs/samoa/instances/DenseInstance.java | 72 + .../labs/samoa/instances/DenseInstanceData.java | 97 + .../yahoo/labs/samoa/instances/Instance.java | 74 + .../labs/samoa/instances/InstanceData.java | 55 + .../samoa/instances/InstanceInformation.java | 112 + .../yahoo/labs/samoa/instances/Instances.java | 246 +++ .../labs/samoa/instances/InstancesHeader.java | 131 ++ .../instances/SingleClassInstanceData.java | 86 + .../samoa/instances/SingleLabelInstance.java | 261 +++ .../labs/samoa/instances/SparseInstance.java | 49 + .../samoa/instances/SparseInstanceData.java | 172 ++ .../com/yahoo/labs/samoa/instances/Utils.java | 88 + samoa-local/pom.xml | 101 + .../java/com/yahoo/labs/samoa/LocalDoTask.java | 89 + .../topology/impl/SimpleComponentFactory.java | 53 + .../labs/samoa/topology/impl/SimpleEngine.java | 37 + .../impl/SimpleEntranceProcessingItem.java | 33 + .../topology/impl/SimpleProcessingItem.java | 87 + .../labs/samoa/topology/impl/SimpleStream.java | 93 + .../samoa/topology/impl/SimpleTopology.java | 44 + samoa-local/src/main/resources/log4j.xml | 35 + .../java/com/yahoo/labs/samoa/AlgosTest.java | 87 + .../impl/SimpleComponentFactoryTest.java | 96 + .../samoa/topology/impl/SimpleEngineTest.java | 57 + .../impl/SimpleEntranceProcessingItemTest.java | 153 ++ .../topology/impl/SimpleProcessingItemTest.java | 120 ++ .../samoa/topology/impl/SimpleStreamTest.java | 111 + .../samoa/topology/impl/SimpleTopologyTest.java | 93 + samoa-s4/pom.xml | 135 ++ samoa-s4/samoa-s4-adapter/pom.xml | 52 + .../samoa/topology/adapter/S4AdapterApp.java | 45 + .../adapter/S4EntranceProcessingItem.java | 74 + .../samoa/topology/adapter/package-info.java | 28 + samoa-s4/src/main/assembly/samoa-s4.xml | 64 + .../samoa/topology/impl/S4ComponentFactory.java | 97 + .../labs/samoa/topology/impl/S4DoTask.java | 263 +++ .../topology/impl/S4EntranceProcessingItem.java | 120 ++ .../yahoo/labs/samoa/topology/impl/S4Event.java | 90 + .../samoa/topology/impl/S4ProcessingItem.java | 188 ++ .../labs/samoa/topology/impl/S4Stream.java | 185 ++ .../labs/samoa/topology/impl/S4Submitter.java | 146 ++ .../labs/samoa/topology/impl/S4Topology.java | 61 + .../samoa/topology/impl/SamoaSerializer.java | 99 + .../topology/impl/SamoaSerializerModule.java | 35 + samoa-samza/pom.xml | 168 ++ samoa-samza/src/main/assembly/samoa-samza.xml | 108 + .../java/com/yahoo/labs/samoa/SamzaDoTask.java | 227 +++ .../samoa/topology/impl/SamoaSystemFactory.java | 57 + .../topology/impl/SamzaComponentFactory.java | 62 + .../labs/samoa/topology/impl/SamzaEngine.java | 197 ++ .../impl/SamzaEntranceProcessingItem.java | 222 ++ .../topology/impl/SamzaProcessingItem.java | 165 ++ .../topology/impl/SamzaProcessingNode.java | 56 + .../labs/samoa/topology/impl/SamzaStream.java | 248 +++ .../labs/samoa/topology/impl/SamzaTopology.java | 64 + .../labs/samoa/utils/SamzaConfigFactory.java | 532 +++++ .../labs/samoa/utils/SamzaKryoSerdeFactory.java | 155 ++ .../samoa/utils/SerializableSerializer.java | 70 + .../yahoo/labs/samoa/utils/SystemsUtils.java | 383 ++++ samoa-samza/src/main/resources/log4j.xml | 69 + samoa-storm/pom.xml | 123 ++ .../com/yahoo/labs/samoa/LocalStormDoTask.java | 78 + .../samoa/topology/impl/StormBoltStream.java | 65 + .../topology/impl/StormComponentFactory.java | 90 + .../labs/samoa/topology/impl/StormDoTask.java | 117 ++ .../impl/StormEntranceProcessingItem.java | 208 ++ .../samoa/topology/impl/StormJarSubmitter.java | 75 + .../topology/impl/StormProcessingItem.java | 170 ++ .../samoa/topology/impl/StormSamoaUtils.java | 108 + .../samoa/topology/impl/StormSpoutStream.java | 65 + .../labs/samoa/topology/impl/StormStream.java | 85 + .../labs/samoa/topology/impl/StormTopology.java | 52 + .../samoa/topology/impl/StormTopologyNode.java | 34 + .../topology/impl/StormTopologySubmitter.java | 133 ++ .../java/com/yahoo/labs/samoa/AlgosTest.java | 68 + .../topology/impl/StormProcessingItemTest.java | 78 + samoa-test/README.md | 14 + samoa-test/pom.xml | 65 + .../assembly/test-jar-with-dependencies.xml | 19 + .../java/com/yahoo/labs/samoa/TestParams.java | 235 +++ .../java/com/yahoo/labs/samoa/TestUtils.java | 153 ++ samoa-threads/pom.xml | 112 + .../yahoo/labs/samoa/LocalThreadsDoTask.java | 70 + .../topology/impl/ThreadsComponentFactory.java | 64 + .../labs/samoa/topology/impl/ThreadsEngine.java | 100 + .../impl/ThreadsEntranceProcessingItem.java | 40 + .../topology/impl/ThreadsEventRunnable.java | 61 + .../topology/impl/ThreadsProcessingItem.java | 101 + .../impl/ThreadsProcessingItemInstance.java | 54 + .../labs/samoa/topology/impl/ThreadsStream.java | 106 + .../samoa/topology/impl/ThreadsTopology.java | 62 + .../java/com/yahoo/labs/samoa/AlgosTest.java | 68 + .../impl/ThreadsComponentFactoryTest.java | 114 ++ .../samoa/topology/impl/ThreadsEngineTest.java | 129 ++ .../impl/ThreadsEntranceProcessingItemTest.java | 133 ++ .../topology/impl/ThreadsEventRunnableTest.java | 67 + .../impl/ThreadsProcessingItemInstanceTest.java | 67 + .../impl/ThreadsProcessingItemTest.java | 173 ++ .../samoa/topology/impl/ThreadsStreamTest.java | 127 ++ .../topology/impl/ThreadsTopologyTest.java | 84 + .../labs/samoa/utils/StreamDestinationTest.java | 80 + 359 files changed, 50650 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-samoa/blob/787864b6/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3cf0208 --- /dev/null +++ b/.gitignore @@ -0,0 +1,14 @@ +#maven +target/ + +#eclipse +.classpath +.project +.settings/ + +#DS_Store +.DS_Store + +#intellij +.idea/ +.iml http://git-wip-us.apache.org/repos/asf/incubator-samoa/blob/787864b6/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..5e4bbb6 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,26 @@ +language: java + +install: +- git clone https://github.com/apache/incubator-s4.git +- cd incubator-s4 +- git checkout tags/0.6.0-Final +- cp ../bin/s4-build/gradlew . +- cp ../bin/s4-build/gradle-wrapper-1.4.jar ./lib/ +- cp ../bin/s4-build/gradle-wrapper-1.4.properties ./lib/ +- ./gradlew install +- ./gradlew s4-tools::installApp +- cd .. +- echo "<settings><servers><server><id>sonatype-nexus-snapshots</id><username>${SOSS_USERNAME}</username><password>${SOSS_PASSWORD}</password></server></servers></settings>" > ${HOME}/.m2/settings.xml +#- cat ${HOME}/.m2/settings.xml + +script: if [[ "$TRAVIS_SECURE_ENV_VARS" == "true" && "$TRAVIS_PULL_REQUEST" == "false" ]]; then CMD=deploy; else CMD=install; fi; echo $CMD; mvn -B -Pall $CMD + +notifications: + email: + on_success: never + on_failure: change + +env: + global: + - secure: "hSyN3Ys3wDMJtL8jAcfFMh8pnG7B2TaKXc4qDWgE9a73XQ77JB8asCeXtQx/0/rNrJeNLBdSrVcXNAaOXXgGZpftJ0WdIBsyAj+tzpVAf+pcEHPVCgR4PHLkm1/UlyGX//1J+DjkDXnRgNfsD8xjZxTeNFH8xFzAU5YaP0AiLmk=" + - secure: "MJLRFWi1uGZ1s5u/A44u4vDSGXF23H/3GGhofvLliaM4ivkeO9uthErlHgloGSmubEVkJMiThBLveZl01tNYRgn5a08qqyIsf/eShMagJDR7cX6FmbU7qOMOSzaAI84GDtrNuDQqaz2I1nTKfnzDYcTGXrpJMwLmbx30E9D/qaY=" http://git-wip-us.apache.org/repos/asf/incubator-samoa/blob/787864b6/CLA.md ---------------------------------------------------------------------- diff --git a/CLA.md b/CLA.md new file mode 100644 index 0000000..2ac6391 --- /dev/null +++ b/CLA.md @@ -0,0 +1,32 @@ +#YAHOO! OPEN SOURCE CONTRIBUTOR LICENSE AGREEMENT# +Thank you for your interest in open source projects of Yahoo!, Inc. and its affiliates (âYahoo!â). In order to clarify the intellectual property license granted with Contributions from any person or entity, Yahoo! must have a Contributor License Agreement (âCLAâ) on file that has been signed by each contributor, indicating agreement to the license terms below. This license is for your protection as a contributor as well as for the protection of Yahoo! and its users; it does not change your rights to use your own Contributions for any other purpose. + +This version of the Agreement allows a person or entity to submit Contributions to Yahoo!, to authorize Contributions to Yahoo! submitted by an entityâs employees, and to grant copyright and patent licenses thereto. + +Please read this document carefully before signing and keep a copy for your records. + +##TERMS AND CONDITIONS## + +You accept and agree to the following terms and conditions for Your present and future Contributions submitted to Yahoo!, in return for the opportunity to have the Contributions be considered for inclusion in one or more Yahoo! open source projects. Except for the license granted herein to Yahoo! and recipients of software distributed or otherwise made available by Yahoo!, You reserve all right, title, and interest in and to Your Contributions. + +1. **Definitions.** + + 1.1 **âContributionâ** means any work of authorship (including software, documentation or other material), including any modifications or additions to an existing work, that is intentionally submitted by You to Yahoo! for inclusion in, or documentation of, any of the products or projects owned or managed by Yahoo! (the **âWorkâ**). For the purposes of this definition, âsubmittedâ means any form of electronic, verbal, or written communication sent to Yahoo! or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, Yahoo! for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by You as âNot a Contribution.â + + 1.2 **âYouâ** (or **âYourâ**) means the copyright owner, or legal entity authorized by the copyright owner, that is making this Agreement with Yahoo!. For legal entities, âYouâ (or âYourâ) includes the entity making a Contribution, or on behalf of which the Contribution is made, and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, âcontrolâ means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, (b) ownership of fifty percent (50%) or more of the outstanding shares, or (c) beneficial ownership of such entity. + +2. **Grant of Copyright License.** Subject to the terms and conditions of this Agreement, You hereby grant to Yahoo! and to recipients of software distributed or otherwise made available by Yahoo! a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare derivative works of, publicly display, publicly perform, sublicense, and distribute and otherwise make available Your Contributions and such derivative works. + +3. **Grant of Patent License.** Subject to the terms and conditions of this Agreement, You hereby grant to Yahoo! and to recipients of software distributed or otherwise made available by Yahoo! a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by You that are necessarily infringed by Your Contribution(s) alone or by combination of Your Contribution(s) with the Work to which such Contribution(s) were submitted. If any entity institutes patent litigation against You or any other entity (including a cross-claim or counterclaim in a lawsuit) alleging that Your Contribution, or the Work to which You have contributed, constitutes direct or contributory patent infringement, then any patent licenses granted to that entity under this Agreement for that Contribution o r Work shall terminate as of the date such litigation is filed. + +4. **Representations.** + + 4.1 You represent that You are legally entitled to grant the above license and enter into this Agreement, whether on behalf of Yourself (if You are an individual person) or on behalf of the entity which You represent (if You are an entity). If You are an individual and Your employer(s) has rights to intellectual property that You create that includes Your Contributions, You represent that You have received permission to make Contributions on behalf of that employer, that Your employer has waived such rights for Your Contributions to Yahoo!, or that Your employer has executed a separate CLA with Yahoo!. If You are an entity rather than an individual, You represent further that each employee of the entity designated on Schedule A below (or in a subsequent written modification to that Schedule) is authorized to submit Contributions on behalf of the entity. + + 4.2 You represent that each of Your Contributions is Your original creation (see Section 6 for submissions on behalf of others). You represent that Your Contribution submissions include complete details of any third-party license or other restriction (including, not limited to, related patents and trademarks) of which You are personally aware and which are associated with any part of Your Contributions. + +5. **Disclaimer.** You are not expected to provide support for Your Contributions, except to the extent You desire to provide support. You may provide support for free, for a fee, or not at all. Unless required by applicable law or agreed to in writing, You provide Your Contributions on an âAS ISâ BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + +6. **Third Party Contributions.** Should You wish to submit work that is not Your original creation, You may submit it to Yahoo! separately from any Contribution, identifying the complete details of its source and of any license or other restriction (including, but not limited to, related patents, trademarks, and license agreements) of which You are personally aware, and conspicuously marking the work as âSubmitted on behalf of a third-party: [named here]â. + +7. **Notice.** It is Your responsibility to notify Yahoo! when any change is required to the list of designated employees authorized to submit Contributions on behalf of the entity, or to the entityâs point of contact with Yahoo!. http://git-wip-us.apache.org/repos/asf/incubator-samoa/blob/787864b6/CLA.pdf ---------------------------------------------------------------------- diff --git a/CLA.pdf b/CLA.pdf new file mode 100644 index 0000000..66b445b Binary files /dev/null and b/CLA.pdf differ http://git-wip-us.apache.org/repos/asf/incubator-samoa/blob/787864b6/CONTRIBUTING.md ---------------------------------------------------------------------- diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..789e63d --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,18 @@ +How to contribute? +================= + +User contribution is essential to keep moving the project forward. + +# Getting Started + +1. [Fork the repository](https://help.github.com/articles/fork-a-repo) on GitHub + +2. Sign the [Contributor License Agreement] (http://www.clahub.com/agreements/gdfm/samoa) + +3. Create a feature branch ```git checkout -b branch_name``` + +4. Make your changes (please make commits that are logical units) + +5. Build and run tests + +6. Submit a [pull request](https://help.github.com/articles/using-pull-requests) http://git-wip-us.apache.org/repos/asf/incubator-samoa/blob/787864b6/LICENSE.txt ---------------------------------------------------------------------- diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. http://git-wip-us.apache.org/repos/asf/incubator-samoa/blob/787864b6/NOTICE.txt ---------------------------------------------------------------------- diff --git a/NOTICE.txt b/NOTICE.txt new file mode 100644 index 0000000..19183f5 --- /dev/null +++ b/NOTICE.txt @@ -0,0 +1,22 @@ +SAMOA +Copyright 2013 Yahoo! Inc. + +Licensed under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at: + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + +Except as specifically stated below, the 3rd party software packages are not distributed as part of +this project, but instead are separately downloaded from the respective provider. + +* MOA version 13.08 (redistributed under the Apache License v2 - http://www.apache.org/licenses/LICENSE-2.0) + Library for data stream mining. Only a small subset of the original library is redistributed with SAMOA. + http://moa.cms.waikato.ac.nz http://git-wip-us.apache.org/repos/asf/incubator-samoa/blob/787864b6/README.md ---------------------------------------------------------------------- diff --git a/README.md b/README.md new file mode 100644 index 0000000..4e6293c --- /dev/null +++ b/README.md @@ -0,0 +1,97 @@ +<!-- + Copyright (c) 2013 Yahoo! Inc. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +[](https://travis-ci.org/yahoo/samoa) + +SAMOA: Scalable Advanced Massive Online Analysis. +================= +SAMOA is a platform for mining on big data streams. +It is a distributed streaming machine learning (ML) framework that contains a +programing abstraction for distributed streaming ML algorithms. + +SAMOA enables development of new ML algorithms without dealing with +the complexity of underlying streaming processing engines (SPE, such +as Apache Storm and Apache S4). SAMOA also provides extensibility in integrating +new SPEs into the framework. These features allow SAMOA users to develop +distributed streaming ML algorithms once and to execute the algorithms +in multiple SPEs, i.e., code the algorithms once and execute them in multiple SPEs. + +## Build + +###Storm mode + +Simply clone the repository and install SAMOA. +```bash +git clone [email protected]:yahoo/samoa.git +cd samoa +mvn -Pstorm package +``` + +The deployable jar for SAMOA will be in `target/SAMOA-Storm-0.0.1-SNAPSHOT.jar`. + +###S4 mode + +If you want to compile SAMOA for S4, you will need to install the S4 dependencies +manually as explained in [Executing SAMOA with Apache S4](../../wiki/Executing-SAMOA-with-Apache-S4). + +Once the dependencies if needed are installed, you can simply clone the repository and install SAMOA. + +```bash +git clone [email protected]:yahoo/samoa.git +cd samoa +mvn -Ps4 package +``` + +###Local mode + +If you want to test SAMOA in a local environment, simply clone the repository and install SAMOA. + +```bash +git clone [email protected]:yahoo/samoa.git +cd samoa +mvn package +``` + +The deployable jar for SAMOA will be in `target/SAMOA-Local-0.0.1-SNAPSHOT.jar`. + +## Documentation + +The documentation is intended to give an introduction on how to use SAMOA in the various possible ways. +As a user you can use it to develop new algorithms and test different Distributed Stream Processing Engines. + +[Wiki Documentation](http://github.com/yahoo/samoa/wiki) + +[Javadoc](http://yahoo.github.io/samoa/docs/api) + +## Slides + +[](https://speakerdeck.com/gdfm/samoa-a-platform-for-mining-big-data-streams) + +G. De Francisci Morales [SAMOA: A Platform for Mining Big Data Streams](http://melmeric.files.wordpress.com/2013/04/samoa-a-platform-for-mining-big-data-streams.pdf) +Keynote Talk at [RAMSS â13](http://www.ramss.ws/2013/program/): 2nd International Workshop on Real-Time Analysis and Mining of Social Streams WWW, Rio De Janeiro, 2013. + +## SAMOA Developer's Guide + +<p><a href="http://yahoo.github.io/samoa/SAMOA-Developers-Guide-0-0-1.pdf";><img style="max-width:95%;border:3px solid black;" src="http://yahoo.github.io/samoa/Manual.png"; alt="SAMOA Developer's Guide" height="250"> </a></p> + +## Contributors +[List of contributors to the SAMOA project](http://yahoo.github.io/samoa/contributors.html) + +## License + +The use and distribution terms for this software are covered by the +Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0.html). + http://git-wip-us.apache.org/repos/asf/incubator-samoa/blob/787864b6/RELEASE.txt ---------------------------------------------------------------------- diff --git a/RELEASE.txt b/RELEASE.txt new file mode 100644 index 0000000..bb93f36 --- /dev/null +++ b/RELEASE.txt @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +Release v0.2.0 + NaiveBayes classitication algorithm. + AMRules regression algorithm. + Samza execution engine. + Multithread execution engine. + HDFS stream source. + +Release v0.1.0 + Initial release. + Vertical Hoeffding Tree classification algorithm. + Clustream clustering algorithm. + Adaptive ensembles (Bagging and Boosting). + Local execution engine. + Storm execution engine. + S4 execution engine. http://git-wip-us.apache.org/repos/asf/incubator-samoa/blob/787864b6/bin/run-class.sh ---------------------------------------------------------------------- diff --git a/bin/run-class.sh b/bin/run-class.sh new file mode 100644 index 0000000..8634df0 --- /dev/null +++ b/bin/run-class.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +### +# #%L +# SAMOA +# %% +# Copyright (C) 2013 - 2014 Yahoo! Inc. +# %% +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# #L% +### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +if [ $# -lt 1 ]; +then + echo "USAGE: $0 classname [opts]" + exit 1 +fi + +home_dir=`pwd` +base_dir=$(dirname $0)/.. +cd $base_dir +base_dir=`pwd` +cd $home_dir + +YARN_HOME="${YARN_HOME:-$HOME/.samza}" +#CLASSPATH=$base_dir:$YARN_HOME/conf +CLASSPATH=$base_dir/target/SAMOA-Samza-*.jar +if [ -z "$JAVA_HOME" ]; then + JAVA="java" +else + JAVA="$JAVA_HOME/bin/java" +fi + +if [ -z "$SAMZA_LOG_DIR" ]; then + SAMZA_LOG_DIR="$base_dir" +fi + +if [ -z "$SAMZA_CONTAINER_NAME" ]; then + SAMZA_CONTAINER_NAME="undefined-samza-container-name" +fi + +if [ -z "$JAVA_OPTS" ]; then + JAVA_OPTS="-Xmx768M -XX:+PrintGCDateStamps" +fi + +if [ -f $base_dir/lib/log4j.xml ]; then + JAVA_OPTS="$JAVA_OPTS -Dlog4j.configuration=file:$base_dir/log4j.xml" +fi + +JAVA_OPTS="$JAVA_OPTS -Xloggc:$SAMZA_LOG_DIR/gc.log -Dsamza.log.dir=$SAMZA_LOG_DIR -Dsamza.container.name=$SAMZA_CONTAINER_NAME" + +echo $JAVA $JAVA_OPTS -cp $CLASSPATH $@ +exec $JAVA $JAVA_OPTS -cp $CLASSPATH $@ http://git-wip-us.apache.org/repos/asf/incubator-samoa/blob/787864b6/bin/run-container.sh ---------------------------------------------------------------------- diff --git a/bin/run-container.sh b/bin/run-container.sh new file mode 100644 index 0000000..c46bf77 --- /dev/null +++ b/bin/run-container.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +### +# #%L +# SAMOA +# %% +# Copyright (C) 2013 - 2014 Yahoo! Inc. +# %% +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# #L% +### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +exec $(dirname $0)/run-class.sh org.apache.samza.container.SamzaContainer $@ http://git-wip-us.apache.org/repos/asf/incubator-samoa/blob/787864b6/bin/s4-build/README.md ---------------------------------------------------------------------- diff --git a/bin/s4-build/README.md b/bin/s4-build/README.md new file mode 100644 index 0000000..fd49c58 --- /dev/null +++ b/bin/s4-build/README.md @@ -0,0 +1,3 @@ +As a workaround for travis CI using gradle 2.1+, which causes issues with the s4 build, this directory contains +pre-generated gradlew scripts and libraries for gradle 1.4, that can be copied and used during the travis CI build. + http://git-wip-us.apache.org/repos/asf/incubator-samoa/blob/787864b6/bin/s4-build/gradle-wrapper-1.4.jar ---------------------------------------------------------------------- diff --git a/bin/s4-build/gradle-wrapper-1.4.jar b/bin/s4-build/gradle-wrapper-1.4.jar new file mode 100644 index 0000000..a7634b0 Binary files /dev/null and b/bin/s4-build/gradle-wrapper-1.4.jar differ http://git-wip-us.apache.org/repos/asf/incubator-samoa/blob/787864b6/bin/s4-build/gradle-wrapper-1.4.properties ---------------------------------------------------------------------- diff --git a/bin/s4-build/gradle-wrapper-1.4.properties b/bin/s4-build/gradle-wrapper-1.4.properties new file mode 100644 index 0000000..8144b9b --- /dev/null +++ b/bin/s4-build/gradle-wrapper-1.4.properties @@ -0,0 +1,25 @@ +### +# #%L +# SAMOA +# %% +# Copyright (C) 2015 Yahoo! Inc. +# %% +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# #L% +### +#Wed Sep 24 11:25:39 CEST 2014 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists +distributionUrl=http\://services.gradle.org/distributions/gradle-1.4-bin.zip http://git-wip-us.apache.org/repos/asf/incubator-samoa/blob/787864b6/bin/s4-build/gradlew ---------------------------------------------------------------------- diff --git a/bin/s4-build/gradlew b/bin/s4-build/gradlew new file mode 100755 index 0000000..3f041e5 --- /dev/null +++ b/bin/s4-build/gradlew @@ -0,0 +1,164 @@ +#!/usr/bin/env bash + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS="" + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn ( ) { + echo "$*" +} + +die ( ) { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; +esac + +# For Cygwin, ensure paths are in UNIX format before anything is touched. +if $cygwin ; then + [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"` +fi + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >&- +APP_HOME="`pwd -P`" +cd "$SAVED" >&- + +CLASSPATH=$APP_HOME/lib/gradle-wrapper-1.4.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin, switch paths to Windows format before running java +if $cygwin ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=$((i+1)) + done + case $i in + (0) set -- ;; + (1) set -- "$args0" ;; + (2) set -- "$args0" "$args1" ;; + (3) set -- "$args0" "$args1" "$args2" ;; + (4) set -- "$args0" "$args1" "$args2" "$args3" ;; + (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules +function splitJvmOpts() { + JVM_OPTS=("$@") +} +eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS +JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME" + +exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@" http://git-wip-us.apache.org/repos/asf/incubator-samoa/blob/787864b6/bin/samoa ---------------------------------------------------------------------- diff --git a/bin/samoa b/bin/samoa new file mode 100755 index 0000000..3d6d340 --- /dev/null +++ b/bin/samoa @@ -0,0 +1,318 @@ +#!/bin/bash + +### +# #%L +# SAMOA +# %% +# Copyright (C) 2013 Yahoo! Inc. +# %% +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# #L% +### + +# ===================================================================================================== # +# Scalable Advanced Massive Online Analysis a.k.a SAMOA # +# ===================================================================================================== # +# ----------------------------------------------------------------------------------------------------- # +# SAMOA is a framework for mining big data streams. It contains machine learning # +# algorithms that run on top of distributed stream processing engines. # +# # +# Usage: # +# ./samoa <platform> <jar> <task & options> # +# Example: # +# ./samoa storm /path/to/SAMOA-Storm-0.0.1.jar "ClusteringTask -s (RandomRBFGeneratorEvents -K 5 -a 2)" # +# # +# ----------------------------------------------------------------------------------------------------- # + +usage() { +echo "Usage: $(basename $0) <platform> <jar> <task & options>" && exit 1 +} +[ "$#" -ge 3 ] || usage + +echo $0 +BASE_DIR=$(dirname $0) +PLATFORM=${1:-NONE} +PLATFORM=$(echo $PLATFORM | tr '[:lower:]' '[:upper:]') +JAR_PATH=$2 +JAR_FILE=$(basename $JAR_PATH) +JAR_DIR=$(dirname $JAR_PATH) +OPTIONS=$3 + + +if [ $PLATFORM = 'S4' ]; then + + echo "Deploying to $PLATFORM" + if [ -z $S4_HOME ];then + echo "S4_HOME is not set!" + echo "Please set S4_HOME to point to your S4 installation" + exit -1 + fi + + # Local functions + deployLocal() + { + # Make sure to kill S4 processes before re-deploying + echo "Starting Zookeeper..." + (cd $S4_HOME ; . s4 zkServer -clean -port=${ZK_SERVER_PORT} &) + sleep 10 + + echo "Defining cluster..." + (cd $S4_HOME ; . s4 newCluster -c=${CLUSTER_NAME} -nbTasks=1 -flp=${CLUSTER_PORT} -zk=${ZK_SERVER}:${ZK_SERVER_PORT} & ) + sleep 5 + + echo "Starting node on cluster $CLUSTER_NAME..." + (cd $S4_HOME ; . s4 node -c=${CLUSTER_NAME} -zk=${ZK_SERVER}:${ZK_SERVER_PORT} & ) + sleep 5 + } + + SAMOA_S4_PROPERTIES="samoa-s4.properties" + PROPERTY_VALUE="" + readProperty() { + property=$1 + PROPERTY_VALUE=`sed '/^\#/d' $BASE_DIR/$SAMOA_S4_PROPERTIES | grep $property | tail -n 1 | cut -d "=" -f2- | sed 's/^[[:space:]]*//;s/[[:space:]]*$//'` + } + # END Local functions + + curr_dir=`pwd` + if [ -f /tmp/httpPID ]; then + pid=`cat /tmp/httpPID` + kill -9 $pid + rm /tmp/httpPID + fi + + cd $JAR_DIR + # Starting HTTP Server + python -m SimpleHTTPServer 8000 & + echo $! > /tmp/httpPID + sleep 2 + cd $curr_dir + + # Reading properties file + readProperty 'http.server.ip' + APP_SERVER_IP=$PROPERTY_VALUE + if [ -z $APP_SERVER_IP ]; then + APP_SERVER_IP=$(hostname) + fi + + readProperty 'http.server.port' + APP_SERVER_PORT=$PROPERTY_VALUE + if [ -z $APP_SERVER_PORT ]; then + APP_SERVER_PORT=8000 + fi + + readProperty 'zookeeper.server' + ZK_SERVER=$PROPERTY_VALUE + if [ -z $ZK_SERVER ]; then + ZK_SERVER=$(hostname) + fi + + readProperty 'zookeeper.port' + ZK_SERVER_PORT=$PROPERTY_VALUE + if [ -z $ZK_SERVER_PORT ]; then + ZK_SERVER_PORT=2181 + fi + + readProperty 'cluster.name' + CLUSTER_NAME=$PROPERTY_VALUE + if [ -z $CLUSTER_NAME ]; then + CLUSTER_NAME="cluster" + fi + + readProperty 'cluster.port' + CLUSTER_PORT=$PROPERTY_VALUE + if [ -z $CLUSTER_NAME ]; then + CLUSTER_PORT=12000 + fi + + readProperty 'samoa.deploy.mode' + DEPLOY_MODE=$PROPERTY_VALUE + if [ -z $DEPLOY_MODE ]; then + DEPLOY_MODE=local + elif [ $DEPLOY_MODE = 'local' ]; then + deployLocal + fi + # END Reading properties file + + APP_NAME="SamoaS4App" + APP_CLASS="com.yahoo.labs.samoa.topology.impl.S4DoTask" + NAMED_PARAMETERS=$(perl -MURI::Escape -e 'print uri_escape($ARGV[0]);' "$OPTIONS") + + echo "Deploying SAMOA..." + (cd $S4_HOME ; . s4 deploy -s4r="http://${APP_SERVER_IP}:${APP_SERVER_PORT}/${JAR_FILE}"; -c="${CLUSTER_NAME}" -p=evalTask="${NAMED_PARAMETERS}" -appClass="${APP_CLASS}" -appName="${APP_NAME}" -zk="${ZK_SERVER}:${ZK_SERVER_PORT}") + + # wait for app to be deployed + sleep 5 + # END Apache S4 deployment + +elif [ $PLATFORM = 'STORM' ]; then + + echo "Deploying to $PLATFORM" + if [ -z $STORM_HOME ];then + echo "STORM_HOME is not set!" + echo "Please set STORM_HOME to point to your Storm installation" + exit -1 + fi + + if [ ! -f $2 ];then + echo "$2 does not exist!" + echo "Please use a valid jar file for Storm execution" + exit -1 + fi + + STORM_EXEC="$STORM_HOME/bin/storm" + + SAMOA_STORM_PROPERTIES="samoa-storm.properties" + MODE_OPTION="samoa.storm.mode" + NUM_WORKER_OPTION="samoa.storm.numworker" + + VALUE="" + getvalue() + { + VALUE=`sed '/^\#/d' $BASE_DIR/$SAMOA_STORM_PROPERTIES | grep "$1" | tail -n 1 | cut -d "=" -f2- | sed 's/^[[:space:]]*//;s/[[:space:]]*$//'` + } + + getvalue "$NUM_WORKER_OPTION" + NUM_WORKER="$VALUE" + + getvalue "$MODE_OPTION" + MODE_ARG="$VALUE" + + COMPLETE_ARG="" + COUNTER=0 + for var in "$@" + do + COUNTER=`expr $COUNTER + 1` + if [ $COUNTER -gt 2 ];then + COMPLETE_ARG="$COMPLETE_ARG $var" + fi + done + + DEPLOYABLE=$JAR_PATH + if [ "$MODE_ARG" = "cluster" ]; then + $STORM_EXEC jar $DEPLOYABLE com.yahoo.labs.samoa.topology.impl.StormDoTask $COMPLETE_ARG $NUM_WORKER $MODE_ARG + elif [ "$MODE_ARG" = "local" ]; then + STORM_JAR=$(ls $STORM_HOME/storm-*.jar) + CLASSPATH="$CLASSPATH:$STORM_HOME/lib/*:$STORM_JAR:$DEPLOYABLE" + java -cp $CLASSPATH com.yahoo.labs.samoa.LocalStormDoTask $COMPLETE_ARG $NUM_WORKER + fi + +elif [ $PLATFORM = 'SAMZA' ]; then + echo "Deploying to SAMZA" + + SAMOA_SAMZA_PROPERTIES=samoa-samza.properties + MODE_OPTION='samoa.samza.mode' + ZOOKEEPER_HOST_OPTION='zookeeper.host' + ZOOKEEPER_PORT_OPTION='zookeeper.port' + KAFKA_BROKER_LIST_OPTION='kafka.broker.list' + KAFKA_REPLICATION_OPTION='kafka.replication.factor' + CHECKPOINT_FREQ_OPTION='checkpoint.commit.ms' + + YARN_AM_MEMORY_OPTION='yarn.am.memory' + YARN_CONTAINER_MEMORY_OPTION='yarn.container.memory' + PI_PER_CONTAINER_OPTION='max.pi.per.container' + + KRYO_REGISTER_OPTION='kryo.register.file' + + HDFS_SAMOA_HOME_OPTION='hdfs.samoa.dir' + + VALUE="" + getvalue() + { + VALUE=`sed '/^\#/d' $BASE_DIR/$SAMOA_SAMZA_PROPERTIES | grep "$1" | tail -n 1 | cut -d "=" -f2- | sed 's/^[[:space:]]*//;s/[[:space:]]*$//'` + } + + getvalue "$ZOOKEEPER_HOST_OPTION" + ZOOKEEPER_HOST="$VALUE" + + getvalue "$ZOOKEEPER_PORT_OPTION" + ZOOKEEPER_PORT="$VALUE" + + getvalue "$KAFKA_BROKER_LIST_OPTION" + KAFKA_BROKER_LIST="$VALUE" + + getvalue "$MODE_OPTION" + MODE_ARG="$VALUE" + + getvalue "$KAFKA_REPLICATION_OPTION" + KAFKA_REPLICATION_FACTOR="$VALUE" + + getvalue "$CHECKPOINT_FREQ_OPTION" + CHECKPOINT_FREQUENCY="$VALUE" + + getvalue "$YARN_AM_MEMORY_OPTION" + YARN_AM_MEMORY="$VALUE" + + getvalue "$YARN_CONTAINER_MEMORY_OPTION" + YARN_CONTAINER_MEMORY="$VALUE" + + getvalue "$KRYO_REGISTER_OPTION" + KRYO_REGISTER_FILE="$VALUE" + + getvalue "$PI_PER_CONTAINER_OPTION" + PI_PER_CONTAINER="$VALUE" + + getvalue "$HDFS_SAMOA_HOME_OPTION" + HDFS_SAMOA_HOME="$VALUE" + + COMPLETE_ARG="" + COUNTER=0 + for var in "$@" + do + COUNTER=`expr $COUNTER + 1` + if [ $COUNTER -gt 2 ];then + COMPLETE_ARG="$COMPLETE_ARG $var" + fi + done + + DEPLOYABLE=$JAR_PATH + java -Dsamza.log.dir=$BASE_DIR/logs -Dsamza.container.name=client -cp $DEPLOYABLE com.yahoo.labs.samoa.SamzaDoTask $COMPLETE_ARG --mode=$MODE_ARG \ + --yarn_home=$YARN_HOME/conf --zookeeper=$ZOOKEEPER_HOST:$ZOOKEEPER_PORT --kafka=$KAFKA_BROKER_LIST \ + --jar_package=$JAR_PATH --yarn_am_mem=$YARN_AM_MEMORY --yarn_container_mem=$YARN_CONTAINER_MEMORY \ + --kafka_replication_factor=$KAFKA_REPLICATION_FACTOR --checkpoint_frequency=$CHECKPOINT_FREQUENCY \ + --kryo_register=$BASE_DIR/$KRYO_REGISTER_FILE --pi_per_container=$PI_PER_CONTAINER \ + --samoa_hdfs_dir=$HDFS_SAMOA_HOME + +elif [ $PLATFORM = 'THREADS' ]; then + + echo "Deploying to LOCAL with MULTITHREADING." + COMPLETE_ARG="" + COUNTER=0 + for var in "$@" + do + COUNTER=`expr $COUNTER + 1` + if [ $COUNTER -gt 2 ];then + COMPLETE_ARG="$COMPLETE_ARG $var" + fi + done + + java $JAVA_OPTS -cp $JAR_PATH com.yahoo.labs.samoa.LocalThreadsDoTask $COMPLETE_ARG + +elif [ $PLATFORM = 'LOCAL' ]; then + + echo "Deploying to $PLATFORM" + COMPLETE_ARG="" + COUNTER=0 + for var in "$@" + do + COUNTER=`expr $COUNTER + 1` + if [ $COUNTER -gt 2 ];then + COMPLETE_ARG="$COMPLETE_ARG $var" + fi + done + + java $JAVA_OPTS -cp $JAR_PATH com.yahoo.labs.samoa.LocalDoTask $COMPLETE_ARG $NUM_WORKER $MODE_ARG +else + echo "Specify a valid platform." + echo "Usage: samoa <platform> <jar> <task & options>" +fi + http://git-wip-us.apache.org/repos/asf/incubator-samoa/blob/787864b6/bin/samoa-s4.properties ---------------------------------------------------------------------- diff --git a/bin/samoa-s4.properties b/bin/samoa-s4.properties new file mode 100644 index 0000000..f31f867 --- /dev/null +++ b/bin/samoa-s4.properties @@ -0,0 +1,46 @@ +### +# #%L +# SAMOA +# %% +# Copyright (C) 2013 Yahoo! Inc. +# %% +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# #L% +### + +# ===================================================== # +# SAMOA S4 properties file # +# ----------------------------------------------------- # +# This file contains specific configuration for # +# the deployment in the S4 platform. # +# ===================================================== # + +# Zookeeper Server +zookeeper.server=localhost +zookeeper.port=2181 + +# Simple HTTP Server prociding the packaged S4 jar +#http.server.ip=localhost +http.server.port=8000 + +# Name of the S4 cluster +cluster.name=cluster +cluster.port=12000 + +# Deployment strategy: local or cluster +samoa.deploy.mode=local + +# Directory for storing the results of the algorithms. +results.dir=/tmp/samoa/results +# Directory for storing the evaluation results, if the algorithms are to be evaluated. +evaluation.dir=/tmp/samoa/evaluation http://git-wip-us.apache.org/repos/asf/incubator-samoa/blob/787864b6/bin/samoa-samza.properties ---------------------------------------------------------------------- diff --git a/bin/samoa-samza.properties b/bin/samoa-samza.properties new file mode 100644 index 0000000..db7804f --- /dev/null +++ b/bin/samoa-samza.properties @@ -0,0 +1,67 @@ +### +# #%L +# SAMOA +# %% +# Copyright (C) 2013 Yahoo! Inc. +# %% +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# #L% +### + +# SAMOA Storm properties file +# This file contains specific configurations for SAMOA deployment in the Storm platform +# Note that you still need to configure Storm client in your machine, +# including setting up Storm configuration file (~/.storm/storm.yaml) with correct settings + +# samoa.samza.mode corresponds to the execution mode of the Task in Storm +# possible values: +# 1. yarn: the Task will be sent into nimbus. The nimbus is configured by Storm configuration file +# 2. local: the Task will be sent using local Storm cluster +samoa.samza.mode=yarn + +# yarn.am.memory : memory (in MB) to be allocated for ApplicationMaster on YARN cluster +# yarn.container.memory : memory (in MB) to be allocated for each worker +# containers on YARN cluster +yarn.am.memory=1024 +yarn.container.memory=1024 + +# max.pi.per.container : maximum number of processing item instances allocated +# to a worker container +# e.g.: if this valua is 2 and there is a ProcessingItem with parallelism of 5. +# The 5 instances of that ProcessingItem will be running on 3 containers (2, 2, and 1) +max.pi.per.container=1 + +# zookeeper.host : host URL of zookeeper server +# zookeeper.port : the port on which zookeeper server listen to client request +zookeeper.host=localhost +zookeeper.port=2181 + +# kafka.broker.list : comma separated list of host:port of brokers in kafka cluster +kafka.broker.list=localhost:9092 + +# kryo.register.file : the file contain the list of +# class-name:serializer-name +# that needs to be registered with kryo +kryo.register.file=samza-kryo + +# kafka.replication.factor : replication factor for kafka streams +# or the number of brokers to handle a kafka stream +# a possitive integer value +kafka.replication.factor=1 + +# checkpoint.commit.ms : frequency to commit a checkpoint (in ms) +checkpoint.commit.ms=60000 + +# hdfs.samoa.dir : SAMOA home directory in HDFS +# execution JAR, serialization files will be stored here +#hdfs.samoa.dir=/.samoa http://git-wip-us.apache.org/repos/asf/incubator-samoa/blob/787864b6/bin/samoa-storm.properties ---------------------------------------------------------------------- diff --git a/bin/samoa-storm.properties b/bin/samoa-storm.properties new file mode 100644 index 0000000..7feed28 --- /dev/null +++ b/bin/samoa-storm.properties @@ -0,0 +1,35 @@ +### +# #%L +# SAMOA +# %% +# Copyright (C) 2013 Yahoo! Inc. +# %% +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# #L% +### + +# SAMOA Storm properties file +# This file contains specific configurations for SAMOA deployment in the Storm platform +# Note that you still need to configure Storm client in your machine, +# including setting up Storm configuration file (~/.storm/storm.yaml) with correct settings + +# samoa.storm.mode corresponds to the execution mode of the Task in Storm +# possible values: +# 1. cluster: the Task will be sent into nimbus. The nimbus is configured by Storm configuration file +# 2. local: the Task will be sent using local Storm cluster +samoa.storm.mode=local + +# samoa.storm.numworker corresponds to the number of worker processes allocated in Storm cluster +# possible values: any integer greater than 0 +samoa.storm.numworker=4 + http://git-wip-us.apache.org/repos/asf/incubator-samoa/blob/787864b6/bin/samza-dist/run-am.sh ---------------------------------------------------------------------- diff --git a/bin/samza-dist/run-am.sh b/bin/samza-dist/run-am.sh new file mode 100644 index 0000000..ad4475f --- /dev/null +++ b/bin/samza-dist/run-am.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +### +# #%L +# SAMOA +# %% +# Copyright (C) 2013 - 2014 Yahoo! Inc. +# %% +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# #L% +### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +exec $(dirname $0)/run-class.sh org.apache.samza.job.yarn.SamzaAppMaster $@ http://git-wip-us.apache.org/repos/asf/incubator-samoa/blob/787864b6/bin/samza-dist/run-class.sh ---------------------------------------------------------------------- diff --git a/bin/samza-dist/run-class.sh b/bin/samza-dist/run-class.sh new file mode 100644 index 0000000..b6a1e79 --- /dev/null +++ b/bin/samza-dist/run-class.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +### +# #%L +# SAMOA +# %% +# Copyright (C) 2013 - 2014 Yahoo! Inc. +# %% +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# #L% +### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +if [ $# -lt 1 ]; +then + echo "USAGE: $0 classname [opts]" + exit 1 +fi + +home_dir=`pwd` +base_dir=$(dirname $0)/.. +cd $base_dir +base_dir=`pwd` +cd $home_dir + +YARN_HOME="${YARN_HOME:-$HOME/.samza}" +CLASSPATH=$base_dir:$YARN_HOME/conf + +if [ -z "$JAVA_HOME" ]; then + JAVA="java" +else + JAVA="$JAVA_HOME/bin/java" +fi + +if [ -z "$SAMZA_LOG_DIR" ]; then + SAMZA_LOG_DIR="$base_dir" +fi + +if [ -z "$SAMZA_CONTAINER_NAME" ]; then + SAMZA_CONTAINER_NAME="undefined-samza-container-name" +fi + +if [ -z "$JAVA_OPTS" ]; then + JAVA_OPTS="-Xmx768M -XX:+PrintGCDateStamps" +fi + +if [ -f $base_dir/lib/log4j.xml ]; then + JAVA_OPTS="$JAVA_OPTS -Dlog4j.configuration=file:$base_dir/log4j.xml" +fi + +JAVA_OPTS="$JAVA_OPTS -Xloggc:$SAMZA_LOG_DIR/gc.log -Dsamza.log.dir=$SAMZA_LOG_DIR -Dsamza.container.name=$SAMZA_CONTAINER_NAME" + +echo $JAVA $JAVA_OPTS -cp $CLASSPATH $@ +exec $JAVA $JAVA_OPTS -cp $CLASSPATH $@ http://git-wip-us.apache.org/repos/asf/incubator-samoa/blob/787864b6/bin/samza-dist/run-container.sh ---------------------------------------------------------------------- diff --git a/bin/samza-dist/run-container.sh b/bin/samza-dist/run-container.sh new file mode 100644 index 0000000..c46bf77 --- /dev/null +++ b/bin/samza-dist/run-container.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +### +# #%L +# SAMOA +# %% +# Copyright (C) 2013 - 2014 Yahoo! Inc. +# %% +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# #L% +### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +exec $(dirname $0)/run-class.sh org.apache.samza.container.SamzaContainer $@ http://git-wip-us.apache.org/repos/asf/incubator-samoa/blob/787864b6/bin/samza-dist/run-job.sh ---------------------------------------------------------------------- diff --git a/bin/samza-dist/run-job.sh b/bin/samza-dist/run-job.sh new file mode 100644 index 0000000..e77c9af --- /dev/null +++ b/bin/samza-dist/run-job.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +### +# #%L +# SAMOA +# %% +# Copyright (C) 2013 - 2014 Yahoo! Inc. +# %% +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# #L% +### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +exec $(dirname $0)/run-class.sh org.apache.samza.job.JobRunner $@ http://git-wip-us.apache.org/repos/asf/incubator-samoa/blob/787864b6/bin/samza-kryo ---------------------------------------------------------------------- diff --git a/bin/samza-kryo b/bin/samza-kryo new file mode 100644 index 0000000..2857a40 --- /dev/null +++ b/bin/samza-kryo @@ -0,0 +1,6 @@ +com.yahoo.labs.samoa.learners.classifiers.trees.AttributeContentEvent:com.yahoo.labs.samoa.learners.classifiers.trees.AttributeContentEvent$AttributeCEFullPrecSerializer +com.yahoo.labs.samoa.learners.classifiers.trees.ComputeContentEvent:com.yahoo.labs.samoa.learners.classifiers.trees.ComputeContentEvent$ComputeCEFullPrecSerializer +com.yahoo.labs.samoa.moa.classifiers.core.AttributeSplitSuggestion:com.yahoo.labs.samoa.utils.SerializableSerializer + +com.yahoo.labs.samoa.learners.classifiers.rules.common.TargetMean:com.yahoo.labs.samoa.learners.classifiers.rules.common.TargetMean$TargetMeanSerializer +com.yahoo.labs.samoa.learners.classifiers.rules.common.Perceptron:com.yahoo.labs.samoa.learners.classifiers.rules.common.Perceptron$PerceptronSerializer
![http://yahoo.github.io/samoa/Manual.png"](http://yahoo.github.io/samoa/Manual.png"){kind=link}