rename all except examples
Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/4f03388e Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/4f03388e Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/4f03388e Branch: refs/heads/develop Commit: 4f03388ef551133149603933bba8a6eac30fd333 Parents: ec79f20 Author: Xusen Yin <[email protected]> Authored: Wed Jul 6 22:14:00 2016 -0700 Committer: Xusen Yin <[email protected]> Committed: Wed Jul 6 22:14:00 2016 -0700 ---------------------------------------------------------------------- .../io/prediction/annotation/DeveloperApi.java | 34 - .../io/prediction/annotation/Experimental.java | 35 - .../authentication/KeyAuthentication.scala | 55 - .../configuration/SSLConfiguration.scala | 71 - .../predictionio/annotation/DeveloperApi.java | 34 + .../predictionio/annotation/Experimental.java | 35 + .../authentication/KeyAuthentication.scala | 55 + .../configuration/SSLConfiguration.scala | 71 + .../controller/CustomQuerySerializer.scala | 37 - .../io/prediction/controller/Deployment.scala | 56 - .../scala/io/prediction/controller/Engine.scala | 829 ------------ .../prediction/controller/EngineFactory.scala | 41 - .../io/prediction/controller/EngineParams.scala | 149 -- .../controller/EngineParamsGenerator.scala | 43 - .../io/prediction/controller/Evaluation.scala | 122 -- .../prediction/controller/FastEvalEngine.scala | 343 ----- .../controller/IdentityPreparator.scala | 92 -- .../io/prediction/controller/LAlgorithm.scala | 130 -- .../prediction/controller/LAverageServing.scala | 41 - .../io/prediction/controller/LDataSource.scala | 67 - .../prediction/controller/LFirstServing.scala | 39 - .../io/prediction/controller/LPreparator.scala | 46 - .../io/prediction/controller/LServing.scala | 52 - .../LocalFileSystemPersistentModel.scala | 74 - .../scala/io/prediction/controller/Metric.scala | 266 ---- .../prediction/controller/MetricEvaluator.scala | 260 ---- .../io/prediction/controller/P2LAlgorithm.scala | 121 -- .../io/prediction/controller/PAlgorithm.scala | 126 -- .../io/prediction/controller/PDataSource.scala | 57 - .../io/prediction/controller/PPreparator.scala | 44 - .../scala/io/prediction/controller/Params.scala | 31 - .../prediction/controller/PersistentModel.scala | 112 -- .../io/prediction/controller/SanityCheck.scala | 30 - .../scala/io/prediction/controller/Utils.scala | 69 - .../java/JavaEngineParamsGenerator.scala | 39 - .../controller/java/JavaEvaluation.scala | 66 - .../controller/java/LJavaAlgorithm.scala | 31 - .../controller/java/LJavaDataSource.scala | 31 - .../controller/java/LJavaPreparator.scala | 29 - .../controller/java/LJavaServing.scala | 26 - .../controller/java/P2LJavaAlgorithm.scala | 33 - .../controller/java/PJavaAlgorithm.scala | 28 - .../controller/java/PJavaDataSource.scala | 28 - .../controller/java/PJavaPreparator.scala | 26 - .../java/SerializableComparator.scala | 20 - .../io/prediction/controller/package.scala | 168 --- .../scala/io/prediction/core/AbstractDoer.scala | 66 - .../io/prediction/core/BaseAlgorithm.scala | 123 -- .../io/prediction/core/BaseDataSource.scala | 52 - .../scala/io/prediction/core/BaseEngine.scala | 100 -- .../io/prediction/core/BaseEvaluator.scala | 72 - .../io/prediction/core/BasePreparator.scala | 42 - .../scala/io/prediction/core/BaseServing.scala | 51 - .../main/scala/io/prediction/core/package.scala | 21 - core/src/main/scala/io/prediction/package.scala | 19 - .../io/prediction/workflow/CoreWorkflow.scala | 163 --- .../io/prediction/workflow/CreateServer.scala | 737 ---------- .../io/prediction/workflow/CreateWorkflow.scala | 274 ---- .../workflow/EngineServerPlugin.scala | 40 - .../workflow/EngineServerPluginContext.scala | 88 -- .../workflow/EngineServerPluginsActor.scala | 46 - .../workflow/EvaluationWorkflow.scala | 42 - .../io/prediction/workflow/FakeWorkflow.scala | 106 -- .../io/prediction/workflow/JsonExtractor.scala | 164 --- .../workflow/JsonExtractorOption.scala | 23 - .../workflow/PersistentModelManifest.scala | 18 - .../scala/io/prediction/workflow/Workflow.scala | 135 -- .../prediction/workflow/WorkflowContext.scala | 45 - .../io/prediction/workflow/WorkflowParams.scala | 42 - .../io/prediction/workflow/WorkflowUtils.scala | 419 ------ .../controller/CustomQuerySerializer.scala | 37 + .../predictionio/controller/Deployment.scala | 56 + .../apache/predictionio/controller/Engine.scala | 829 ++++++++++++ .../predictionio/controller/EngineFactory.scala | 41 + .../predictionio/controller/EngineParams.scala | 149 ++ .../controller/EngineParamsGenerator.scala | 43 + .../predictionio/controller/Evaluation.scala | 122 ++ .../controller/FastEvalEngine.scala | 343 +++++ .../controller/IdentityPreparator.scala | 92 ++ .../predictionio/controller/LAlgorithm.scala | 130 ++ .../controller/LAverageServing.scala | 41 + .../predictionio/controller/LDataSource.scala | 67 + .../predictionio/controller/LFirstServing.scala | 39 + .../predictionio/controller/LPreparator.scala | 46 + .../predictionio/controller/LServing.scala | 52 + .../LocalFileSystemPersistentModel.scala | 74 + .../apache/predictionio/controller/Metric.scala | 266 ++++ .../controller/MetricEvaluator.scala | 260 ++++ .../predictionio/controller/P2LAlgorithm.scala | 121 ++ .../predictionio/controller/PAlgorithm.scala | 126 ++ .../predictionio/controller/PDataSource.scala | 57 + .../predictionio/controller/PPreparator.scala | 44 + .../apache/predictionio/controller/Params.scala | 31 + .../controller/PersistentModel.scala | 112 ++ .../predictionio/controller/SanityCheck.scala | 30 + .../apache/predictionio/controller/Utils.scala | 69 + .../java/JavaEngineParamsGenerator.scala | 39 + .../controller/java/JavaEvaluation.scala | 66 + .../controller/java/LJavaAlgorithm.scala | 31 + .../controller/java/LJavaDataSource.scala | 31 + .../controller/java/LJavaPreparator.scala | 29 + .../controller/java/LJavaServing.scala | 26 + .../controller/java/P2LJavaAlgorithm.scala | 33 + .../controller/java/PJavaAlgorithm.scala | 28 + .../controller/java/PJavaDataSource.scala | 28 + .../controller/java/PJavaPreparator.scala | 26 + .../java/SerializableComparator.scala | 20 + .../predictionio/controller/package.scala | 168 +++ .../apache/predictionio/core/AbstractDoer.scala | 66 + .../predictionio/core/BaseAlgorithm.scala | 123 ++ .../predictionio/core/BaseDataSource.scala | 52 + .../apache/predictionio/core/BaseEngine.scala | 100 ++ .../predictionio/core/BaseEvaluator.scala | 72 + .../predictionio/core/BasePreparator.scala | 42 + .../apache/predictionio/core/BaseServing.scala | 51 + .../org/apache/predictionio/core/package.scala | 21 + .../scala/org/apache/predictionio/package.scala | 19 + .../predictionio/workflow/CoreWorkflow.scala | 163 +++ .../predictionio/workflow/CreateServer.scala | 737 ++++++++++ .../predictionio/workflow/CreateWorkflow.scala | 274 ++++ .../workflow/EngineServerPlugin.scala | 40 + .../workflow/EngineServerPluginContext.scala | 88 ++ .../workflow/EngineServerPluginsActor.scala | 46 + .../workflow/EvaluationWorkflow.scala | 42 + .../predictionio/workflow/FakeWorkflow.scala | 106 ++ .../predictionio/workflow/JsonExtractor.scala | 164 +++ .../workflow/JsonExtractorOption.scala | 23 + .../workflow/PersistentModelManifest.scala | 18 + .../apache/predictionio/workflow/Workflow.scala | 135 ++ .../predictionio/workflow/WorkflowContext.scala | 45 + .../predictionio/workflow/WorkflowParams.scala | 42 + .../predictionio/workflow/WorkflowUtils.scala | 419 ++++++ .../controller/metric_evaluator.scala.html | 95 -- .../io/prediction/workflow/index.scala.html | 92 -- .../controller/metric_evaluator.scala.html | 95 ++ .../predictionio/workflow/index.scala.html | 92 ++ .../java/io/prediction/workflow/JavaParams.java | 30 - .../java/io/prediction/workflow/JavaQuery.java | 46 - .../workflow/JavaQueryTypeAdapterFactory.java | 60 - .../predictionio/workflow/JavaParams.java | 30 + .../apache/predictionio/workflow/JavaQuery.java | 46 + .../workflow/JavaQueryTypeAdapterFactory.java | 60 + .../io/prediction/controller/EngineTest.scala | 615 --------- .../prediction/controller/EvaluationTest.scala | 46 - .../prediction/controller/EvaluatorTest.scala | 93 -- .../controller/FastEvalEngineTest.scala | 181 --- .../controller/MetricEvaluatorTest.scala | 52 - .../io/prediction/controller/MetricTest.scala | 143 -- .../io/prediction/controller/SampleEngine.scala | 472 ------- .../scala/io/prediction/workflow/BaseTest.scala | 75 - .../workflow/EngineWorkflowTest.scala | 0 .../workflow/EvaluationWorkflowTest.scala | 61 - .../workflow/JsonExtractorSuite.scala | 383 ------ .../predictionio/controller/EngineTest.scala | 615 +++++++++ .../controller/EvaluationTest.scala | 46 + .../predictionio/controller/EvaluatorTest.scala | 93 ++ .../controller/FastEvalEngineTest.scala | 181 +++ .../controller/MetricEvaluatorTest.scala | 52 + .../predictionio/controller/MetricTest.scala | 143 ++ .../predictionio/controller/SampleEngine.scala | 472 +++++++ .../apache/predictionio/workflow/BaseTest.scala | 75 + .../workflow/EngineWorkflowTest.scala | 0 .../workflow/EvaluationWorkflowTest.scala | 61 + .../workflow/JsonExtractorSuite.scala | 383 ++++++ .../main/scala/io/prediction/data/Utils.scala | 50 - .../scala/io/prediction/data/api/Common.scala | 80 -- .../io/prediction/data/api/EventInfo.scala | 24 - .../io/prediction/data/api/EventServer.scala | 640 --------- .../prediction/data/api/EventServerPlugin.scala | 33 - .../data/api/EventServerPluginContext.scala | 49 - .../io/prediction/data/api/PluginsActor.scala | 52 - .../scala/io/prediction/data/api/Stats.scala | 79 -- .../io/prediction/data/api/StatsActor.scala | 74 - .../scala/io/prediction/data/api/Webhooks.scala | 151 --- .../data/api/WebhooksConnectors.scala | 34 - .../main/scala/io/prediction/data/package.scala | 21 - .../io/prediction/data/storage/AccessKeys.scala | 71 - .../scala/io/prediction/data/storage/Apps.scala | 58 - .../io/prediction/data/storage/BiMap.scala | 164 --- .../io/prediction/data/storage/Channels.scala | 79 -- .../io/prediction/data/storage/DataMap.scala | 241 ---- .../data/storage/DateTimeJson4sSupport.scala | 47 - .../data/storage/EngineInstances.scala | 177 --- .../data/storage/EngineManifests.scala | 117 -- .../io/prediction/data/storage/EntityMap.scala | 98 -- .../data/storage/EvaluationInstances.scala | 135 -- .../io/prediction/data/storage/Event.scala | 164 --- .../data/storage/EventJson4sSupport.scala | 236 ---- .../data/storage/LEventAggregator.scala | 145 -- .../io/prediction/data/storage/LEvents.scala | 489 ------- .../io/prediction/data/storage/Models.scala | 80 -- .../data/storage/PEventAggregator.scala | 209 --- .../io/prediction/data/storage/PEvents.scala | 182 --- .../prediction/data/storage/PropertyMap.scala | 96 -- .../io/prediction/data/storage/Storage.scala | 403 ------ .../io/prediction/data/storage/Utils.scala | 47 - .../storage/elasticsearch/ESAccessKeys.scala | 116 -- .../data/storage/elasticsearch/ESApps.scala | 127 -- .../data/storage/elasticsearch/ESChannels.scala | 114 -- .../elasticsearch/ESEngineInstances.scala | 155 --- .../elasticsearch/ESEngineManifests.scala | 81 -- .../elasticsearch/ESEvaluationInstances.scala | 133 -- .../storage/elasticsearch/ESSequences.scala | 61 - .../data/storage/elasticsearch/ESUtils.scala | 45 - .../storage/elasticsearch/StorageClient.scala | 47 - .../data/storage/elasticsearch/package.scala | 22 - .../data/storage/hbase/HBEventsUtil.scala | 412 ------ .../data/storage/hbase/HBLEvents.scala | 192 --- .../data/storage/hbase/HBPEvents.scala | 112 -- .../data/storage/hbase/PIOHBaseUtil.scala | 28 - .../data/storage/hbase/StorageClient.scala | 83 -- .../prediction/data/storage/hbase/package.scala | 22 - .../data/storage/hbase/upgrade/HB_0_8_0.scala | 190 --- .../data/storage/hbase/upgrade/Upgrade.scala | 72 - .../storage/hbase/upgrade/Upgrade_0_8_3.scala | 221 --- .../data/storage/hdfs/HDFSModels.scala | 60 - .../data/storage/hdfs/StorageClient.scala | 33 - .../prediction/data/storage/hdfs/package.scala | 22 - .../data/storage/jdbc/JDBCAccessKeys.scala | 84 -- .../prediction/data/storage/jdbc/JDBCApps.scala | 86 -- .../data/storage/jdbc/JDBCChannels.scala | 66 - .../data/storage/jdbc/JDBCEngineInstances.scala | 194 --- .../data/storage/jdbc/JDBCEngineManifests.scala | 111 -- .../storage/jdbc/JDBCEvaluationInstances.scala | 162 --- .../data/storage/jdbc/JDBCLEvents.scala | 241 ---- .../data/storage/jdbc/JDBCModels.scala | 52 - .../data/storage/jdbc/JDBCPEvents.scala | 160 --- .../data/storage/jdbc/JDBCUtils.scala | 103 -- .../data/storage/jdbc/StorageClient.scala | 50 - .../prediction/data/storage/jdbc/package.scala | 23 - .../data/storage/localfs/LocalFSModels.scala | 59 - .../data/storage/localfs/StorageClient.scala | 43 - .../data/storage/localfs/package.scala | 22 - .../io/prediction/data/storage/package.scala | 26 - .../scala/io/prediction/data/store/Common.scala | 50 - .../io/prediction/data/store/LEventStore.scala | 142 -- .../io/prediction/data/store/PEventStore.scala | 116 -- .../data/store/java/LJavaEventStore.scala | 142 -- .../data/store/java/OptionHelper.scala | 29 - .../data/store/java/PJavaEventStore.scala | 109 -- .../io/prediction/data/store/package.scala | 21 - .../io/prediction/data/view/DataView.scala | 110 -- .../io/prediction/data/view/LBatchView.scala | 200 --- .../io/prediction/data/view/PBatchView.scala | 209 --- .../io/prediction/data/view/QuickTest.scala | 94 -- .../data/webhooks/ConnectorException.scala | 31 - .../data/webhooks/ConnectorUtil.scala | 46 - .../data/webhooks/FormConnector.scala | 32 - .../data/webhooks/JsonConnector.scala | 31 - .../exampleform/ExampleFormConnector.scala | 123 -- .../examplejson/ExampleJsonConnector.scala | 153 --- .../webhooks/mailchimp/MailChimpConnector.scala | 305 ----- .../webhooks/segmentio/SegmentIOConnector.scala | 306 ----- .../org/apache/predictionio/data/Utils.scala | 50 + .../apache/predictionio/data/api/Common.scala | 80 ++ .../predictionio/data/api/EventInfo.scala | 24 + .../predictionio/data/api/EventServer.scala | 640 +++++++++ .../data/api/EventServerPlugin.scala | 33 + .../data/api/EventServerPluginContext.scala | 49 + .../predictionio/data/api/PluginsActor.scala | 52 + .../apache/predictionio/data/api/Stats.scala | 79 ++ .../predictionio/data/api/StatsActor.scala | 74 + .../apache/predictionio/data/api/Webhooks.scala | 151 +++ .../data/api/WebhooksConnectors.scala | 34 + .../org/apache/predictionio/data/package.scala | 21 + .../predictionio/data/storage/AccessKeys.scala | 71 + .../apache/predictionio/data/storage/Apps.scala | 58 + .../predictionio/data/storage/BiMap.scala | 164 +++ .../predictionio/data/storage/Channels.scala | 79 ++ .../predictionio/data/storage/DataMap.scala | 241 ++++ .../data/storage/DateTimeJson4sSupport.scala | 47 + .../data/storage/EngineInstances.scala | 177 +++ .../data/storage/EngineManifests.scala | 117 ++ .../predictionio/data/storage/EntityMap.scala | 98 ++ .../data/storage/EvaluationInstances.scala | 135 ++ .../predictionio/data/storage/Event.scala | 164 +++ .../data/storage/EventJson4sSupport.scala | 236 ++++ .../data/storage/LEventAggregator.scala | 145 ++ .../predictionio/data/storage/LEvents.scala | 489 +++++++ .../predictionio/data/storage/Models.scala | 80 ++ .../data/storage/PEventAggregator.scala | 209 +++ .../predictionio/data/storage/PEvents.scala | 182 +++ .../predictionio/data/storage/PropertyMap.scala | 96 ++ .../predictionio/data/storage/Storage.scala | 403 ++++++ .../predictionio/data/storage/Utils.scala | 47 + .../storage/elasticsearch/ESAccessKeys.scala | 116 ++ .../data/storage/elasticsearch/ESApps.scala | 127 ++ .../data/storage/elasticsearch/ESChannels.scala | 114 ++ .../elasticsearch/ESEngineInstances.scala | 155 +++ .../elasticsearch/ESEngineManifests.scala | 81 ++ .../elasticsearch/ESEvaluationInstances.scala | 133 ++ .../storage/elasticsearch/ESSequences.scala | 61 + .../data/storage/elasticsearch/ESUtils.scala | 45 + .../storage/elasticsearch/StorageClient.scala | 47 + .../data/storage/elasticsearch/package.scala | 22 + .../data/storage/hbase/HBEventsUtil.scala | 412 ++++++ .../data/storage/hbase/HBLEvents.scala | 192 +++ .../data/storage/hbase/HBPEvents.scala | 112 ++ .../data/storage/hbase/PIOHBaseUtil.scala | 28 + .../data/storage/hbase/StorageClient.scala | 83 ++ .../data/storage/hbase/package.scala | 22 + .../data/storage/hbase/upgrade/HB_0_8_0.scala | 190 +++ .../data/storage/hbase/upgrade/Upgrade.scala | 72 + .../storage/hbase/upgrade/Upgrade_0_8_3.scala | 221 +++ .../data/storage/hdfs/HDFSModels.scala | 60 + .../data/storage/hdfs/StorageClient.scala | 33 + .../data/storage/hdfs/package.scala | 22 + .../data/storage/jdbc/JDBCAccessKeys.scala | 84 ++ .../data/storage/jdbc/JDBCApps.scala | 86 ++ .../data/storage/jdbc/JDBCChannels.scala | 66 + .../data/storage/jdbc/JDBCEngineInstances.scala | 194 +++ .../data/storage/jdbc/JDBCEngineManifests.scala | 111 ++ .../storage/jdbc/JDBCEvaluationInstances.scala | 162 +++ .../data/storage/jdbc/JDBCLEvents.scala | 241 ++++ .../data/storage/jdbc/JDBCModels.scala | 52 + .../data/storage/jdbc/JDBCPEvents.scala | 160 +++ .../data/storage/jdbc/JDBCUtils.scala | 103 ++ .../data/storage/jdbc/StorageClient.scala | 50 + .../data/storage/jdbc/package.scala | 23 + .../data/storage/localfs/LocalFSModels.scala | 59 + .../data/storage/localfs/StorageClient.scala | 43 + .../data/storage/localfs/package.scala | 22 + .../predictionio/data/storage/package.scala | 26 + .../apache/predictionio/data/store/Common.scala | 50 + .../predictionio/data/store/LEventStore.scala | 142 ++ .../predictionio/data/store/PEventStore.scala | 116 ++ .../data/store/java/LJavaEventStore.scala | 142 ++ .../data/store/java/OptionHelper.scala | 29 + .../data/store/java/PJavaEventStore.scala | 109 ++ .../predictionio/data/store/package.scala | 21 + .../predictionio/data/view/DataView.scala | 110 ++ .../predictionio/data/view/LBatchView.scala | 200 +++ .../predictionio/data/view/PBatchView.scala | 209 +++ .../predictionio/data/view/QuickTest.scala | 94 ++ .../data/webhooks/ConnectorException.scala | 31 + .../data/webhooks/ConnectorUtil.scala | 46 + .../data/webhooks/FormConnector.scala | 32 + .../data/webhooks/JsonConnector.scala | 31 + .../exampleform/ExampleFormConnector.scala | 123 ++ .../examplejson/ExampleJsonConnector.scala | 153 +++ .../webhooks/mailchimp/MailChimpConnector.scala | 305 +++++ .../webhooks/segmentio/SegmentIOConnector.scala | 306 +++++ .../prediction/data/api/EventServiceSpec.scala | 68 - .../prediction/data/api/SegmentIOAuthSpec.scala | 175 --- .../io/prediction/data/storage/BiMapSpec.scala | 196 --- .../prediction/data/storage/DataMapSpec.scala | 243 ---- .../data/storage/LEventAggregatorSpec.scala | 103 -- .../prediction/data/storage/LEventsSpec.scala | 245 ---- .../data/storage/PEventAggregatorSpec.scala | 72 - .../prediction/data/storage/PEventsSpec.scala | 210 --- .../data/storage/StorageTestUtils.scala | 42 - .../io/prediction/data/storage/TestEvents.scala | 263 ---- .../data/webhooks/ConnectorTestUtil.scala | 47 - .../exampleform/ExampleFormConnectorSpec.scala | 164 --- .../examplejson/ExampleJsonConnectorSpec.scala | 179 --- .../mailchimp/MailChimpConnectorSpec.scala | 254 ---- .../segmentio/SegmentIOConnectorSpec.scala | 335 ----- .../data/api/EventServiceSpec.scala | 68 + .../data/api/SegmentIOAuthSpec.scala | 175 +++ .../predictionio/data/storage/BiMapSpec.scala | 196 +++ .../predictionio/data/storage/DataMapSpec.scala | 243 ++++ .../data/storage/LEventAggregatorSpec.scala | 103 ++ .../predictionio/data/storage/LEventsSpec.scala | 245 ++++ .../data/storage/PEventAggregatorSpec.scala | 72 + .../predictionio/data/storage/PEventsSpec.scala | 210 +++ .../data/storage/StorageTestUtils.scala | 42 + .../predictionio/data/storage/TestEvents.scala | 263 ++++ .../data/webhooks/ConnectorTestUtil.scala | 47 + .../exampleform/ExampleFormConnectorSpec.scala | 164 +++ .../examplejson/ExampleJsonConnectorSpec.scala | 179 +++ .../mailchimp/MailChimpConnectorSpec.scala | 254 ++++ .../segmentio/SegmentIOConnectorSpec.scala | 335 +++++ .../prediction/e2/engine/BinaryVectorizer.scala | 61 - .../e2/engine/CategoricalNaiveBayes.scala | 176 --- .../io/prediction/e2/engine/MarkovChain.scala | 89 -- .../e2/evaluation/CrossValidation.scala | 64 - .../main/scala/io/prediction/e2/package.scala | 22 - e2/src/main/scala/io/prediction/package.scala | 21 - .../e2/engine/BinaryVectorizer.scala | 61 + .../e2/engine/CategoricalNaiveBayes.scala | 176 +++ .../predictionio/e2/engine/MarkovChain.scala | 89 ++ .../e2/evaluation/CrossValidation.scala | 64 + .../org/apache/predictionio/e2/package.scala | 22 + .../scala/org/apache/predictionio/package.scala | 21 + .../e2/engine/BinaryVectorizerTest.scala | 56 - .../e2/engine/CategoricalNaiveBayesTest.scala | 132 -- .../prediction/e2/engine/MarkovChainTest.scala | 49 - .../e2/evaluation/CrossValidationTest.scala | 111 -- .../e2/fixture/BinaryVectorizerFixture.scala | 59 - .../e2/fixture/MarkovChainFixture.scala | 39 - .../e2/fixture/NaiveBayesFixture.scala | 48 - .../e2/fixture/SharedSparkContext.scala | 51 - .../e2/engine/BinaryVectorizerTest.scala | 56 + .../e2/engine/CategoricalNaiveBayesTest.scala | 132 ++ .../e2/engine/MarkovChainTest.scala | 49 + .../e2/evaluation/CrossValidationTest.scala | 111 ++ .../e2/fixture/BinaryVectorizerFixture.scala | 59 + .../e2/fixture/MarkovChainFixture.scala | 39 + .../e2/fixture/NaiveBayesFixture.scala | 48 + .../e2/fixture/SharedSparkContext.scala | 51 + .../io/prediction/tools/RegisterEngine.scala | 84 -- .../scala/io/prediction/tools/RunServer.scala | 178 --- .../scala/io/prediction/tools/RunWorkflow.scala | 212 --- .../main/scala/io/prediction/tools/Runner.scala | 211 --- .../io/prediction/tools/admin/AdminAPI.scala | 156 --- .../prediction/tools/admin/CommandClient.scala | 160 --- .../scala/io/prediction/tools/admin/README.md | 161 --- .../io/prediction/tools/console/AccessKey.scala | 83 -- .../scala/io/prediction/tools/console/App.scala | 537 -------- .../io/prediction/tools/console/Console.scala | 1277 ------------------ .../io/prediction/tools/console/Export.scala | 42 - .../io/prediction/tools/console/Import.scala | 39 - .../io/prediction/tools/console/Template.scala | 429 ------ .../tools/dashboard/CorsSupport.scala | 75 - .../prediction/tools/dashboard/Dashboard.scala | 156 --- .../prediction/tools/export/EventsToFile.scala | 104 -- .../prediction/tools/imprt/FileToEvents.scala | 103 -- .../predictionio/tools/RegisterEngine.scala | 84 ++ .../apache/predictionio/tools/RunServer.scala | 178 +++ .../apache/predictionio/tools/RunWorkflow.scala | 212 +++ .../org/apache/predictionio/tools/Runner.scala | 211 +++ .../predictionio/tools/admin/AdminAPI.scala | 156 +++ .../tools/admin/CommandClient.scala | 160 +++ .../apache/predictionio/tools/admin/README.md | 161 +++ .../predictionio/tools/console/AccessKey.scala | 83 ++ .../apache/predictionio/tools/console/App.scala | 537 ++++++++ .../predictionio/tools/console/Console.scala | 1277 ++++++++++++++++++ .../predictionio/tools/console/Export.scala | 42 + .../predictionio/tools/console/Import.scala | 39 + .../predictionio/tools/console/Template.scala | 429 ++++++ .../tools/dashboard/CorsSupport.scala | 75 + .../tools/dashboard/Dashboard.scala | 156 +++ .../tools/export/EventsToFile.scala | 104 ++ .../predictionio/tools/imprt/FileToEvents.scala | 103 ++ .../tools/console/accesskey.scala.txt | 20 - .../tools/console/adminserver.scala.txt | 6 - .../io/prediction/tools/console/app.scala.txt | 74 - .../io/prediction/tools/console/build.scala.txt | 11 - .../tools/console/dashboard.scala.txt | 6 - .../prediction/tools/console/deploy.scala.txt | 29 - .../io/prediction/tools/console/eval.scala.txt | 10 - .../tools/console/eventserver.scala.txt | 8 - .../prediction/tools/console/export.scala.txt | 14 - .../io/prediction/tools/console/imprt.scala.txt | 12 - .../io/prediction/tools/console/main.scala.txt | 52 - .../io/prediction/tools/console/run.scala.txt | 17 - .../prediction/tools/console/status.scala.txt | 3 - .../prediction/tools/console/template.scala.txt | 25 - .../io/prediction/tools/console/train.scala.txt | 28 - .../prediction/tools/console/upgrade.scala.txt | 15 - .../prediction/tools/console/version.scala.txt | 3 - .../prediction/tools/dashboard/index.scala.html | 99 -- .../itemrank/params/algorithmsJson.scala.txt | 16 - .../itemrank/params/datasourceJson.scala.txt | 26 - .../itemrank/params/preparatorJson.scala.txt | 10 - .../itemrank/params/servingJson.scala.txt | 1 - .../itemrec/params/algorithmsJson.scala.txt | 15 - .../itemrec/params/datasourceJson.scala.txt | 26 - .../itemrec/params/preparatorJson.scala.txt | 10 - .../itemrec/params/servingJson.scala.txt | 1 - .../itemsim/params/algorithmsJson.scala.txt | 13 - .../itemsim/params/datasourceJson.scala.txt | 26 - .../itemsim/params/preparatorJson.scala.txt | 10 - .../itemsim/params/servingJson.scala.txt | 1 - .../tools/templates/scala/buildSbt.scala.txt | 14 - .../tools/templates/scala/engineJson.scala.txt | 9 - .../templates/scala/manifestJson.scala.txt | 9 - .../scala/project/assemblySbt.scala.txt | 1 - .../scala/src/main/scala/engine.scala.txt | 76 -- .../tools/console/accesskey.scala.txt | 20 + .../tools/console/adminserver.scala.txt | 6 + .../predictionio/tools/console/app.scala.txt | 74 + .../predictionio/tools/console/build.scala.txt | 11 + .../tools/console/dashboard.scala.txt | 6 + .../predictionio/tools/console/deploy.scala.txt | 29 + .../predictionio/tools/console/eval.scala.txt | 10 + .../tools/console/eventserver.scala.txt | 8 + .../predictionio/tools/console/export.scala.txt | 14 + .../predictionio/tools/console/imprt.scala.txt | 12 + .../predictionio/tools/console/main.scala.txt | 52 + .../predictionio/tools/console/run.scala.txt | 17 + .../predictionio/tools/console/status.scala.txt | 3 + .../tools/console/template.scala.txt | 25 + .../predictionio/tools/console/train.scala.txt | 28 + .../tools/console/upgrade.scala.txt | 15 + .../tools/console/version.scala.txt | 3 + .../tools/dashboard/index.scala.html | 99 ++ .../itemrank/params/algorithmsJson.scala.txt | 16 + .../itemrank/params/datasourceJson.scala.txt | 26 + .../itemrank/params/preparatorJson.scala.txt | 10 + .../itemrank/params/servingJson.scala.txt | 1 + .../itemrec/params/algorithmsJson.scala.txt | 15 + .../itemrec/params/datasourceJson.scala.txt | 26 + .../itemrec/params/preparatorJson.scala.txt | 10 + .../itemrec/params/servingJson.scala.txt | 1 + .../itemsim/params/algorithmsJson.scala.txt | 13 + .../itemsim/params/datasourceJson.scala.txt | 26 + .../itemsim/params/preparatorJson.scala.txt | 10 + .../itemsim/params/servingJson.scala.txt | 1 + .../tools/templates/scala/buildSbt.scala.txt | 14 + .../tools/templates/scala/engineJson.scala.txt | 9 + .../templates/scala/manifestJson.scala.txt | 9 + .../scala/project/assemblySbt.scala.txt | 1 + .../scala/src/main/scala/engine.scala.txt | 76 ++ .../prediction/tools/admin/AdminAPISpec.scala | 66 - .../predictionio/tools/admin/AdminAPISpec.scala | 66 + 506 files changed, 28276 insertions(+), 28276 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/4f03388e/common/src/main/scala/io/prediction/annotation/DeveloperApi.java ---------------------------------------------------------------------- diff --git a/common/src/main/scala/io/prediction/annotation/DeveloperApi.java b/common/src/main/scala/io/prediction/annotation/DeveloperApi.java deleted file mode 100644 index 63536ef..0000000 --- a/common/src/main/scala/io/prediction/annotation/DeveloperApi.java +++ /dev/null @@ -1,34 +0,0 @@ -/** Copyright 2015 TappingStone, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.prediction.annotation; - -import java.lang.annotation.*; - -/** - * A lower-level, unstable API intended for developers. - * - * Developer API's might change or be removed in minor versions of Spark. - * - * NOTE: If there exists a Scaladoc comment that immediately precedes this - * annotation, the first line of the comment must be ":: DeveloperApi ::" with - * no trailing blank line. This is because of the known issue that Scaladoc - * displays only either the annotation or the comment, whichever comes first. - */ -@Retention(RetentionPolicy.RUNTIME) -@Target({ElementType.TYPE, ElementType.FIELD, ElementType.METHOD, - ElementType.PARAMETER, ElementType.CONSTRUCTOR, ElementType.LOCAL_VARIABLE, - ElementType.PACKAGE}) -public @interface DeveloperApi {} http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/4f03388e/common/src/main/scala/io/prediction/annotation/Experimental.java ---------------------------------------------------------------------- diff --git a/common/src/main/scala/io/prediction/annotation/Experimental.java b/common/src/main/scala/io/prediction/annotation/Experimental.java deleted file mode 100644 index 86ec052..0000000 --- a/common/src/main/scala/io/prediction/annotation/Experimental.java +++ /dev/null @@ -1,35 +0,0 @@ -/** Copyright 2015 TappingStone, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.prediction.annotation; - -import java.lang.annotation.*; - -/** - * An experimental user-facing API. - * - * Experimental API's might change or be removed, or be adopted as first-class - * API's. - * - * NOTE: If there exists a Scaladoc comment that immediately precedes this - * annotation, the first line of the comment must be ":: Experimental ::" with - * no trailing blank line. This is because of the known issue that Scaladoc - * displays only either the annotation or the comment, whichever comes first. - */ -@Retention(RetentionPolicy.RUNTIME) -@Target({ElementType.TYPE, ElementType.FIELD, ElementType.METHOD, - ElementType.PARAMETER, ElementType.CONSTRUCTOR, ElementType.LOCAL_VARIABLE, - ElementType.PACKAGE}) -public @interface Experimental {} http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/4f03388e/common/src/main/scala/io/prediction/authentication/KeyAuthentication.scala ---------------------------------------------------------------------- diff --git a/common/src/main/scala/io/prediction/authentication/KeyAuthentication.scala b/common/src/main/scala/io/prediction/authentication/KeyAuthentication.scala deleted file mode 100644 index 752b5e1..0000000 --- a/common/src/main/scala/io/prediction/authentication/KeyAuthentication.scala +++ /dev/null @@ -1,55 +0,0 @@ -/** Copyright 2015 TappingStone, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.prediction.authentication - -/** - * This is a (very) simple authentication for the dashboard and engine servers - * It is highly recommended to implement a stonger authentication mechanism - */ - -import com.typesafe.config.ConfigFactory -import spray.http.HttpRequest -import spray.routing.authentication._ -import spray.routing.{AuthenticationFailedRejection, RequestContext} - -import scala.concurrent.ExecutionContext.Implicits.global -import scala.concurrent.Future - -trait KeyAuthentication { - - object ServerKey { - private val config = ConfigFactory.load("server.conf") - val get = config.getString("io.prediction.server.accessKey") - val param = "accessKey" - } - - def withAccessKeyFromFile: RequestContext => Future[Authentication[HttpRequest]] = { - ctx: RequestContext => - val accessKeyParamOpt = ctx.request.uri.query.get(ServerKey.param) - Future { - - val passedKey = accessKeyParamOpt.getOrElse { - Left(AuthenticationFailedRejection( - AuthenticationFailedRejection.CredentialsRejected, List())) - } - - if (passedKey.equals(ServerKey.get)) Right(ctx.request) - else Left(AuthenticationFailedRejection( - AuthenticationFailedRejection.CredentialsRejected, List())) - - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/4f03388e/common/src/main/scala/io/prediction/configuration/SSLConfiguration.scala ---------------------------------------------------------------------- diff --git a/common/src/main/scala/io/prediction/configuration/SSLConfiguration.scala b/common/src/main/scala/io/prediction/configuration/SSLConfiguration.scala deleted file mode 100644 index f784130..0000000 --- a/common/src/main/scala/io/prediction/configuration/SSLConfiguration.scala +++ /dev/null @@ -1,71 +0,0 @@ -/** Copyright 2015 TappingStone, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.prediction.configuration - -/** - * Created by ykhodorkovsky on 2/26/16. - */ - -import java.io.FileInputStream -import java.security.KeyStore -import javax.net.ssl.{KeyManagerFactory, SSLContext, TrustManagerFactory} - -import com.typesafe.config.ConfigFactory -import spray.io.ServerSSLEngineProvider - -trait SSLConfiguration { - - private val serverConfig = ConfigFactory.load("server.conf") - - private val keyStoreResource = - serverConfig.getString("io.prediction.server.ssl-keystore-resource") - private val password = serverConfig.getString("io.prediction.server.ssl-keystore-pass") - private val keyAlias = serverConfig.getString("io.prediction.server.ssl-key-alias") - - private val keyStore = { - - // Loading keystore from specified file - val clientStore = KeyStore.getInstance("JKS") - val inputStream = new FileInputStream( - getClass().getClassLoader().getResource(keyStoreResource).getFile()) - clientStore.load(inputStream, password.toCharArray) - inputStream.close() - clientStore - } - - // Creating SSL context - implicit def sslContext: SSLContext = { - val context = SSLContext.getInstance("TLS") - val tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm) - val kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm) - kmf.init(keyStore, password.toCharArray) - tmf.init(keyStore) - context.init(kmf.getKeyManagers, tmf.getTrustManagers, null) - context - } - - // provide implicit SSLEngine with some protocols - implicit def sslEngineProvider: ServerSSLEngineProvider = { - ServerSSLEngineProvider { engine => - engine.setEnabledCipherSuites(Array( - "TLS_RSA_WITH_AES_256_CBC_SHA", - "TLS_ECDH_ECDSA_WITH_RC4_128_SHA", - "TLS_RSA_WITH_AES_128_CBC_SHA")) - engine.setEnabledProtocols(Array("TLSv1", "TLSv1.2", "TLSv1.1")) - engine - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/4f03388e/common/src/main/scala/org/apache/predictionio/annotation/DeveloperApi.java ---------------------------------------------------------------------- diff --git a/common/src/main/scala/org/apache/predictionio/annotation/DeveloperApi.java b/common/src/main/scala/org/apache/predictionio/annotation/DeveloperApi.java new file mode 100644 index 0000000..d372184 --- /dev/null +++ b/common/src/main/scala/org/apache/predictionio/annotation/DeveloperApi.java @@ -0,0 +1,34 @@ +/** Copyright 2015 TappingStone, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.predictionio.annotation; + +import java.lang.annotation.*; + +/** + * A lower-level, unstable API intended for developers. + * + * Developer API's might change or be removed in minor versions of Spark. + * + * NOTE: If there exists a Scaladoc comment that immediately precedes this + * annotation, the first line of the comment must be ":: DeveloperApi ::" with + * no trailing blank line. This is because of the known issue that Scaladoc + * displays only either the annotation or the comment, whichever comes first. + */ +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.TYPE, ElementType.FIELD, ElementType.METHOD, + ElementType.PARAMETER, ElementType.CONSTRUCTOR, ElementType.LOCAL_VARIABLE, + ElementType.PACKAGE}) +public @interface DeveloperApi {} http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/4f03388e/common/src/main/scala/org/apache/predictionio/annotation/Experimental.java ---------------------------------------------------------------------- diff --git a/common/src/main/scala/org/apache/predictionio/annotation/Experimental.java b/common/src/main/scala/org/apache/predictionio/annotation/Experimental.java new file mode 100644 index 0000000..a304505 --- /dev/null +++ b/common/src/main/scala/org/apache/predictionio/annotation/Experimental.java @@ -0,0 +1,35 @@ +/** Copyright 2015 TappingStone, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.predictionio.annotation; + +import java.lang.annotation.*; + +/** + * An experimental user-facing API. + * + * Experimental API's might change or be removed, or be adopted as first-class + * API's. + * + * NOTE: If there exists a Scaladoc comment that immediately precedes this + * annotation, the first line of the comment must be ":: Experimental ::" with + * no trailing blank line. This is because of the known issue that Scaladoc + * displays only either the annotation or the comment, whichever comes first. + */ +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.TYPE, ElementType.FIELD, ElementType.METHOD, + ElementType.PARAMETER, ElementType.CONSTRUCTOR, ElementType.LOCAL_VARIABLE, + ElementType.PACKAGE}) +public @interface Experimental {} http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/4f03388e/common/src/main/scala/org/apache/predictionio/authentication/KeyAuthentication.scala ---------------------------------------------------------------------- diff --git a/common/src/main/scala/org/apache/predictionio/authentication/KeyAuthentication.scala b/common/src/main/scala/org/apache/predictionio/authentication/KeyAuthentication.scala new file mode 100644 index 0000000..0553952 --- /dev/null +++ b/common/src/main/scala/org/apache/predictionio/authentication/KeyAuthentication.scala @@ -0,0 +1,55 @@ +/** Copyright 2015 TappingStone, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.predictionio.authentication + +/** + * This is a (very) simple authentication for the dashboard and engine servers + * It is highly recommended to implement a stonger authentication mechanism + */ + +import com.typesafe.config.ConfigFactory +import spray.http.HttpRequest +import spray.routing.authentication._ +import spray.routing.{AuthenticationFailedRejection, RequestContext} + +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.Future + +trait KeyAuthentication { + + object ServerKey { + private val config = ConfigFactory.load("server.conf") + val get = config.getString("io.prediction.server.accessKey") + val param = "accessKey" + } + + def withAccessKeyFromFile: RequestContext => Future[Authentication[HttpRequest]] = { + ctx: RequestContext => + val accessKeyParamOpt = ctx.request.uri.query.get(ServerKey.param) + Future { + + val passedKey = accessKeyParamOpt.getOrElse { + Left(AuthenticationFailedRejection( + AuthenticationFailedRejection.CredentialsRejected, List())) + } + + if (passedKey.equals(ServerKey.get)) Right(ctx.request) + else Left(AuthenticationFailedRejection( + AuthenticationFailedRejection.CredentialsRejected, List())) + + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/4f03388e/common/src/main/scala/org/apache/predictionio/configuration/SSLConfiguration.scala ---------------------------------------------------------------------- diff --git a/common/src/main/scala/org/apache/predictionio/configuration/SSLConfiguration.scala b/common/src/main/scala/org/apache/predictionio/configuration/SSLConfiguration.scala new file mode 100644 index 0000000..2a9344d --- /dev/null +++ b/common/src/main/scala/org/apache/predictionio/configuration/SSLConfiguration.scala @@ -0,0 +1,71 @@ +/** Copyright 2015 TappingStone, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.predictionio.configuration + +/** + * Created by ykhodorkovsky on 2/26/16. + */ + +import java.io.FileInputStream +import java.security.KeyStore +import javax.net.ssl.{KeyManagerFactory, SSLContext, TrustManagerFactory} + +import com.typesafe.config.ConfigFactory +import spray.io.ServerSSLEngineProvider + +trait SSLConfiguration { + + private val serverConfig = ConfigFactory.load("server.conf") + + private val keyStoreResource = + serverConfig.getString("io.prediction.server.ssl-keystore-resource") + private val password = serverConfig.getString("io.prediction.server.ssl-keystore-pass") + private val keyAlias = serverConfig.getString("io.prediction.server.ssl-key-alias") + + private val keyStore = { + + // Loading keystore from specified file + val clientStore = KeyStore.getInstance("JKS") + val inputStream = new FileInputStream( + getClass().getClassLoader().getResource(keyStoreResource).getFile()) + clientStore.load(inputStream, password.toCharArray) + inputStream.close() + clientStore + } + + // Creating SSL context + implicit def sslContext: SSLContext = { + val context = SSLContext.getInstance("TLS") + val tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm) + val kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm) + kmf.init(keyStore, password.toCharArray) + tmf.init(keyStore) + context.init(kmf.getKeyManagers, tmf.getTrustManagers, null) + context + } + + // provide implicit SSLEngine with some protocols + implicit def sslEngineProvider: ServerSSLEngineProvider = { + ServerSSLEngineProvider { engine => + engine.setEnabledCipherSuites(Array( + "TLS_RSA_WITH_AES_256_CBC_SHA", + "TLS_ECDH_ECDSA_WITH_RC4_128_SHA", + "TLS_RSA_WITH_AES_128_CBC_SHA")) + engine.setEnabledProtocols(Array("TLSv1", "TLSv1.2", "TLSv1.1")) + engine + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/4f03388e/core/src/main/scala/io/prediction/controller/CustomQuerySerializer.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/io/prediction/controller/CustomQuerySerializer.scala b/core/src/main/scala/io/prediction/controller/CustomQuerySerializer.scala deleted file mode 100644 index 35ab6c3..0000000 --- a/core/src/main/scala/io/prediction/controller/CustomQuerySerializer.scala +++ /dev/null @@ -1,37 +0,0 @@ -/** Copyright 2015 TappingStone, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.prediction.controller - -import io.prediction.core.BaseQuerySerializer - -/** If your query class cannot be automatically serialized/deserialized to/from - * JSON, implement a trait by extending this trait, and overriding the - * `querySerializer` member with your - * [[https://github.com/json4s/json4s#serializing-non-supported-types custom JSON4S serializer]]. - * Algorithm and serving classes using your query class would only need to mix - * in the trait to enable the custom serializer. - * - * @group Helper - */ -trait CustomQuerySerializer extends BaseQuerySerializer - -/** DEPRECATED. Use [[CustomQuerySerializer]] instead. - * - * @group Helper - */ -@deprecated("Use CustomQuerySerializer instead.", "0.9.2") -trait WithQuerySerializer extends CustomQuerySerializer - http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/4f03388e/core/src/main/scala/io/prediction/controller/Deployment.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/io/prediction/controller/Deployment.scala b/core/src/main/scala/io/prediction/controller/Deployment.scala deleted file mode 100644 index 49e14d5..0000000 --- a/core/src/main/scala/io/prediction/controller/Deployment.scala +++ /dev/null @@ -1,56 +0,0 @@ -/** Copyright 2015 TappingStone, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.prediction.controller - -import io.prediction.core.BaseEngine - -import scala.language.implicitConversions - -/** Defines a deployment that contains an [[Engine]] - * - * @group Engine - */ -trait Deployment extends EngineFactory { - protected[this] var _engine: BaseEngine[_, _, _, _] = _ - protected[this] var engineSet: Boolean = false - - /** Returns the [[Engine]] of this [[Deployment]] */ - def apply(): BaseEngine[_, _, _, _] = { - assert(engineSet, "Engine not set") - _engine - } - - /** Returns the [[Engine]] contained in this [[Deployment]]. */ - private [prediction] - def engine: BaseEngine[_, _, _, _] = { - assert(engineSet, "Engine not set") - _engine - } - - /** Sets the [[Engine]] for this [[Deployment]] - * - * @param engine An implementation of [[Engine]] - * @tparam EI Evaluation information class - * @tparam Q Query class - * @tparam P Predicted result class - * @tparam A Actual result class - */ - def engine_=[EI, Q, P, A](engine: BaseEngine[EI, Q, P, A]) { - assert(!engineSet, "Engine can be set at most once") - _engine = engine - engineSet = true - } -} http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/4f03388e/core/src/main/scala/io/prediction/controller/Engine.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/io/prediction/controller/Engine.scala b/core/src/main/scala/io/prediction/controller/Engine.scala deleted file mode 100644 index 5cc2e31..0000000 --- a/core/src/main/scala/io/prediction/controller/Engine.scala +++ /dev/null @@ -1,829 +0,0 @@ -/** Copyright 2015 TappingStone, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.prediction.controller - -import grizzled.slf4j.Logger -import io.prediction.core.BaseAlgorithm -import io.prediction.core.BaseDataSource -import io.prediction.core.BaseEngine -import io.prediction.core.BasePreparator -import io.prediction.core.BaseServing -import io.prediction.core.Doer -import io.prediction.data.storage.EngineInstance -import io.prediction.data.storage.StorageClientException -import io.prediction.workflow.CreateWorkflow -import io.prediction.workflow.EngineLanguage -import io.prediction.workflow.JsonExtractorOption.JsonExtractorOption -import io.prediction.workflow.NameParamsSerializer -import io.prediction.workflow.PersistentModelManifest -import io.prediction.workflow.SparkWorkflowUtils -import io.prediction.workflow.StopAfterPrepareInterruption -import io.prediction.workflow.StopAfterReadInterruption -import io.prediction.workflow.WorkflowParams -import io.prediction.workflow.WorkflowUtils -import org.apache.spark.SparkContext -import org.apache.spark.SparkContext._ -import org.apache.spark.rdd.RDD -import org.json4s._ -import org.json4s.native.JsonMethods._ -import org.json4s.native.Serialization.read - -import scala.collection.JavaConversions -import scala.language.implicitConversions - -/** This class chains up the entire data process. PredictionIO uses this - * information to create workflows and deployments. In Scala, you should - * implement an object that extends the [[EngineFactory]] trait similar to the - * following example. - * - * {{{ - * object ItemRankEngine extends EngineFactory { - * def apply() = { - * new Engine( - * classOf[ItemRankDataSource], - * classOf[ItemRankPreparator], - * Map( - * "knn" -> classOf[KNNAlgorithm], - * "rand" -> classOf[RandomAlgorithm], - * "mahoutItemBased" -> classOf[MahoutItemBasedAlgorithm]), - * classOf[ItemRankServing]) - * } - * } - * }}} - * - * @see [[EngineFactory]] - * @tparam TD Training data class. - * @tparam EI Evaluation info class. - * @tparam PD Prepared data class. - * @tparam Q Input query class. - * @tparam P Output prediction class. - * @tparam A Actual value class. - * @param dataSourceClassMap Map of data source names to class. - * @param preparatorClassMap Map of preparator names to class. - * @param algorithmClassMap Map of algorithm names to classes. - * @param servingClassMap Map of serving names to class. - * @group Engine - */ -class Engine[TD, EI, PD, Q, P, A]( - val dataSourceClassMap: Map[String, - Class[_ <: BaseDataSource[TD, EI, Q, A]]], - val preparatorClassMap: Map[String, Class[_ <: BasePreparator[TD, PD]]], - val algorithmClassMap: Map[String, Class[_ <: BaseAlgorithm[PD, _, Q, P]]], - val servingClassMap: Map[String, Class[_ <: BaseServing[Q, P]]]) - extends BaseEngine[EI, Q, P, A] { - - private[prediction] - implicit lazy val formats = Utils.json4sDefaultFormats + - new NameParamsSerializer - - @transient lazy protected val logger = Logger[this.type] - - /** This auxiliary constructor is provided for backward compatibility. - * - * @param dataSourceClass Data source class. - * @param preparatorClass Preparator class. - * @param algorithmClassMap Map of algorithm names to classes. - * @param servingClass Serving class. - */ - def this( - dataSourceClass: Class[_ <: BaseDataSource[TD, EI, Q, A]], - preparatorClass: Class[_ <: BasePreparator[TD, PD]], - algorithmClassMap: Map[String, Class[_ <: BaseAlgorithm[PD, _, Q, P]]], - servingClass: Class[_ <: BaseServing[Q, P]]) = this( - Map("" -> dataSourceClass), - Map("" -> preparatorClass), - algorithmClassMap, - Map("" -> servingClass) - ) - - /** Java-friendly constructor - * - * @param dataSourceClass Data source class. - * @param preparatorClass Preparator class. - * @param algorithmClassMap Map of algorithm names to classes. - * @param servingClass Serving class. - */ - def this(dataSourceClass: Class[_ <: BaseDataSource[TD, EI, Q, A]], - preparatorClass: Class[_ <: BasePreparator[TD, PD]], - algorithmClassMap: _root_.java.util.Map[String, Class[_ <: BaseAlgorithm[PD, _, Q, P]]], - servingClass: Class[_ <: BaseServing[Q, P]]) = this( - Map("" -> dataSourceClass), - Map("" -> preparatorClass), - JavaConversions.mapAsScalaMap(algorithmClassMap).toMap, - Map("" -> servingClass) - ) - - /** Returns a new Engine instance, mimicking case class's copy method behavior. - */ - def copy( - dataSourceClassMap: Map[String, Class[_ <: BaseDataSource[TD, EI, Q, A]]] - = dataSourceClassMap, - preparatorClassMap: Map[String, Class[_ <: BasePreparator[TD, PD]]] - = preparatorClassMap, - algorithmClassMap: Map[String, Class[_ <: BaseAlgorithm[PD, _, Q, P]]] - = algorithmClassMap, - servingClassMap: Map[String, Class[_ <: BaseServing[Q, P]]] - = servingClassMap): Engine[TD, EI, PD, Q, P, A] = { - new Engine( - dataSourceClassMap, - preparatorClassMap, - algorithmClassMap, - servingClassMap) - } - - /** Training this engine would return a list of models. - * - * @param sc An instance of SparkContext. - * @param engineParams An instance of [[EngineParams]] for running a single training. - * @param params An instance of [[WorkflowParams]] that controls the workflow. - * @return A list of models. - */ - def train( - sc: SparkContext, - engineParams: EngineParams, - engineInstanceId: String, - params: WorkflowParams): Seq[Any] = { - val (dataSourceName, dataSourceParams) = engineParams.dataSourceParams - val dataSource = Doer(dataSourceClassMap(dataSourceName), dataSourceParams) - - val (preparatorName, preparatorParams) = engineParams.preparatorParams - val preparator = Doer(preparatorClassMap(preparatorName), preparatorParams) - - val algoParamsList = engineParams.algorithmParamsList - require( - algoParamsList.size > 0, - "EngineParams.algorithmParamsList must have at least 1 element.") - - val algorithms = algoParamsList.map { case (algoName, algoParams) => - Doer(algorithmClassMap(algoName), algoParams) - } - - val models = Engine.train( - sc, dataSource, preparator, algorithms, params) - - val algoCount = algorithms.size - val algoTuples: Seq[(String, Params, BaseAlgorithm[_, _, _, _], Any)] = - (0 until algoCount).map { ax => { - // val (name, params) = algoParamsList(ax) - val (name, params) = algoParamsList(ax) - (name, params, algorithms(ax), models(ax)) - }} - - makeSerializableModels( - sc, - engineInstanceId = engineInstanceId, - algoTuples = algoTuples) - } - - /** Algorithm models can be persisted before deploy. However, it is also - * possible that models are not persisted. This method retrains non-persisted - * models and return a list of models that can be used directly in deploy. - */ - private[prediction] - def prepareDeploy( - sc: SparkContext, - engineParams: EngineParams, - engineInstanceId: String, - persistedModels: Seq[Any], - params: WorkflowParams): Seq[Any] = { - - val algoParamsList = engineParams.algorithmParamsList - val algorithms = algoParamsList.map { case (algoName, algoParams) => - Doer(algorithmClassMap(algoName), algoParams) - } - - val models = if (persistedModels.exists(m => m.isInstanceOf[Unit.type])) { - // If any of persistedModels is Unit, we need to re-train the model. - logger.info("Some persisted models are Unit, need to re-train.") - val (dataSourceName, dataSourceParams) = engineParams.dataSourceParams - val dataSource = Doer(dataSourceClassMap(dataSourceName), dataSourceParams) - - val (preparatorName, preparatorParams) = engineParams.preparatorParams - val preparator = Doer(preparatorClassMap(preparatorName), preparatorParams) - - val td = dataSource.readTrainingBase(sc) - val pd = preparator.prepareBase(sc, td) - - val models = algorithms.zip(persistedModels).map { case (algo, m) => - m match { - case Unit => algo.trainBase(sc, pd) - case _ => m - } - } - models - } else { - logger.info("Using persisted model") - persistedModels - } - - models - .zip(algorithms) - .zip(algoParamsList) - .zipWithIndex - .map { - case (((model, algo), (algoName, algoParams)), ax) => { - model match { - case modelManifest: PersistentModelManifest => { - logger.info("Custom-persisted model detected for algorithm " + - algo.getClass.getName) - SparkWorkflowUtils.getPersistentModel( - modelManifest, - Seq(engineInstanceId, ax, algoName).mkString("-"), - algoParams, - Some(sc), - getClass.getClassLoader) - } - case m => { - try { - logger.info( - s"Loaded model ${m.getClass.getName} for algorithm " + - s"${algo.getClass.getName}") - sc.stop - m - } catch { - case e: NullPointerException => - logger.warn( - s"Null model detected for algorithm ${algo.getClass.getName}") - m - } - } - } // model match - } - } - } - - /** Extract model for persistent layer. - * - * PredictionIO presist models for future use. It allows custom - * implementation for persisting models. You need to implement the - * [[io.prediction.controller.PersistentModel]] interface. This method - * traverses all models in the workflow. If the model is a - * [[io.prediction.controller.PersistentModel]], it calls the save method - * for custom persistence logic. - * - * For model doesn't support custom logic, PredictionIO serializes the whole - * model if the corresponding algorithm is local. On the other hand, if the - * model is parallel (i.e. model associated with a number of huge RDDS), this - * method return Unit, in which case PredictionIO will retrain the whole - * model from scratch next time it is used. - */ - private def makeSerializableModels( - sc: SparkContext, - engineInstanceId: String, - // AlgoName, Algo, Model - algoTuples: Seq[(String, Params, BaseAlgorithm[_, _, _, _], Any)] - ): Seq[Any] = { - - logger.info(s"engineInstanceId=$engineInstanceId") - - algoTuples - .zipWithIndex - .map { case ((name, params, algo, model), ax) => - algo.makePersistentModel( - sc = sc, - modelId = Seq(engineInstanceId, ax, name).mkString("-"), - algoParams = params, - bm = model) - } - } - - /** This is implemented such that [[io.prediction.controller.Evaluation]] can - * use this method to generate inputs for [[io.prediction.controller.Metric]]. - * - * @param sc An instance of SparkContext. - * @param engineParams An instance of [[EngineParams]] for running a single evaluation. - * @param params An instance of [[WorkflowParams]] that controls the workflow. - * @return A list of evaluation information and RDD of query, predicted - * result, and actual result tuple tuple. - */ - def eval( - sc: SparkContext, - engineParams: EngineParams, - params: WorkflowParams) - : Seq[(EI, RDD[(Q, P, A)])] = { - val (dataSourceName, dataSourceParams) = engineParams.dataSourceParams - val dataSource = Doer(dataSourceClassMap(dataSourceName), dataSourceParams) - - val (preparatorName, preparatorParams) = engineParams.preparatorParams - val preparator = Doer(preparatorClassMap(preparatorName), preparatorParams) - - val algoParamsList = engineParams.algorithmParamsList - require( - algoParamsList.size > 0, - "EngineParams.algorithmParamsList must have at least 1 element.") - - val algorithms = algoParamsList.map { case (algoName, algoParams) => { - try { - Doer(algorithmClassMap(algoName), algoParams) - } catch { - case e: NoSuchElementException => { - if (algoName == "") { - logger.error("Empty algorithm name supplied but it could not " + - "match with any algorithm in the engine's definition. " + - "Existing algorithm name(s) are: " + - s"${algorithmClassMap.keys.mkString(", ")}. Aborting.") - } else { - logger.error(s"$algoName cannot be found in the engine's " + - "definition. Existing algorithm name(s) are: " + - s"${algorithmClassMap.keys.mkString(", ")}. Aborting.") - } - sys.exit(1) - } - } - }} - - val (servingName, servingParams) = engineParams.servingParams - val serving = Doer(servingClassMap(servingName), servingParams) - - Engine.eval(sc, dataSource, preparator, algorithms, serving) - } - - override def jValueToEngineParams( - variantJson: JValue, - jsonExtractor: JsonExtractorOption): EngineParams = { - - val engineLanguage = EngineLanguage.Scala - // Extract EngineParams - logger.info(s"Extracting datasource params...") - val dataSourceParams: (String, Params) = - WorkflowUtils.getParamsFromJsonByFieldAndClass( - variantJson, - "datasource", - dataSourceClassMap, - engineLanguage, - jsonExtractor) - logger.info(s"Datasource params: $dataSourceParams") - - logger.info(s"Extracting preparator params...") - val preparatorParams: (String, Params) = - WorkflowUtils.getParamsFromJsonByFieldAndClass( - variantJson, - "preparator", - preparatorClassMap, - engineLanguage, - jsonExtractor) - logger.info(s"Preparator params: $preparatorParams") - - val algorithmsParams: Seq[(String, Params)] = - variantJson findField { - case JField("algorithms", _) => true - case _ => false - } map { jv => - val algorithmsParamsJson = jv._2 - algorithmsParamsJson match { - case JArray(s) => s.map { algorithmParamsJValue => - val eap = algorithmParamsJValue.extract[CreateWorkflow.AlgorithmParams] - ( - eap.name, - WorkflowUtils.extractParams( - engineLanguage, - compact(render(eap.params)), - algorithmClassMap(eap.name), - jsonExtractor) - ) - } - case _ => Nil - } - } getOrElse Seq(("", EmptyParams())) - - logger.info(s"Extracting serving params...") - val servingParams: (String, Params) = - WorkflowUtils.getParamsFromJsonByFieldAndClass( - variantJson, - "serving", - servingClassMap, - engineLanguage, - jsonExtractor) - logger.info(s"Serving params: $servingParams") - - new EngineParams( - dataSourceParams = dataSourceParams, - preparatorParams = preparatorParams, - algorithmParamsList = algorithmsParams, - servingParams = servingParams) - } - - private[prediction] def engineInstanceToEngineParams( - engineInstance: EngineInstance, - jsonExtractor: JsonExtractorOption): EngineParams = { - - implicit val formats = DefaultFormats - val engineLanguage = EngineLanguage.Scala - - val dataSourceParamsWithName: (String, Params) = { - val (name, params) = - read[(String, JValue)](engineInstance.dataSourceParams) - if (!dataSourceClassMap.contains(name)) { - logger.error(s"Unable to find datasource class with name '$name'" + - " defined in Engine.") - sys.exit(1) - } - val extractedParams = WorkflowUtils.extractParams( - engineLanguage, - compact(render(params)), - dataSourceClassMap(name), - jsonExtractor) - (name, extractedParams) - } - - val preparatorParamsWithName: (String, Params) = { - val (name, params) = - read[(String, JValue)](engineInstance.preparatorParams) - if (!preparatorClassMap.contains(name)) { - logger.error(s"Unable to find preparator class with name '$name'" + - " defined in Engine.") - sys.exit(1) - } - val extractedParams = WorkflowUtils.extractParams( - engineLanguage, - compact(render(params)), - preparatorClassMap(name), - jsonExtractor) - (name, extractedParams) - } - - val algorithmsParamsWithNames = - read[Seq[(String, JValue)]](engineInstance.algorithmsParams).map { - case (algoName, params) => - val extractedParams = WorkflowUtils.extractParams( - engineLanguage, - compact(render(params)), - algorithmClassMap(algoName), - jsonExtractor) - (algoName, extractedParams) - } - - val servingParamsWithName: (String, Params) = { - val (name, params) = read[(String, JValue)](engineInstance.servingParams) - if (!servingClassMap.contains(name)) { - logger.error(s"Unable to find serving class with name '$name'" + - " defined in Engine.") - sys.exit(1) - } - val extractedParams = WorkflowUtils.extractParams( - engineLanguage, - compact(render(params)), - servingClassMap(name), - jsonExtractor) - (name, extractedParams) - } - - new EngineParams( - dataSourceParams = dataSourceParamsWithName, - preparatorParams = preparatorParamsWithName, - algorithmParamsList = algorithmsParamsWithNames, - servingParams = servingParamsWithName) - } -} - -/** This object contains concrete implementation for some methods of the - * [[Engine]] class. - * - * @group Engine - */ -object Engine { - private type EX = Int - private type AX = Int - private type QX = Long - - @transient lazy private val logger = Logger[this.type] - - /** Helper class to accept either a single data source, or a map of data - * sources, with a companion object providing implicit conversions, so - * using this class directly is not necessary. - * - * @tparam TD Training data class - * @tparam EI Evaluation information class - * @tparam Q Input query class - * @tparam A Actual result class - */ - class DataSourceMap[TD, EI, Q, A]( - val m: Map[String, Class[_ <: BaseDataSource[TD, EI, Q, A]]]) { - def this(c: Class[_ <: BaseDataSource[TD, EI, Q, A]]) = this(Map("" -> c)) - } - - /** Companion object providing implicit conversions, so using this directly - * is not necessary. - */ - object DataSourceMap { - implicit def cToMap[TD, EI, Q, A]( - c: Class[_ <: BaseDataSource[TD, EI, Q, A]]): - DataSourceMap[TD, EI, Q, A] = new DataSourceMap(c) - implicit def mToMap[TD, EI, Q, A]( - m: Map[String, Class[_ <: BaseDataSource[TD, EI, Q, A]]]): - DataSourceMap[TD, EI, Q, A] = new DataSourceMap(m) - } - - /** Helper class to accept either a single preparator, or a map of - * preparators, with a companion object providing implicit conversions, so - * using this class directly is not necessary. - * - * @tparam TD Training data class - * @tparam PD Prepared data class - */ - class PreparatorMap[TD, PD]( - val m: Map[String, Class[_ <: BasePreparator[TD, PD]]]) { - def this(c: Class[_ <: BasePreparator[TD, PD]]) = this(Map("" -> c)) - } - - /** Companion object providing implicit conversions, so using this directly - * is not necessary. - */ - object PreparatorMap { - implicit def cToMap[TD, PD]( - c: Class[_ <: BasePreparator[TD, PD]]): - PreparatorMap[TD, PD] = new PreparatorMap(c) - implicit def mToMap[TD, PD]( - m: Map[String, Class[_ <: BasePreparator[TD, PD]]]): - PreparatorMap[TD, PD] = new PreparatorMap(m) - } - - /** Helper class to accept either a single serving, or a map of serving, with - * a companion object providing implicit conversions, so using this class - * directly is not necessary. - * - * @tparam Q Input query class - * @tparam P Predicted result class - */ - class ServingMap[Q, P]( - val m: Map[String, Class[_ <: BaseServing[Q, P]]]) { - def this(c: Class[_ <: BaseServing[Q, P]]) = this(Map("" -> c)) - } - - /** Companion object providing implicit conversions, so using this directly - * is not necessary. - */ - object ServingMap { - implicit def cToMap[Q, P]( - c: Class[_ <: BaseServing[Q, P]]): ServingMap[Q, P] = - new ServingMap(c) - implicit def mToMap[Q, P]( - m: Map[String, Class[_ <: BaseServing[Q, P]]]): ServingMap[Q, P] = - new ServingMap(m) - } - - /** Convenient method for returning an instance of [[Engine]]. - * - * @param dataSourceMap Accepts either an instance of Class of the data - * source, or a Map of data source classes (implicitly - * converted to [[DataSourceMap]]. - * @param preparatorMap Accepts either an instance of Class of the - * preparator, or a Map of preparator classes - * (implicitly converted to [[PreparatorMap]]. - * @param algorithmClassMap Accepts a Map of algorithm classes. - * @param servingMap Accepts either an instance of Class of the serving, or - * a Map of serving classes (implicitly converted to - * [[ServingMap]]. - * @tparam TD Training data class - * @tparam EI Evaluation information class - * @tparam PD Prepared data class - * @tparam Q Input query class - * @tparam P Predicted result class - * @tparam A Actual result class - * @return An instance of [[Engine]] - */ - def apply[TD, EI, PD, Q, P, A]( - dataSourceMap: DataSourceMap[TD, EI, Q, A], - preparatorMap: PreparatorMap[TD, PD], - algorithmClassMap: Map[String, Class[_ <: BaseAlgorithm[PD, _, Q, P]]], - servingMap: ServingMap[Q, P]): Engine[TD, EI, PD, Q, P, A] = new Engine( - dataSourceMap.m, - preparatorMap.m, - algorithmClassMap, - servingMap.m - ) - - /** Provides concrete implementation of training for [[Engine]]. - * - * @param sc An instance of SparkContext - * @param dataSource An instance of data source - * @param preparator An instance of preparator - * @param algorithmList A list of algorithm instances - * @param params An instance of [[WorkflowParams]] that controls the training - * process. - * @tparam TD Training data class - * @tparam PD Prepared data class - * @tparam Q Input query class - * @return A list of trained models - */ - def train[TD, PD, Q]( - sc: SparkContext, - dataSource: BaseDataSource[TD, _, Q, _], - preparator: BasePreparator[TD, PD], - algorithmList: Seq[BaseAlgorithm[PD, _, Q, _]], - params: WorkflowParams - ): Seq[Any] = { - logger.info("EngineWorkflow.train") - logger.info(s"DataSource: $dataSource") - logger.info(s"Preparator: $preparator") - logger.info(s"AlgorithmList: $algorithmList") - - if (params.skipSanityCheck) { - logger.info("Data sanity check is off.") - } else { - logger.info("Data sanity check is on.") - } - - val td = try { - dataSource.readTrainingBase(sc) - } catch { - case e: StorageClientException => - logger.error(s"Error occured reading from data source. (Reason: " + - e.getMessage + ") Please see the log for debugging details.", e) - sys.exit(1) - } - - if (!params.skipSanityCheck) { - td match { - case sanityCheckable: SanityCheck => { - logger.info(s"${td.getClass.getName} supports data sanity" + - " check. Performing check.") - sanityCheckable.sanityCheck() - } - case _ => { - logger.info(s"${td.getClass.getName} does not support" + - " data sanity check. Skipping check.") - } - } - } - - if (params.stopAfterRead) { - logger.info("Stopping here because --stop-after-read is set.") - throw StopAfterReadInterruption() - } - - val pd = preparator.prepareBase(sc, td) - - if (!params.skipSanityCheck) { - pd match { - case sanityCheckable: SanityCheck => { - logger.info(s"${pd.getClass.getName} supports data sanity" + - " check. Performing check.") - sanityCheckable.sanityCheck() - } - case _ => { - logger.info(s"${pd.getClass.getName} does not support" + - " data sanity check. Skipping check.") - } - } - } - - if (params.stopAfterPrepare) { - logger.info("Stopping here because --stop-after-prepare is set.") - throw StopAfterPrepareInterruption() - } - - val models: Seq[Any] = algorithmList.map(_.trainBase(sc, pd)) - - if (!params.skipSanityCheck) { - models.foreach { model => { - model match { - case sanityCheckable: SanityCheck => { - logger.info(s"${model.getClass.getName} supports data sanity" + - " check. Performing check.") - sanityCheckable.sanityCheck() - } - case _ => { - logger.info(s"${model.getClass.getName} does not support" + - " data sanity check. Skipping check.") - } - } - }} - } - - logger.info("EngineWorkflow.train completed") - models - } - - /** Provides concrete implementation of evaluation for [[Engine]]. - * - * @param sc An instance of SparkContext - * @param dataSource An instance of data source - * @param preparator An instance of preparator - * @param algorithmList A list of algorithm instances - * @param serving An instance of serving - * @tparam TD Training data class - * @tparam PD Prepared data class - * @tparam Q Input query class - * @tparam P Predicted result class - * @tparam A Actual result class - * @tparam EI Evaluation information class - * @return A list of evaluation information, RDD of query, predicted result, - * and actual result tuple tuple. - */ - def eval[TD, PD, Q, P, A, EI]( - sc: SparkContext, - dataSource: BaseDataSource[TD, EI, Q, A], - preparator: BasePreparator[TD, PD], - algorithmList: Seq[BaseAlgorithm[PD, _, Q, P]], - serving: BaseServing[Q, P]): Seq[(EI, RDD[(Q, P, A)])] = { - logger.info(s"DataSource: $dataSource") - logger.info(s"Preparator: $preparator") - logger.info(s"AlgorithmList: $algorithmList") - logger.info(s"Serving: $serving") - - val algoMap: Map[AX, BaseAlgorithm[PD, _, Q, P]] = algorithmList - .zipWithIndex - .map(_.swap) - .toMap - val algoCount = algoMap.size - - val evalTupleMap: Map[EX, (TD, EI, RDD[(Q, A)])] = dataSource - .readEvalBase(sc) - .zipWithIndex - .map(_.swap) - .toMap - - val evalCount = evalTupleMap.size - - val evalTrainMap: Map[EX, TD] = evalTupleMap.mapValues(_._1) - val evalInfoMap: Map[EX, EI] = evalTupleMap.mapValues(_._2) - val evalQAsMap: Map[EX, RDD[(QX, (Q, A))]] = evalTupleMap - .mapValues(_._3) - .mapValues{ _.zipWithUniqueId().map(_.swap) } - - val preparedMap: Map[EX, PD] = evalTrainMap.mapValues { td => { - preparator.prepareBase(sc, td) - }} - - val algoModelsMap: Map[EX, Map[AX, Any]] = preparedMap.mapValues { pd => { - algoMap.mapValues(_.trainBase(sc,pd)) - }} - - val suppQAsMap: Map[EX, RDD[(QX, (Q, A))]] = evalQAsMap.mapValues { qas => - qas.map { case (qx, (q, a)) => (qx, (serving.supplementBase(q), a)) } - } - - val algoPredictsMap: Map[EX, RDD[(QX, Seq[P])]] = (0 until evalCount) - .map { ex => { - val modelMap: Map[AX, Any] = algoModelsMap(ex) - - val qs: RDD[(QX, Q)] = suppQAsMap(ex).mapValues(_._1) - - val algoPredicts: Seq[RDD[(QX, (AX, P))]] = (0 until algoCount) - .map { ax => { - val algo = algoMap(ax) - val model = modelMap(ax) - val rawPredicts: RDD[(QX, P)] = algo.batchPredictBase(sc, model, qs) - val predicts: RDD[(QX, (AX, P))] = rawPredicts.map { case (qx, p) => { - (qx, (ax, p)) - }} - predicts - }} - - val unionAlgoPredicts: RDD[(QX, Seq[P])] = sc.union(algoPredicts) - .groupByKey() - .mapValues { ps => { - assert (ps.size == algoCount, "Must have same length as algoCount") - // TODO. Check size == algoCount - ps.toSeq.sortBy(_._1).map(_._2) - }} - - (ex, unionAlgoPredicts) - }} - .toMap - - val servingQPAMap: Map[EX, RDD[(Q, P, A)]] = algoPredictsMap - .map { case (ex, psMap) => { - // The query passed to serving.serve is the original one, not - // supplemented. - val qasMap: RDD[(QX, (Q, A))] = evalQAsMap(ex) - val qpsaMap: RDD[(QX, Q, Seq[P], A)] = psMap.join(qasMap) - .map { case (qx, t) => (qx, t._2._1, t._1, t._2._2) } - - val qpaMap: RDD[(Q, P, A)] = qpsaMap.map { - case (qx, q, ps, a) => (q, serving.serveBase(q, ps), a) - } - (ex, qpaMap) - }} - - (0 until evalCount).map { ex => { - (evalInfoMap(ex), servingQPAMap(ex)) - }} - .toSeq - } -} - -/** Mix in this trait for queries that contain prId (PredictedResultId). - * This is useful when your engine expects queries to also be associated with - * prId keys when feedback loop is enabled. - * - * @group Helper - */ -@deprecated("To be removed in future releases.", "0.9.2") -trait WithPrId { - val prId: String = "" -} http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/4f03388e/core/src/main/scala/io/prediction/controller/EngineFactory.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/io/prediction/controller/EngineFactory.scala b/core/src/main/scala/io/prediction/controller/EngineFactory.scala deleted file mode 100644 index f6988c7..0000000 --- a/core/src/main/scala/io/prediction/controller/EngineFactory.scala +++ /dev/null @@ -1,41 +0,0 @@ -/** Copyright 2015 TappingStone, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.prediction.controller - -import io.prediction.core.BaseEngine - -import scala.language.implicitConversions - -/** If you intend to let PredictionIO create workflow and deploy serving - * automatically, you will need to implement an object that extends this class - * and return an [[Engine]]. - * - * @group Engine - */ -abstract class EngineFactory { - /** Creates an instance of an [[Engine]]. */ - def apply(): BaseEngine[_, _, _, _] - - /** Override this method to programmatically return engine parameters. */ - def engineParams(key: String): EngineParams = EngineParams() -} - -/** DEPRECATED. Use [[EngineFactory]] instead. - * - * @group Engine - */ -@deprecated("Use EngineFactory instead.", "0.9.2") -trait IEngineFactory extends EngineFactory
