This is an automated email from the ASF dual-hosted git repository. srowen pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 3ce1dff [SPARK-30930][ML] Remove ML/MLLIB DeveloperApi annotations 3ce1dff is described below commit 3ce1dff7ba4799295a2bf89e6df35e900afc587b Author: Huaxin Gao <huax...@us.ibm.com> AuthorDate: Mon Mar 16 12:41:22 2020 -0500 [SPARK-30930][ML] Remove ML/MLLIB DeveloperApi annotations ### What changes were proposed in this pull request? jira link: https://issues.apache.org/jira/browse/SPARK-30930 Remove ML/MLLIB DeveloperApi annotations. ### Why are the changes needed? The Developer APIs in ML/MLLIB have been there for a long time. They are stable now and are very unlikely to be changed or removed, so I unmark these Developer APIs in this PR. ### Does this PR introduce any user-facing change? Yes. DeveloperApi annotations are removed from docs. ### How was this patch tested? existing tests Closes #27859 from huaxingao/spark-30930. Authored-by: Huaxin Gao <huax...@us.ibm.com> Signed-off-by: Sean Owen <sro...@gmail.com> --- .../main/scala/org/apache/spark/ml/Estimator.scala | 4 +-- .../src/main/scala/org/apache/spark/ml/Model.scala | 3 -- .../main/scala/org/apache/spark/ml/Pipeline.scala | 4 --- .../main/scala/org/apache/spark/ml/Predictor.scala | 6 +--- .../scala/org/apache/spark/ml/Transformer.scala | 6 +--- .../apache/spark/ml/attribute/AttributeGroup.scala | 5 ---- .../apache/spark/ml/attribute/AttributeType.scala | 7 ----- .../org/apache/spark/ml/attribute/attributes.scala | 21 -------------- .../spark/ml/classification/Classifier.scala | 8 +----- .../classification/ProbabilisticClassifier.scala | 8 +----- .../scala/org/apache/spark/ml/clustering/LDA.scala | 8 +----- .../org/apache/spark/ml/evaluation/Evaluator.scala | 4 +-- .../org/apache/spark/ml/linalg/SQLDataTypes.scala | 4 +-- .../scala/org/apache/spark/ml/param/params.scala | 28 +----------------- .../ml/param/shared/SharedParamsCodeGen.scala | 2 -- .../spark/ml/param/shared/sharedParams.scala | 33 ---------------------- .../org/apache/spark/ml/recommendation/ALS.scala | 8 +----- .../org/apache/spark/ml/regression/Regressor.scala | 5 ---- .../org/apache/spark/ml/tree/treeParams.scala | 6 ++-- .../org/apache/spark/ml/util/Identifiable.scala | 10 +------ .../scala/org/apache/spark/ml/util/ReadWrite.scala | 8 +----- .../org/apache/spark/mllib/clustering/LDA.scala | 8 +----- .../spark/mllib/clustering/LDAOptimizer.scala | 11 +------- .../spark/mllib/feature/StandardScaler.scala | 10 +------ .../spark/mllib/feature/VectorTransformer.scala | 4 +-- .../apache/spark/mllib/optimization/Gradient.scala | 9 ------ .../spark/mllib/optimization/GradientDescent.scala | 5 ---- .../apache/spark/mllib/optimization/LBFGS.scala | 5 ---- .../spark/mllib/optimization/Optimizer.scala | 3 -- .../apache/spark/mllib/optimization/Updater.scala | 9 ------ .../apache/spark/mllib/pmml/PMMLExportable.scala | 4 +-- .../spark/mllib/random/RandomDataGenerator.scala | 18 +----------- .../org/apache/spark/mllib/random/RandomRDDs.scala | 20 ++----------- .../spark/mllib/rdd/MLPairRDDFunctions.scala | 7 ----- .../org/apache/spark/mllib/rdd/RDDFunctions.scala | 8 +----- .../apache/spark/mllib/recommendation/ALS.scala | 8 +----- .../regression/GeneralizedLinearAlgorithm.scala | 6 +--- .../regression/StreamingLinearAlgorithm.scala | 4 +-- .../mllib/stat/MultivariateOnlineSummarizer.scala | 4 +-- .../stat/distribution/MultivariateGaussian.scala | 4 +-- .../apache/spark/mllib/tree/impurity/Entropy.scala | 6 +--- .../apache/spark/mllib/tree/impurity/Gini.scala | 6 +--- .../spark/mllib/tree/impurity/Impurity.scala | 6 +--- .../spark/mllib/tree/impurity/Variance.scala | 7 +---- .../spark/mllib/tree/loss/AbsoluteError.scala | 4 +-- .../org/apache/spark/mllib/tree/loss/LogLoss.scala | 4 +-- .../org/apache/spark/mllib/tree/loss/Loss.scala | 4 +-- .../spark/mllib/tree/loss/SquaredError.scala | 4 +-- .../mllib/tree/model/InformationGainStats.scala | 4 +-- .../org/apache/spark/mllib/tree/model/Node.scala | 4 +-- .../apache/spark/mllib/tree/model/Predict.scala | 4 +-- .../org/apache/spark/mllib/tree/model/Split.scala | 4 +-- .../mllib/tree/model/treeEnsembleModels.scala | 6 +--- .../apache/spark/mllib/util/DataValidators.scala | 4 +-- .../spark/mllib/util/KMeansDataGenerator.scala | 4 +-- .../spark/mllib/util/LinearDataGenerator.scala | 4 +-- .../util/LogisticRegressionDataGenerator.scala | 4 +-- .../apache/spark/mllib/util/MFDataGenerator.scala | 4 +-- .../apache/spark/mllib/util/SVMDataGenerator.scala | 4 +-- .../apache/spark/mllib/util/modelSaveLoad.scala | 8 +----- python/pyspark/ml/base.py | 2 -- python/pyspark/ml/param/__init__.py | 2 -- python/pyspark/ml/pipeline.py | 2 -- python/pyspark/ml/util.py | 8 ------ python/pyspark/mllib/feature.py | 2 -- 65 files changed, 49 insertions(+), 397 deletions(-) diff --git a/mllib/src/main/scala/org/apache/spark/ml/Estimator.scala b/mllib/src/main/scala/org/apache/spark/ml/Estimator.scala index 1247882..8815eb2 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/Estimator.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/Estimator.scala @@ -19,15 +19,13 @@ package org.apache.spark.ml import scala.annotation.varargs -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since import org.apache.spark.ml.param.{ParamMap, ParamPair} import org.apache.spark.sql.Dataset /** - * :: DeveloperApi :: * Abstract class for estimators that fit models to data. */ -@DeveloperApi abstract class Estimator[M <: Model[M]] extends PipelineStage { /** diff --git a/mllib/src/main/scala/org/apache/spark/ml/Model.scala b/mllib/src/main/scala/org/apache/spark/ml/Model.scala index c581fed..709a112 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/Model.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/Model.scala @@ -17,16 +17,13 @@ package org.apache.spark.ml -import org.apache.spark.annotation.DeveloperApi import org.apache.spark.ml.param.ParamMap /** - * :: DeveloperApi :: * A fitted model, i.e., a [[Transformer]] produced by an [[Estimator]]. * * @tparam M model type */ -@DeveloperApi abstract class Model[M <: Model[M]] extends Transformer { /** * The parent estimator that produced this model. diff --git a/mllib/src/main/scala/org/apache/spark/ml/Pipeline.scala b/mllib/src/main/scala/org/apache/spark/ml/Pipeline.scala index d44031f..0d2ede5 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/Pipeline.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/Pipeline.scala @@ -36,14 +36,11 @@ import org.apache.spark.sql.{DataFrame, Dataset} import org.apache.spark.sql.types.StructType /** - * :: DeveloperApi :: * A stage in a pipeline, either an [[Estimator]] or a [[Transformer]]. */ -@DeveloperApi abstract class PipelineStage extends Params with Logging { /** - * :: DeveloperApi :: * * Check transform validity and derive the output schema from the input schema. * @@ -54,7 +51,6 @@ abstract class PipelineStage extends Params with Logging { * Typical implementation should first conduct verification on schema change and parameter * validity, including complex parameter interaction checks. */ - @DeveloperApi def transformSchema(schema: StructType): StructType /** diff --git a/mllib/src/main/scala/org/apache/spark/ml/Predictor.scala b/mllib/src/main/scala/org/apache/spark/ml/Predictor.scala index 98dd692..febeba7 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/Predictor.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/Predictor.scala @@ -17,7 +17,7 @@ package org.apache.spark.ml -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since import org.apache.spark.ml.feature.{Instance, LabeledPoint} import org.apache.spark.ml.linalg.{Vector, VectorUDT} import org.apache.spark.ml.param._ @@ -99,7 +99,6 @@ private[ml] trait PredictorParams extends Params } /** - * :: DeveloperApi :: * Abstraction for prediction problems (regression and classification). It accepts all NumericType * labels and will automatically cast it to DoubleType in `fit()`. If this predictor supports * weights, it accepts all NumericType weights, which will be automatically casted to DoubleType @@ -112,7 +111,6 @@ private[ml] trait PredictorParams extends Params * @tparam M Specialization of [[PredictionModel]]. If you subclass this type, use this type * parameter to specify the concrete type for the corresponding model. */ -@DeveloperApi abstract class Predictor[ FeaturesType, Learner <: Predictor[FeaturesType, Learner, M], @@ -190,7 +188,6 @@ abstract class Predictor[ } /** - * :: DeveloperApi :: * Abstraction for a model for prediction tasks (regression and classification). * * @tparam FeaturesType Type of features. @@ -198,7 +195,6 @@ abstract class Predictor[ * @tparam M Specialization of [[PredictionModel]]. If you subclass this type, use this type * parameter to specify the concrete type for the corresponding model. */ -@DeveloperApi abstract class PredictionModel[FeaturesType, M <: PredictionModel[FeaturesType, M]] extends Model[M] with PredictorParams { diff --git a/mllib/src/main/scala/org/apache/spark/ml/Transformer.scala b/mllib/src/main/scala/org/apache/spark/ml/Transformer.scala index 1652131..3b60b5a 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/Transformer.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/Transformer.scala @@ -20,7 +20,7 @@ package org.apache.spark.ml import scala.annotation.varargs import scala.reflect.runtime.universe.TypeTag -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since import org.apache.spark.internal.Logging import org.apache.spark.ml.param._ import org.apache.spark.ml.param.shared._ @@ -29,10 +29,8 @@ import org.apache.spark.sql.functions._ import org.apache.spark.sql.types._ /** - * :: DeveloperApi :: * Abstract class for transformers that transform one dataset into another. */ -@DeveloperApi abstract class Transformer extends PipelineStage { /** @@ -75,11 +73,9 @@ abstract class Transformer extends PipelineStage { } /** - * :: DeveloperApi :: * Abstract class for transformers that take one input column, apply transformation, and output the * result as a new column. */ -@DeveloperApi abstract class UnaryTransformer[IN: TypeTag, OUT: TypeTag, T <: UnaryTransformer[IN, OUT, T]] extends Transformer with HasInputCol with HasOutputCol with Logging { diff --git a/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeGroup.scala b/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeGroup.scala index b3d24b7..91dd6ab 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeGroup.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeGroup.scala @@ -19,12 +19,10 @@ package org.apache.spark.ml.attribute import scala.collection.mutable.ArrayBuffer -import org.apache.spark.annotation.DeveloperApi import org.apache.spark.ml.linalg.VectorUDT import org.apache.spark.sql.types.{Metadata, MetadataBuilder, StructField} /** - * :: DeveloperApi :: * Attributes that describe a vector ML column. * * @param name name of the attribute group (the ML column name) @@ -33,7 +31,6 @@ import org.apache.spark.sql.types.{Metadata, MetadataBuilder, StructField} * @param attrs optional array of attributes. Attribute will be copied with their corresponding * indices in the array. */ -@DeveloperApi class AttributeGroup private ( val name: String, val numAttributes: Option[Int], @@ -186,10 +183,8 @@ class AttributeGroup private ( } /** - * :: DeveloperApi :: * Factory methods to create attribute groups. */ -@DeveloperApi object AttributeGroup { import AttributeKeys._ diff --git a/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeType.scala b/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeType.scala index 078fecf..f1f3a97 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeType.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeType.scala @@ -17,20 +17,13 @@ package org.apache.spark.ml.attribute -import org.apache.spark.annotation.DeveloperApi /** - * :: DeveloperApi :: * An enum-like type for attribute types: [[AttributeType$#Numeric]], [[AttributeType$#Nominal]], * and [[AttributeType$#Binary]]. */ -@DeveloperApi sealed abstract class AttributeType(val name: String) -/** - * :: DeveloperApi :: - */ -@DeveloperApi object AttributeType { /** Numeric type. */ diff --git a/mllib/src/main/scala/org/apache/spark/ml/attribute/attributes.scala b/mllib/src/main/scala/org/apache/spark/ml/attribute/attributes.scala index 756dd67..576c771 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/attribute/attributes.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/attribute/attributes.scala @@ -19,14 +19,11 @@ package org.apache.spark.ml.attribute import scala.annotation.varargs -import org.apache.spark.annotation.DeveloperApi import org.apache.spark.sql.types.{DoubleType, Metadata, MetadataBuilder, NumericType, StructField} /** - * :: DeveloperApi :: * Abstract class for ML attributes. */ -@DeveloperApi sealed abstract class Attribute extends Serializable { name.foreach { n => @@ -150,10 +147,6 @@ private[attribute] trait AttributeFactory { def fromStructField(field: StructField): Attribute = decodeStructField(field, false) } -/** - * :: DeveloperApi :: - */ -@DeveloperApi object Attribute extends AttributeFactory { private[attribute] override def fromMetadata(metadata: Metadata): Attribute = { @@ -182,7 +175,6 @@ object Attribute extends AttributeFactory { /** - * :: DeveloperApi :: * A numeric attribute with optional summary statistics. * @param name optional name * @param index optional index @@ -191,7 +183,6 @@ object Attribute extends AttributeFactory { * @param std optional standard deviation * @param sparsity optional sparsity (ratio of zeros) */ -@DeveloperApi class NumericAttribute private[ml] ( override val name: Option[String] = None, override val index: Option[Int] = None, @@ -299,10 +290,8 @@ class NumericAttribute private[ml] ( } /** - * :: DeveloperApi :: * Factory methods for numeric attributes. */ -@DeveloperApi object NumericAttribute extends AttributeFactory { /** The default numeric attribute. */ @@ -321,7 +310,6 @@ object NumericAttribute extends AttributeFactory { } /** - * :: DeveloperApi :: * A nominal attribute. * @param name optional name * @param index optional index @@ -330,7 +318,6 @@ object NumericAttribute extends AttributeFactory { * defined. * @param values optional values. At most one of `numValues` and `values` can be defined. */ -@DeveloperApi class NominalAttribute private[ml] ( override val name: Option[String] = None, override val index: Option[Int] = None, @@ -464,10 +451,8 @@ class NominalAttribute private[ml] ( } /** - * :: DeveloperApi :: * Factory methods for nominal attributes. */ -@DeveloperApi object NominalAttribute extends AttributeFactory { /** The default nominal attribute. */ @@ -487,13 +472,11 @@ object NominalAttribute extends AttributeFactory { } /** - * :: DeveloperApi :: * A binary attribute. * @param name optional name * @param index optional index * @param values optional values. If set, its size must be 2. */ -@DeveloperApi class BinaryAttribute private[ml] ( override val name: Option[String] = None, override val index: Option[Int] = None, @@ -566,10 +549,8 @@ class BinaryAttribute private[ml] ( } /** - * :: DeveloperApi :: * Factory methods for binary attributes. */ -@DeveloperApi object BinaryAttribute extends AttributeFactory { /** The default binary attribute. */ @@ -586,10 +567,8 @@ object BinaryAttribute extends AttributeFactory { } /** - * :: DeveloperApi :: * An unresolved attribute. */ -@DeveloperApi object UnresolvedAttribute extends Attribute { override def attrType: AttributeType = AttributeType.Unresolved diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/Classifier.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/Classifier.scala index ac92299..1f3f291 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/classification/Classifier.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/classification/Classifier.scala @@ -18,7 +18,7 @@ package org.apache.spark.ml.classification import org.apache.spark.SparkException -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since import org.apache.spark.ml.{PredictionModel, Predictor, PredictorParams} import org.apache.spark.ml.feature.{Instance, LabeledPoint} import org.apache.spark.ml.linalg.{Vector, VectorUDT} @@ -62,8 +62,6 @@ private[spark] trait ClassifierParams } /** - * :: DeveloperApi :: - * * Single-label binary or multiclass classification. * Classes are indexed {0, 1, ..., numClasses - 1}. * @@ -71,7 +69,6 @@ private[spark] trait ClassifierParams * @tparam E Concrete Estimator type * @tparam M Concrete Model type */ -@DeveloperApi abstract class Classifier[ FeaturesType, E <: Classifier[FeaturesType, E, M], @@ -166,15 +163,12 @@ abstract class Classifier[ } /** - * :: DeveloperApi :: - * * Model produced by a [[Classifier]]. * Classes are indexed {0, 1, ..., numClasses - 1}. * * @tparam FeaturesType Type of input features. E.g., `Vector` * @tparam M Concrete Model type */ -@DeveloperApi abstract class ClassificationModel[FeaturesType, M <: ClassificationModel[FeaturesType, M]] extends PredictionModel[FeaturesType, M] with ClassifierParams { diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/ProbabilisticClassifier.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/ProbabilisticClassifier.scala index cc5da34..9758e3c 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/classification/ProbabilisticClassifier.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/classification/ProbabilisticClassifier.scala @@ -17,7 +17,7 @@ package org.apache.spark.ml.classification -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since import org.apache.spark.ml.linalg.{DenseVector, Vector, VectorUDT} import org.apache.spark.ml.param.shared._ import org.apache.spark.ml.util.SchemaUtils @@ -41,15 +41,12 @@ private[ml] trait ProbabilisticClassifierParams /** - * :: DeveloperApi :: - * * Single-label binary or multiclass classifier which can output class conditional probabilities. * * @tparam FeaturesType Type of input features. E.g., `Vector` * @tparam E Concrete Estimator type * @tparam M Concrete Model type */ -@DeveloperApi abstract class ProbabilisticClassifier[ FeaturesType, E <: ProbabilisticClassifier[FeaturesType, E, M], @@ -65,15 +62,12 @@ abstract class ProbabilisticClassifier[ /** - * :: DeveloperApi :: - * * Model produced by a [[ProbabilisticClassifier]]. * Classes are indexed {0, 1, ..., numClasses - 1}. * * @tparam FeaturesType Type of input features. E.g., `Vector` * @tparam M Concrete Model type */ -@DeveloperApi abstract class ProbabilisticClassificationModel[ FeaturesType, M <: ProbabilisticClassificationModel[FeaturesType, M]] diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala index e30be8c..4e86b3b 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala @@ -26,7 +26,7 @@ import org.json4s.DefaultFormats import org.json4s.JsonAST.JObject import org.json4s.jackson.JsonMethods._ -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since import org.apache.spark.internal.Logging import org.apache.spark.ml.{Estimator, Model} import org.apache.spark.ml.linalg._ @@ -762,8 +762,6 @@ class DistributedLDAModel private[ml] ( private var _checkpointFiles: Array[String] = oldDistributedModel.checkpointFiles /** - * :: DeveloperApi :: - * * If using checkpointing and `LDA.keepLastCheckpoint` is set to true, then there may be * saved checkpoint files. This method is provided so that users can manage those files. * @@ -773,18 +771,14 @@ class DistributedLDAModel private[ml] ( * * @return Checkpoint files from training */ - @DeveloperApi @Since("2.0.0") def getCheckpointFiles: Array[String] = _checkpointFiles /** - * :: DeveloperApi :: - * * Remove any remaining checkpoint files from training. * * @see [[getCheckpointFiles]] */ - @DeveloperApi @Since("2.0.0") def deleteCheckpointFiles(): Unit = { val hadoopConf = sparkSession.sparkContext.hadoopConfiguration diff --git a/mllib/src/main/scala/org/apache/spark/ml/evaluation/Evaluator.scala b/mllib/src/main/scala/org/apache/spark/ml/evaluation/Evaluator.scala index e7b949d..f7a2d0d 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/evaluation/Evaluator.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/evaluation/Evaluator.scala @@ -17,16 +17,14 @@ package org.apache.spark.ml.evaluation -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since import org.apache.spark.ml.param.{ParamMap, Params} import org.apache.spark.sql.Dataset /** - * :: DeveloperApi :: * Abstract class for evaluators that compute metrics from predictions. */ @Since("1.5.0") -@DeveloperApi abstract class Evaluator extends Params { /** diff --git a/mllib/src/main/scala/org/apache/spark/ml/linalg/SQLDataTypes.scala b/mllib/src/main/scala/org/apache/spark/ml/linalg/SQLDataTypes.scala index a66ba27..1cab37a 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/linalg/SQLDataTypes.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/linalg/SQLDataTypes.scala @@ -17,15 +17,13 @@ package org.apache.spark.ml.linalg -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since import org.apache.spark.sql.types.DataType /** - * :: DeveloperApi :: * SQL data types for vectors and matrices. */ @Since("2.0.0") -@DeveloperApi object SQLDataTypes { /** Data type for [[Vector]]. */ diff --git a/mllib/src/main/scala/org/apache/spark/ml/param/params.scala b/mllib/src/main/scala/org/apache/spark/ml/param/params.scala index 4c50f1e..53ca35c 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/param/params.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/param/params.scala @@ -29,12 +29,11 @@ import org.json4s._ import org.json4s.jackson.JsonMethods._ import org.apache.spark.SparkException -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since import org.apache.spark.ml.linalg.{JsonMatrixConverter, JsonVectorConverter, Matrix, Vector} import org.apache.spark.ml.util.Identifiable /** - * :: DeveloperApi :: * A param with self-contained documentation and optionally default value. Primitive-typed param * should use the specialized versions, which are more friendly to Java users. * @@ -45,7 +44,6 @@ import org.apache.spark.ml.util.Identifiable * See [[ParamValidators]] for factory methods for common validation functions. * @tparam T param value type */ -@DeveloperApi class Param[T](val parent: String, val name: String, val doc: String, val isValid: T => Boolean) extends Serializable { @@ -159,11 +157,9 @@ private[ml] object Param { } /** - * :: DeveloperApi :: * Factory methods for common validation functions for `Param.isValid`. * The numerical methods only support Int, Long, Float, and Double. */ -@DeveloperApi object ParamValidators { /** (private[param]) Default validation always return true */ @@ -323,10 +319,8 @@ object ParamValidators { // specialize primitive-typed params because Java doesn't recognize scala.Double, scala.Int, ... /** - * :: DeveloperApi :: * Specialized version of `Param[Double]` for Java. */ -@DeveloperApi class DoubleParam(parent: String, name: String, doc: String, isValid: Double => Boolean) extends Param[Double](parent, name, doc, isValid) { @@ -383,10 +377,8 @@ private[param] object DoubleParam { } /** - * :: DeveloperApi :: * Specialized version of `Param[Int]` for Java. */ -@DeveloperApi class IntParam(parent: String, name: String, doc: String, isValid: Int => Boolean) extends Param[Int](parent, name, doc, isValid) { @@ -412,10 +404,8 @@ class IntParam(parent: String, name: String, doc: String, isValid: Int => Boolea } /** - * :: DeveloperApi :: * Specialized version of `Param[Float]` for Java. */ -@DeveloperApi class FloatParam(parent: String, name: String, doc: String, isValid: Float => Boolean) extends Param[Float](parent, name, doc, isValid) { @@ -473,10 +463,8 @@ private object FloatParam { } /** - * :: DeveloperApi :: * Specialized version of `Param[Long]` for Java. */ -@DeveloperApi class LongParam(parent: String, name: String, doc: String, isValid: Long => Boolean) extends Param[Long](parent, name, doc, isValid) { @@ -502,10 +490,8 @@ class LongParam(parent: String, name: String, doc: String, isValid: Long => Bool } /** - * :: DeveloperApi :: * Specialized version of `Param[Boolean]` for Java. */ -@DeveloperApi class BooleanParam(parent: String, name: String, doc: String) // No need for isValid extends Param[Boolean](parent, name, doc) { @@ -525,10 +511,8 @@ class BooleanParam(parent: String, name: String, doc: String) // No need for isV } /** - * :: DeveloperApi :: * Specialized version of `Param[Array[String]]` for Java. */ -@DeveloperApi class StringArrayParam(parent: Params, name: String, doc: String, isValid: Array[String] => Boolean) extends Param[Array[String]](parent, name, doc, isValid) { @@ -550,10 +534,8 @@ class StringArrayParam(parent: Params, name: String, doc: String, isValid: Array } /** - * :: DeveloperApi :: * Specialized version of `Param[Array[Double]]` for Java. */ -@DeveloperApi class DoubleArrayParam(parent: Params, name: String, doc: String, isValid: Array[Double] => Boolean) extends Param[Array[Double]](parent, name, doc, isValid) { @@ -580,10 +562,8 @@ class DoubleArrayParam(parent: Params, name: String, doc: String, isValid: Array } /** - * :: DeveloperApi :: * Specialized version of `Param[Array[Array[Double]]]` for Java. */ -@DeveloperApi class DoubleArrayArrayParam( parent: Params, name: String, @@ -619,10 +599,8 @@ class DoubleArrayArrayParam( } /** - * :: DeveloperApi :: * Specialized version of `Param[Array[Int]]` for Java. */ -@DeveloperApi class IntArrayParam(parent: Params, name: String, doc: String, isValid: Array[Int] => Boolean) extends Param[Array[Int]](parent, name, doc, isValid) { @@ -657,11 +635,9 @@ case class ParamPair[T] @Since("1.2.0") ( } /** - * :: DeveloperApi :: * Trait for components that take parameters. This also provides an internal param map to store * parameter values attached to the instance. */ -@DeveloperApi trait Params extends Identifiable with Serializable { /** @@ -915,13 +891,11 @@ private[ml] object Params { } /** - * :: DeveloperApi :: * Java-friendly wrapper for [[Params]]. * Java developers who need to extend [[Params]] should use this class instead. * If you need to extend an abstract class which already extends [[Params]], then that abstract * class should be Java-friendly as well. */ -@DeveloperApi abstract class JavaParams extends Params /** diff --git a/mllib/src/main/scala/org/apache/spark/ml/param/shared/SharedParamsCodeGen.scala b/mllib/src/main/scala/org/apache/spark/ml/param/shared/SharedParamsCodeGen.scala index 6194dfa..7fd5f59 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/param/shared/SharedParamsCodeGen.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/param/shared/SharedParamsCodeGen.scala @@ -207,7 +207,6 @@ private[shared] object SharedParamsCodeGen { | * Trait for shared param $name$defaultValueDoc. This trait may be changed or | * removed between minor versions. | */ - |@DeveloperApi |trait Has$Name extends Params { | | /** @@ -244,7 +243,6 @@ private[shared] object SharedParamsCodeGen { | |package org.apache.spark.ml.param.shared | - |import org.apache.spark.annotation.DeveloperApi |import org.apache.spark.ml.param._ | |// DO NOT MODIFY THIS FILE! It was generated by SharedParamsCodeGen. diff --git a/mllib/src/main/scala/org/apache/spark/ml/param/shared/sharedParams.scala b/mllib/src/main/scala/org/apache/spark/ml/param/shared/sharedParams.scala index 0c0d2b5..60203eb 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/param/shared/sharedParams.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/param/shared/sharedParams.scala @@ -17,7 +17,6 @@ package org.apache.spark.ml.param.shared -import org.apache.spark.annotation.DeveloperApi import org.apache.spark.ml.param._ // DO NOT MODIFY THIS FILE! It was generated by SharedParamsCodeGen. @@ -28,7 +27,6 @@ import org.apache.spark.ml.param._ * Trait for shared param regParam. This trait may be changed or * removed between minor versions. */ -@DeveloperApi trait HasRegParam extends Params { /** @@ -45,7 +43,6 @@ trait HasRegParam extends Params { * Trait for shared param maxIter. This trait may be changed or * removed between minor versions. */ -@DeveloperApi trait HasMaxIter extends Params { /** @@ -62,7 +59,6 @@ trait HasMaxIter extends Params { * Trait for shared param featuresCol (default: "features"). This trait may be changed or * removed between minor versions. */ -@DeveloperApi trait HasFeaturesCol extends Params { /** @@ -81,7 +77,6 @@ trait HasFeaturesCol extends Params { * Trait for shared param labelCol (default: "label"). This trait may be changed or * removed between minor versions. */ -@DeveloperApi trait HasLabelCol extends Params { /** @@ -100,7 +95,6 @@ trait HasLabelCol extends Params { * Trait for shared param predictionCol (default: "prediction"). This trait may be changed or * removed between minor versions. */ -@DeveloperApi trait HasPredictionCol extends Params { /** @@ -119,7 +113,6 @@ trait HasPredictionCol extends Params { * Trait for shared param rawPredictionCol (default: "rawPrediction"). This trait may be changed or * removed between minor versions. */ -@DeveloperApi trait HasRawPredictionCol extends Params { /** @@ -138,7 +131,6 @@ trait HasRawPredictionCol extends Params { * Trait for shared param probabilityCol (default: "probability"). This trait may be changed or * removed between minor versions. */ -@DeveloperApi trait HasProbabilityCol extends Params { /** @@ -157,7 +149,6 @@ trait HasProbabilityCol extends Params { * Trait for shared param varianceCol. This trait may be changed or * removed between minor versions. */ -@DeveloperApi trait HasVarianceCol extends Params { /** @@ -174,7 +165,6 @@ trait HasVarianceCol extends Params { * Trait for shared param threshold. This trait may be changed or * removed between minor versions. */ -@DeveloperApi trait HasThreshold extends Params { /** @@ -191,7 +181,6 @@ trait HasThreshold extends Params { * Trait for shared param thresholds. This trait may be changed or * removed between minor versions. */ -@DeveloperApi trait HasThresholds extends Params { /** @@ -208,7 +197,6 @@ trait HasThresholds extends Params { * Trait for shared param inputCol. This trait may be changed or * removed between minor versions. */ -@DeveloperApi trait HasInputCol extends Params { /** @@ -225,7 +213,6 @@ trait HasInputCol extends Params { * Trait for shared param inputCols. This trait may be changed or * removed between minor versions. */ -@DeveloperApi trait HasInputCols extends Params { /** @@ -242,7 +229,6 @@ trait HasInputCols extends Params { * Trait for shared param outputCol (default: uid + "__output"). This trait may be changed or * removed between minor versions. */ -@DeveloperApi trait HasOutputCol extends Params { /** @@ -261,7 +247,6 @@ trait HasOutputCol extends Params { * Trait for shared param outputCols. This trait may be changed or * removed between minor versions. */ -@DeveloperApi trait HasOutputCols extends Params { /** @@ -278,7 +263,6 @@ trait HasOutputCols extends Params { * Trait for shared param numFeatures (default: 262144). This trait may be changed or * removed between minor versions. */ -@DeveloperApi trait HasNumFeatures extends Params { /** @@ -297,7 +281,6 @@ trait HasNumFeatures extends Params { * Trait for shared param checkpointInterval. This trait may be changed or * removed between minor versions. */ -@DeveloperApi trait HasCheckpointInterval extends Params { /** @@ -314,7 +297,6 @@ trait HasCheckpointInterval extends Params { * Trait for shared param fitIntercept (default: true). This trait may be changed or * removed between minor versions. */ -@DeveloperApi trait HasFitIntercept extends Params { /** @@ -333,7 +315,6 @@ trait HasFitIntercept extends Params { * Trait for shared param handleInvalid. This trait may be changed or * removed between minor versions. */ -@DeveloperApi trait HasHandleInvalid extends Params { /** @@ -350,7 +331,6 @@ trait HasHandleInvalid extends Params { * Trait for shared param standardization (default: true). This trait may be changed or * removed between minor versions. */ -@DeveloperApi trait HasStandardization extends Params { /** @@ -369,7 +349,6 @@ trait HasStandardization extends Params { * Trait for shared param seed (default: this.getClass.getName.hashCode.toLong). This trait may be changed or * removed between minor versions. */ -@DeveloperApi trait HasSeed extends Params { /** @@ -388,7 +367,6 @@ trait HasSeed extends Params { * Trait for shared param elasticNetParam. This trait may be changed or * removed between minor versions. */ -@DeveloperApi trait HasElasticNetParam extends Params { /** @@ -405,7 +383,6 @@ trait HasElasticNetParam extends Params { * Trait for shared param tol. This trait may be changed or * removed between minor versions. */ -@DeveloperApi trait HasTol extends Params { /** @@ -422,7 +399,6 @@ trait HasTol extends Params { * Trait for shared param relativeError (default: 0.001). This trait may be changed or * removed between minor versions. */ -@DeveloperApi trait HasRelativeError extends Params { /** @@ -441,7 +417,6 @@ trait HasRelativeError extends Params { * Trait for shared param stepSize. This trait may be changed or * removed between minor versions. */ -@DeveloperApi trait HasStepSize extends Params { /** @@ -458,7 +433,6 @@ trait HasStepSize extends Params { * Trait for shared param weightCol. This trait may be changed or * removed between minor versions. */ -@DeveloperApi trait HasWeightCol extends Params { /** @@ -475,7 +449,6 @@ trait HasWeightCol extends Params { * Trait for shared param solver. This trait may be changed or * removed between minor versions. */ -@DeveloperApi trait HasSolver extends Params { /** @@ -492,7 +465,6 @@ trait HasSolver extends Params { * Trait for shared param aggregationDepth (default: 2). This trait may be changed or * removed between minor versions. */ -@DeveloperApi trait HasAggregationDepth extends Params { /** @@ -511,7 +483,6 @@ trait HasAggregationDepth extends Params { * Trait for shared param collectSubModels (default: false). This trait may be changed or * removed between minor versions. */ -@DeveloperApi trait HasCollectSubModels extends Params { /** @@ -530,7 +501,6 @@ trait HasCollectSubModels extends Params { * Trait for shared param loss. This trait may be changed or * removed between minor versions. */ -@DeveloperApi trait HasLoss extends Params { /** @@ -547,7 +517,6 @@ trait HasLoss extends Params { * Trait for shared param distanceMeasure (default: "euclidean"). This trait may be changed or * removed between minor versions. */ -@DeveloperApi trait HasDistanceMeasure extends Params { /** @@ -566,7 +535,6 @@ trait HasDistanceMeasure extends Params { * Trait for shared param validationIndicatorCol. This trait may be changed or * removed between minor versions. */ -@DeveloperApi trait HasValidationIndicatorCol extends Params { /** @@ -583,7 +551,6 @@ trait HasValidationIndicatorCol extends Params { * Trait for shared param blockSize. This trait may be changed or * removed between minor versions. */ -@DeveloperApi trait HasBlockSize extends Params { /** diff --git a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala index 9404dcd..412f8bf 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala @@ -32,7 +32,7 @@ import org.json4s.DefaultFormats import org.json4s.JsonDSL._ import org.apache.spark.{Dependency, Partitioner, ShuffleDependency, SparkContext, SparkException} -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since import org.apache.spark.internal.Logging import org.apache.spark.ml.{Estimator, Model} import org.apache.spark.ml.linalg.BLAS @@ -731,21 +731,17 @@ class ALS(@Since("1.4.0") override val uid: String) extends Estimator[ALSModel] /** - * :: DeveloperApi :: * An implementation of ALS that supports generic ID types, specialized for Int and Long. This is * exposed as a developer API for users who do need other ID types. But it is not recommended * because it increases the shuffle size and memory requirement during training. For simplicity, * users and items must have the same type. The number of distinct users/items should be smaller * than 2 billion. */ -@DeveloperApi object ALS extends DefaultParamsReadable[ALS] with Logging { /** - * :: DeveloperApi :: * Rating class for better code readability. */ - @DeveloperApi case class Rating[@specialized(Int, Long) ID](user: ID, item: ID, rating: Float) @Since("1.6.0") @@ -908,7 +904,6 @@ object ALS extends DefaultParamsReadable[ALS] with Logging { } /** - * :: DeveloperApi :: * Implementation of the ALS algorithm. * * This implementation of the ALS factorization algorithm partitions the two sets of factors among @@ -933,7 +928,6 @@ object ALS extends DefaultParamsReadable[ALS] with Logging { * "block" as referring to a subset of an RDD containing the ratings rather than a contiguous * submatrix of the ratings matrix. */ - @DeveloperApi def train[ID: ClassTag]( // scalastyle:ignore ratings: RDD[Rating[ID]], rank: Int = 10, diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/Regressor.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/Regressor.scala index 314cf42..c28dac6 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/regression/Regressor.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/regression/Regressor.scala @@ -17,7 +17,6 @@ package org.apache.spark.ml.regression -import org.apache.spark.annotation.DeveloperApi import org.apache.spark.ml.{PredictionModel, Predictor, PredictorParams} @@ -28,7 +27,6 @@ import org.apache.spark.ml.{PredictionModel, Predictor, PredictorParams} * @tparam Learner Concrete Estimator type * @tparam M Concrete Model type */ -@DeveloperApi abstract class Regressor[ FeaturesType, Learner <: Regressor[FeaturesType, Learner, M], @@ -39,14 +37,11 @@ abstract class Regressor[ } /** - * :: DeveloperApi :: - * * Model produced by a `Regressor`. * * @tparam FeaturesType Type of input features. E.g., [[org.apache.spark.mllib.linalg.Vector]] * @tparam M Concrete Model type. */ -@DeveloperApi abstract class RegressionModel[FeaturesType, M <: RegressionModel[FeaturesType, M]] extends PredictionModel[FeaturesType, M] with PredictorParams { diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/treeParams.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/treeParams.scala index a273cd7..7e2c287 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/tree/treeParams.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/tree/treeParams.scala @@ -36,7 +36,7 @@ import org.apache.spark.sql.types.{DataType, DoubleType, StructType} /** * Parameters for Decision Tree-based algorithms. * - * Note: Marked as private and DeveloperApi since this may be made public in the future. + * Note: Marked as private since this may be made public in the future. */ private[ml] trait DecisionTreeParams extends PredictorParams with HasCheckpointInterval with HasSeed with HasWeightCol { @@ -320,7 +320,7 @@ private[spark] object TreeEnsembleParams { /** * Parameters for Decision Tree-based ensemble algorithms. * - * Note: Marked as private and DeveloperApi since this may be made public in the future. + * Note: Marked as private since this may be made public in the future. */ private[ml] trait TreeEnsembleParams extends DecisionTreeParams { @@ -477,7 +477,7 @@ private[ml] trait RandomForestRegressorParams /** * Parameters for Gradient-Boosted Tree algorithms. * - * Note: Marked as private and DeveloperApi since this may be made public in the future. + * Note: Marked as private since this may be made public in the future. */ private[ml] trait GBTParams extends TreeEnsembleParams with HasMaxIter with HasStepSize with HasValidationIndicatorCol { diff --git a/mllib/src/main/scala/org/apache/spark/ml/util/Identifiable.scala b/mllib/src/main/scala/org/apache/spark/ml/util/Identifiable.scala index bd213e7..653ffb7 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/util/Identifiable.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/util/Identifiable.scala @@ -19,18 +19,13 @@ package org.apache.spark.ml.util import java.util.UUID -import org.apache.spark.annotation.DeveloperApi - /** - * :: DeveloperApi :: - * * Trait for an object with an immutable unique ID that identifies itself and its derivatives. * * WARNING: There have not yet been final discussions on this API, so it may be broken in future * releases. */ -@DeveloperApi trait Identifiable { /** @@ -41,10 +36,7 @@ trait Identifiable { override def toString: String = uid } -/** - * :: DeveloperApi :: - */ -@DeveloperApi + object Identifiable { /** diff --git a/mllib/src/main/scala/org/apache/spark/ml/util/ReadWrite.scala b/mllib/src/main/scala/org/apache/spark/ml/util/ReadWrite.scala index dd0c40f..fec05cc 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/util/ReadWrite.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/util/ReadWrite.scala @@ -31,7 +31,7 @@ import org.json4s.JsonDSL._ import org.json4s.jackson.JsonMethods._ import org.apache.spark.{SparkContext, SparkException} -import org.apache.spark.annotation.{DeveloperApi, Since, Unstable} +import org.apache.spark.annotation.{Since, Unstable} import org.apache.spark.internal.Logging import org.apache.spark.ml._ import org.apache.spark.ml.classification.{OneVsRest, OneVsRestModel} @@ -301,8 +301,6 @@ trait GeneralMLWritable extends MLWritable { } /** - * :: DeveloperApi :: - * * Helper trait for making simple `Params` types writable. If a `Params` class stores * all data as [[org.apache.spark.ml.param.Param]] values, then extending this trait will provide * a default implementation of writing saved instances of the class. @@ -311,7 +309,6 @@ trait GeneralMLWritable extends MLWritable { * * @see `DefaultParamsReadable`, the counterpart to this trait */ -@DeveloperApi trait DefaultParamsWritable extends MLWritable { self: Params => override def write: MLWriter = new DefaultParamsWriter(this) @@ -360,8 +357,6 @@ trait MLReadable[T] { /** - * :: DeveloperApi :: - * * Helper trait for making simple `Params` types readable. If a `Params` class stores * all data as [[org.apache.spark.ml.param.Param]] values, then extending this trait will provide * a default implementation of reading saved instances of the class. @@ -371,7 +366,6 @@ trait MLReadable[T] { * @tparam T ML instance type * @see `DefaultParamsWritable`, the counterpart to this trait */ -@DeveloperApi trait DefaultParamsReadable[T] extends MLReadable[T] { override def read: MLReader[T] = new DefaultParamsReader[T] diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDA.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDA.scala index e858ac9..8c43c0b 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDA.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDA.scala @@ -21,7 +21,7 @@ import java.util.Locale import breeze.linalg.{DenseVector => BDV} -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since import org.apache.spark.api.java.JavaPairRDD import org.apache.spark.graphx._ import org.apache.spark.internal.Logging @@ -281,21 +281,15 @@ class LDA private ( /** - * :: DeveloperApi :: - * * LDAOptimizer used to perform the actual calculation */ @Since("1.4.0") - @DeveloperApi def getOptimizer: LDAOptimizer = ldaOptimizer /** - * :: DeveloperApi :: - * * LDAOptimizer used to perform the actual calculation (default = EMLDAOptimizer) */ @Since("1.4.0") - @DeveloperApi def setOptimizer(optimizer: LDAOptimizer): this.type = { this.ldaOptimizer = optimizer this diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAOptimizer.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAOptimizer.scala index dc90f6c..b2742ee 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAOptimizer.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAOptimizer.scala @@ -23,7 +23,7 @@ import breeze.linalg.{all, normalize, sum, DenseMatrix => BDM, DenseVector => BD import breeze.numerics.{abs, exp, trigamma} import breeze.stats.distributions.{Gamma, RandBasis} -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since import org.apache.spark.graphx._ import org.apache.spark.graphx.util.PeriodicGraphCheckpointer import org.apache.spark.internal.Logging @@ -33,13 +33,10 @@ import org.apache.spark.storage.StorageLevel /** - * :: DeveloperApi :: - * * An LDAOptimizer specifies which optimization/learning/inference algorithm to use, and it can * hold optimizer-specific parameters for users to set. */ @Since("1.4.0") -@DeveloperApi trait LDAOptimizer { /* @@ -62,8 +59,6 @@ trait LDAOptimizer { } /** - * :: DeveloperApi :: - * * Optimizer for EM algorithm which stores data + parameter graph, plus algorithm parameters. * * Currently, the underlying implementation uses Expectation-Maximization (EM), implemented @@ -78,7 +73,6 @@ trait LDAOptimizer { * "On Smoothing and Inference for Topic Models." UAI, 2009. */ @Since("1.4.0") -@DeveloperApi final class EMLDAOptimizer extends LDAOptimizer { import LDA._ @@ -253,8 +247,6 @@ final class EMLDAOptimizer extends LDAOptimizer { /** - * :: DeveloperApi :: - * * An online optimizer for LDA. The Optimizer implements the Online variational Bayes LDA * algorithm, which processes a subset of the corpus on each iteration, and updates the term-topic * distribution adaptively. @@ -263,7 +255,6 @@ final class EMLDAOptimizer extends LDAOptimizer { * Hoffman, Blei and Bach, "Online Learning for Latent Dirichlet Allocation." NIPS, 2010. */ @Since("1.4.0") -@DeveloperApi final class OnlineLDAOptimizer extends LDAOptimizer with Logging { // LDA common parameters diff --git a/mllib/src/main/scala/org/apache/spark/mllib/feature/StandardScaler.scala b/mllib/src/main/scala/org/apache/spark/mllib/feature/StandardScaler.scala index 21e01ef..78c974e 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/feature/StandardScaler.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/feature/StandardScaler.scala @@ -17,7 +17,7 @@ package org.apache.spark.mllib.feature -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since import org.apache.spark.internal.Logging import org.apache.spark.ml.feature.{StandardScalerModel => NewStandardScalerModel} import org.apache.spark.mllib.linalg.{DenseVector, SparseVector, Vector, Vectors} @@ -96,22 +96,14 @@ class StandardScalerModel @Since("1.3.0") ( @Since("1.3.0") def this(std: Vector) = this(std, null) - /** - * :: DeveloperApi :: - */ @Since("1.3.0") - @DeveloperApi def setWithMean(withMean: Boolean): this.type = { require(!(withMean && this.mean == null), "cannot set withMean to true while mean is null") this.withMean = withMean this } - /** - * :: DeveloperApi :: - */ @Since("1.3.0") - @DeveloperApi def setWithStd(withStd: Boolean): this.type = { require(!(withStd && this.std == null), "cannot set withStd to true while std is null") diff --git a/mllib/src/main/scala/org/apache/spark/mllib/feature/VectorTransformer.scala b/mllib/src/main/scala/org/apache/spark/mllib/feature/VectorTransformer.scala index 9db7250..ac58ffe 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/feature/VectorTransformer.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/feature/VectorTransformer.scala @@ -17,17 +17,15 @@ package org.apache.spark.mllib.feature -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since import org.apache.spark.api.java.JavaRDD import org.apache.spark.mllib.linalg.Vector import org.apache.spark.rdd.RDD /** - * :: DeveloperApi :: * Trait for transformation of a vector */ @Since("1.1.0") -@DeveloperApi trait VectorTransformer extends Serializable { /** diff --git a/mllib/src/main/scala/org/apache/spark/mllib/optimization/Gradient.scala b/mllib/src/main/scala/org/apache/spark/mllib/optimization/Gradient.scala index 46a1829..cdeed33 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/optimization/Gradient.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/optimization/Gradient.scala @@ -17,16 +17,13 @@ package org.apache.spark.mllib.optimization -import org.apache.spark.annotation.DeveloperApi import org.apache.spark.mllib.linalg.{DenseVector, Vector, Vectors} import org.apache.spark.mllib.linalg.BLAS.{axpy, dot, scal} import org.apache.spark.mllib.util.MLUtils /** - * :: DeveloperApi :: * Class used to compute the gradient for a loss function, given a single data point. */ -@DeveloperApi abstract class Gradient extends Serializable { /** * Compute the gradient and loss given the features of a single data point. @@ -58,7 +55,6 @@ abstract class Gradient extends Serializable { } /** - * :: DeveloperApi :: * Compute gradient and loss for a multinomial logistic loss function, as used * in multi-class classification (it is also used in binary logistic regression). * @@ -162,7 +158,6 @@ abstract class Gradient extends Serializable { * Multinomial Logistic Regression. By default, it is binary logistic regression * so numClasses will be set to 2. */ -@DeveloperApi class LogisticGradient(numClasses: Int) extends Gradient { def this() = this(2) @@ -275,13 +270,11 @@ class LogisticGradient(numClasses: Int) extends Gradient { } /** - * :: DeveloperApi :: * Compute gradient and loss for a Least-squared loss function, as used in linear regression. * This is correct for the averaged least squares loss function (mean squared error) * L = 1/2n ||A weights-y||^2 * See also the documentation for the precise formulation. */ -@DeveloperApi class LeastSquaresGradient extends Gradient { override def compute(data: Vector, label: Double, weights: Vector): (Vector, Double) = { val diff = dot(data, weights) - label @@ -303,13 +296,11 @@ class LeastSquaresGradient extends Gradient { } /** - * :: DeveloperApi :: * Compute gradient and loss for a Hinge loss function, as used in SVM binary classification. * See also the documentation for the precise formulation. * * @note This assumes that the labels are {0,1} */ -@DeveloperApi class HingeGradient extends Gradient { override def compute(data: Vector, label: Double, weights: Vector): (Vector, Double) = { val dotProduct = dot(data, weights) diff --git a/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala b/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala index b65b95d..1336ffd 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala @@ -21,7 +21,6 @@ import scala.collection.mutable.ArrayBuffer import breeze.linalg.{norm, DenseVector => BDV} -import org.apache.spark.annotation.DeveloperApi import org.apache.spark.internal.Logging import org.apache.spark.mllib.linalg.{Vector, Vectors} import org.apache.spark.rdd.RDD @@ -124,13 +123,11 @@ class GradientDescent private[spark] (private var gradient: Gradient, private va } /** - * :: DeveloperApi :: * Runs gradient descent on the given training data. * @param data training data * @param initialWeights initial weights * @return solution vector */ - @DeveloperApi def optimize(data: RDD[(Double, Vector)], initialWeights: Vector): Vector = { val (weights, _) = GradientDescent.runMiniBatchSGD( data, @@ -148,10 +145,8 @@ class GradientDescent private[spark] (private var gradient: Gradient, private va } /** - * :: DeveloperApi :: * Top-level method to run gradient descent. */ -@DeveloperApi object GradientDescent extends Logging { /** * Run stochastic gradient descent (SGD) in parallel using mini batches. diff --git a/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala b/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala index 65c1159..1ee9241 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala @@ -22,21 +22,18 @@ import scala.collection.mutable import breeze.linalg.{DenseVector => BDV} import breeze.optimize.{CachedDiffFunction, DiffFunction, LBFGS => BreezeLBFGS} -import org.apache.spark.annotation.DeveloperApi import org.apache.spark.internal.Logging import org.apache.spark.mllib.linalg.{Vector, Vectors} import org.apache.spark.mllib.linalg.BLAS.axpy import org.apache.spark.rdd.RDD /** - * :: DeveloperApi :: * Class used to solve an optimization problem using Limited-memory BFGS. * Reference: <a href="http://en.wikipedia.org/wiki/Limited-memory_BFGS"> * Wikipedia on Limited-memory BFGS</a> * @param gradient Gradient function to be used. * @param updater Updater to be used to update weights after every iteration. */ -@DeveloperApi class LBFGS(private var gradient: Gradient, private var updater: Updater) extends Optimizer with Logging { @@ -154,10 +151,8 @@ class LBFGS(private var gradient: Gradient, private var updater: Updater) } /** - * :: DeveloperApi :: * Top-level method to run L-BFGS. */ -@DeveloperApi object LBFGS extends Logging { /** * Run Limited-memory BFGS (L-BFGS) in parallel. diff --git a/mllib/src/main/scala/org/apache/spark/mllib/optimization/Optimizer.scala b/mllib/src/main/scala/org/apache/spark/mllib/optimization/Optimizer.scala index d8e5672..582b520 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/optimization/Optimizer.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/optimization/Optimizer.scala @@ -17,15 +17,12 @@ package org.apache.spark.mllib.optimization -import org.apache.spark.annotation.DeveloperApi import org.apache.spark.mllib.linalg.Vector import org.apache.spark.rdd.RDD /** - * :: DeveloperApi :: * Trait for optimization problem solvers. */ -@DeveloperApi trait Optimizer extends Serializable { /** diff --git a/mllib/src/main/scala/org/apache/spark/mllib/optimization/Updater.scala b/mllib/src/main/scala/org/apache/spark/mllib/optimization/Updater.scala index 142f0ec..1865dd9 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/optimization/Updater.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/optimization/Updater.scala @@ -21,11 +21,9 @@ import scala.math._ import breeze.linalg.{axpy => brzAxpy, norm => brzNorm, Vector => BV} -import org.apache.spark.annotation.DeveloperApi import org.apache.spark.mllib.linalg.{Vector, Vectors} /** - * :: DeveloperApi :: * Class used to perform steps (weight update) using Gradient Descent methods. * * For general minimization problems, or for regularized problems of the form @@ -37,7 +35,6 @@ import org.apache.spark.mllib.linalg.{Vector, Vectors} * The updater is responsible to also perform the update coming from the * regularization term R(w) (if any regularization is used). */ -@DeveloperApi abstract class Updater extends Serializable { /** * Compute an updated value for weights given the gradient, stepSize, iteration number and @@ -62,11 +59,9 @@ abstract class Updater extends Serializable { } /** - * :: DeveloperApi :: * A simple updater for gradient descent *without* any regularization. * Uses a step-size decreasing with the square root of the number of iterations. */ -@DeveloperApi class SimpleUpdater extends Updater { override def compute( weightsOld: Vector, @@ -83,7 +78,6 @@ class SimpleUpdater extends Updater { } /** - * :: DeveloperApi :: * Updater for L1 regularized problems. * R(w) = ||w||_1 * Uses a step-size decreasing with the square root of the number of iterations. @@ -101,7 +95,6 @@ class SimpleUpdater extends Updater { * * Equivalently, set weight component to signum(w) * max(0.0, abs(w) - shrinkageVal) */ -@DeveloperApi class L1Updater extends Updater { override def compute( weightsOld: Vector, @@ -128,12 +121,10 @@ class L1Updater extends Updater { } /** - * :: DeveloperApi :: * Updater for L2 regularized problems. * R(w) = 1/2 ||w||^2 * Uses a step-size decreasing with the square root of the number of iterations. */ -@DeveloperApi class SquaredL2Updater extends Updater { override def compute( weightsOld: Vector, diff --git a/mllib/src/main/scala/org/apache/spark/mllib/pmml/PMMLExportable.scala b/mllib/src/main/scala/org/apache/spark/mllib/pmml/PMMLExportable.scala index 340386c..32471b0 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/pmml/PMMLExportable.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/pmml/PMMLExportable.scala @@ -23,16 +23,14 @@ import javax.xml.transform.stream.StreamResult import org.jpmml.model.JAXBUtil import org.apache.spark.SparkContext -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since import org.apache.spark.mllib.pmml.export.PMMLModelExportFactory /** - * :: DeveloperApi :: * Export model to the PMML format * Predictive Model Markup Language (PMML) is an XML-based file format * developed by the Data Mining Group (www.dmg.org). */ -@DeveloperApi @Since("1.4.0") trait PMMLExportable { diff --git a/mllib/src/main/scala/org/apache/spark/mllib/random/RandomDataGenerator.scala b/mllib/src/main/scala/org/apache/spark/mllib/random/RandomDataGenerator.scala index d3b5488..3a2ecf1 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/random/RandomDataGenerator.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/random/RandomDataGenerator.scala @@ -19,14 +19,12 @@ package org.apache.spark.mllib.random import org.apache.commons.math3.distribution._ -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since import org.apache.spark.util.random.{Pseudorandom, XORShiftRandom} /** - * :: DeveloperApi :: * Trait for random data generators that generate i.i.d. data. */ -@DeveloperApi @Since("1.1.0") trait RandomDataGenerator[T] extends Pseudorandom with Serializable { @@ -45,10 +43,8 @@ trait RandomDataGenerator[T] extends Pseudorandom with Serializable { } /** - * :: DeveloperApi :: * Generates i.i.d. samples from U[0.0, 1.0] */ -@DeveloperApi @Since("1.1.0") class UniformGenerator extends RandomDataGenerator[Double] { @@ -68,10 +64,8 @@ class UniformGenerator extends RandomDataGenerator[Double] { } /** - * :: DeveloperApi :: * Generates i.i.d. samples from the standard normal distribution. */ -@DeveloperApi @Since("1.1.0") class StandardNormalGenerator extends RandomDataGenerator[Double] { @@ -91,12 +85,10 @@ class StandardNormalGenerator extends RandomDataGenerator[Double] { } /** - * :: DeveloperApi :: * Generates i.i.d. samples from the Poisson distribution with the given mean. * * @param mean mean for the Poisson distribution. */ -@DeveloperApi @Since("1.1.0") class PoissonGenerator @Since("1.1.0") ( @Since("1.1.0") val mean: Double) extends RandomDataGenerator[Double] { @@ -116,12 +108,10 @@ class PoissonGenerator @Since("1.1.0") ( } /** - * :: DeveloperApi :: * Generates i.i.d. samples from the exponential distribution with the given mean. * * @param mean mean for the exponential distribution. */ -@DeveloperApi @Since("1.3.0") class ExponentialGenerator @Since("1.3.0") ( @Since("1.3.0") val mean: Double) extends RandomDataGenerator[Double] { @@ -141,13 +131,11 @@ class ExponentialGenerator @Since("1.3.0") ( } /** - * :: DeveloperApi :: * Generates i.i.d. samples from the gamma distribution with the given shape and scale. * * @param shape shape for the gamma distribution. * @param scale scale for the gamma distribution */ -@DeveloperApi @Since("1.3.0") class GammaGenerator @Since("1.3.0") ( @Since("1.3.0") val shape: Double, @@ -168,14 +156,12 @@ class GammaGenerator @Since("1.3.0") ( } /** - * :: DeveloperApi :: * Generates i.i.d. samples from the log normal distribution with the * given mean and standard deviation. * * @param mean mean for the log normal distribution. * @param std standard deviation for the log normal distribution */ -@DeveloperApi @Since("1.3.0") class LogNormalGenerator @Since("1.3.0") ( @Since("1.3.0") val mean: Double, @@ -196,14 +182,12 @@ class LogNormalGenerator @Since("1.3.0") ( } /** - * :: DeveloperApi :: * Generates i.i.d. samples from the Weibull distribution with the * given shape and scale parameter. * * @param alpha shape parameter for the Weibull distribution. * @param beta scale parameter for the Weibull distribution. */ -@DeveloperApi class WeibullGenerator( val alpha: Double, val beta: Double) extends RandomDataGenerator[Double] { diff --git a/mllib/src/main/scala/org/apache/spark/mllib/random/RandomRDDs.scala b/mllib/src/main/scala/org/apache/spark/mllib/random/RandomRDDs.scala index 258b176..eacccfc 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/random/RandomRDDs.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/random/RandomRDDs.scala @@ -20,7 +20,7 @@ package org.apache.spark.mllib.random import scala.reflect.ClassTag import org.apache.spark.SparkContext -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since import org.apache.spark.api.java.{JavaDoubleRDD, JavaRDD, JavaSparkContext} import org.apache.spark.api.java.JavaSparkContext.fakeClassTag import org.apache.spark.mllib.linalg.Vector @@ -372,7 +372,6 @@ object RandomRDDs { /** - * :: DeveloperApi :: * Generates an RDD comprised of `i.i.d.` samples produced by the input RandomDataGenerator. * * @param sc SparkContext used to create the RDD. @@ -382,7 +381,6 @@ object RandomRDDs { * @param seed Random seed (default: a random long integer). * @return RDD[T] comprised of `i.i.d.` samples produced by generator. */ - @DeveloperApi @Since("1.1.0") def randomRDD[T: ClassTag]( sc: SparkContext, @@ -394,7 +392,6 @@ object RandomRDDs { } /** - * :: DeveloperApi :: * Generates an RDD comprised of `i.i.d.` samples produced by the input RandomDataGenerator. * * @param jsc JavaSparkContext used to create the RDD. @@ -404,7 +401,6 @@ object RandomRDDs { * @param seed Random seed (default: a random long integer). * @return RDD[T] comprised of `i.i.d.` samples produced by generator. */ - @DeveloperApi @Since("1.6.0") def randomJavaRDD[T]( jsc: JavaSparkContext, @@ -418,10 +414,8 @@ object RandomRDDs { } /** - * :: DeveloperApi :: * `RandomRDDs.randomJavaRDD` with the default seed. */ - @DeveloperApi @Since("1.6.0") def randomJavaRDD[T]( jsc: JavaSparkContext, @@ -432,10 +426,8 @@ object RandomRDDs { } /** - * :: DeveloperApi :: * `RandomRDDs.randomJavaRDD` with the default seed & numPartitions */ - @DeveloperApi @Since("1.6.0") def randomJavaRDD[T]( jsc: JavaSparkContext, @@ -831,7 +823,6 @@ object RandomRDDs { /** - * :: DeveloperApi :: * Generates an RDD[Vector] with vectors containing `i.i.d.` samples produced by the * input RandomDataGenerator. * @@ -843,7 +834,6 @@ object RandomRDDs { * @param seed Random seed (default: a random long integer). * @return RDD[Vector] with vectors containing `i.i.d.` samples produced by generator. */ - @DeveloperApi @Since("1.1.0") def randomVectorRDD(sc: SparkContext, generator: RandomDataGenerator[Double], @@ -856,10 +846,8 @@ object RandomRDDs { } /** - * :: DeveloperApi :: * Java-friendly version of `RandomRDDs.randomVectorRDD`. */ - @DeveloperApi @Since("1.6.0") def randomJavaVectorRDD( jsc: JavaSparkContext, @@ -871,11 +859,9 @@ object RandomRDDs { randomVectorRDD(jsc.sc, generator, numRows, numCols, numPartitions, seed).toJavaRDD() } - /** - * :: DeveloperApi :: + /** :: * `RandomRDDs.randomJavaVectorRDD` with the default seed. */ - @DeveloperApi @Since("1.6.0") def randomJavaVectorRDD( jsc: JavaSparkContext, @@ -887,10 +873,8 @@ object RandomRDDs { } /** - * :: DeveloperApi :: * `RandomRDDs.randomJavaVectorRDD` with the default number of partitions and the default seed. */ - @DeveloperApi @Since("1.6.0") def randomJavaVectorRDD( jsc: JavaSparkContext, diff --git a/mllib/src/main/scala/org/apache/spark/mllib/rdd/MLPairRDDFunctions.scala b/mllib/src/main/scala/org/apache/spark/mllib/rdd/MLPairRDDFunctions.scala index e28e1af..8f78bcc 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/rdd/MLPairRDDFunctions.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/rdd/MLPairRDDFunctions.scala @@ -20,15 +20,12 @@ package org.apache.spark.mllib.rdd import scala.language.implicitConversions import scala.reflect.ClassTag -import org.apache.spark.annotation.DeveloperApi import org.apache.spark.rdd.RDD import org.apache.spark.util.BoundedPriorityQueue /** - * :: DeveloperApi :: * Machine learning specific Pair RDD functions. */ -@DeveloperApi class MLPairRDDFunctions[K: ClassTag, V: ClassTag](self: RDD[(K, V)]) extends Serializable { /** * Returns the top k (largest) elements for each key from this RDD as defined by the specified @@ -51,10 +48,6 @@ class MLPairRDDFunctions[K: ClassTag, V: ClassTag](self: RDD[(K, V)]) extends Se } } -/** - * :: DeveloperApi :: - */ -@DeveloperApi object MLPairRDDFunctions { /** Implicit conversion from a pair RDD to MLPairRDDFunctions. */ implicit def fromPairRDD[K: ClassTag, V: ClassTag](rdd: RDD[(K, V)]): MLPairRDDFunctions[K, V] = diff --git a/mllib/src/main/scala/org/apache/spark/mllib/rdd/RDDFunctions.scala b/mllib/src/main/scala/org/apache/spark/mllib/rdd/RDDFunctions.scala index 32e6ecf..1a13cbb 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/rdd/RDDFunctions.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/rdd/RDDFunctions.scala @@ -20,14 +20,11 @@ package org.apache.spark.mllib.rdd import scala.language.implicitConversions import scala.reflect.ClassTag -import org.apache.spark.annotation.DeveloperApi import org.apache.spark.rdd.RDD /** - * :: DeveloperApi :: * Machine learning specific RDD functions. */ -@DeveloperApi class RDDFunctions[T: ClassTag](self: RDD[T]) extends Serializable { /** @@ -54,10 +51,7 @@ class RDDFunctions[T: ClassTag](self: RDD[T]) extends Serializable { } -/** - * :: DeveloperApi :: - */ -@DeveloperApi + object RDDFunctions { /** Implicit conversion from an RDD to RDDFunctions. */ diff --git a/mllib/src/main/scala/org/apache/spark/mllib/recommendation/ALS.scala b/mllib/src/main/scala/org/apache/spark/mllib/recommendation/ALS.scala index f3f15ba..1ae0b58 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/recommendation/ALS.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/recommendation/ALS.scala @@ -17,7 +17,7 @@ package org.apache.spark.mllib.recommendation -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since import org.apache.spark.api.java.JavaRDD import org.apache.spark.internal.Logging import org.apache.spark.ml.recommendation.{ALS => NewALS} @@ -195,12 +195,10 @@ class ALS private ( } /** - * :: DeveloperApi :: * Sets storage level for intermediate RDDs (user/product in/out links). The default value is * `MEMORY_AND_DISK`. Users can change it to a serialized storage, e.g., `MEMORY_AND_DISK_SER` and * set `spark.rdd.compress` to `true` to reduce the space requirement, at the cost of speed. */ - @DeveloperApi @Since("1.1.0") def setIntermediateRDDStorageLevel(storageLevel: StorageLevel): this.type = { require(storageLevel != StorageLevel.NONE, @@ -210,13 +208,11 @@ class ALS private ( } /** - * :: DeveloperApi :: * Sets storage level for final RDDs (user/product used in MatrixFactorizationModel). The default * value is `MEMORY_AND_DISK`. Users can change it to a serialized storage, e.g. * `MEMORY_AND_DISK_SER` and set `spark.rdd.compress` to `true` to reduce the space requirement, * at the cost of speed. */ - @DeveloperApi @Since("1.3.0") def setFinalRDDStorageLevel(storageLevel: StorageLevel): this.type = { this.finalRDDStorageLevel = storageLevel @@ -224,14 +220,12 @@ class ALS private ( } /** - * :: DeveloperApi :: * Set period (in iterations) between checkpoints (default = 10). Checkpointing helps with * recovery (when nodes fail) and StackOverflow exceptions caused by long lineage. It also helps * with eliminating temporary shuffle files on disk, which can be important when there are many * ALS iterations. If the checkpoint directory is not set in [[org.apache.spark.SparkContext]], * this setting is ignored. */ - @DeveloperApi @Since("1.4.0") def setCheckpointInterval(checkpointInterval: Int): this.type = { this.checkpointInterval = checkpointInterval diff --git a/mllib/src/main/scala/org/apache/spark/mllib/regression/GeneralizedLinearAlgorithm.scala b/mllib/src/main/scala/org/apache/spark/mllib/regression/GeneralizedLinearAlgorithm.scala index ef3ff2b..6d0b3fa 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/regression/GeneralizedLinearAlgorithm.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/regression/GeneralizedLinearAlgorithm.scala @@ -18,7 +18,7 @@ package org.apache.spark.mllib.regression import org.apache.spark.SparkException -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since import org.apache.spark.internal.Logging import org.apache.spark.mllib.feature.StandardScaler import org.apache.spark.mllib.linalg.{Vector, Vectors} @@ -28,7 +28,6 @@ import org.apache.spark.rdd.RDD import org.apache.spark.storage.StorageLevel /** - * :: DeveloperApi :: * GeneralizedLinearModel (GLM) represents a model trained using * GeneralizedLinearAlgorithm. GLMs consist of a weight vector and * an intercept. @@ -38,7 +37,6 @@ import org.apache.spark.storage.StorageLevel * */ @Since("0.8.0") -@DeveloperApi abstract class GeneralizedLinearModel @Since("1.0.0") ( @Since("1.0.0") val weights: Vector, @Since("0.8.0") val intercept: Double) @@ -94,13 +92,11 @@ abstract class GeneralizedLinearModel @Since("1.0.0") ( } /** - * :: DeveloperApi :: * GeneralizedLinearAlgorithm implements methods to train a Generalized Linear Model (GLM). * This class should be extended with an Optimizer to create a new GLM. * */ @Since("0.8.0") -@DeveloperApi abstract class GeneralizedLinearAlgorithm[M <: GeneralizedLinearModel] extends Logging with Serializable { diff --git a/mllib/src/main/scala/org/apache/spark/mllib/regression/StreamingLinearAlgorithm.scala b/mllib/src/main/scala/org/apache/spark/mllib/regression/StreamingLinearAlgorithm.scala index f44c8fe..eb83f9a 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/regression/StreamingLinearAlgorithm.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/regression/StreamingLinearAlgorithm.scala @@ -19,7 +19,7 @@ package org.apache.spark.mllib.regression import scala.reflect.ClassTag -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since import org.apache.spark.api.java.JavaSparkContext.fakeClassTag import org.apache.spark.internal.Logging import org.apache.spark.mllib.linalg.Vector @@ -27,7 +27,6 @@ import org.apache.spark.streaming.api.java.{JavaDStream, JavaPairDStream} import org.apache.spark.streaming.dstream.DStream /** - * :: DeveloperApi :: * StreamingLinearAlgorithm implements methods for continuously * training a generalized linear model on streaming data, * and using it for prediction on (possibly different) streaming data. @@ -56,7 +55,6 @@ import org.apache.spark.streaming.dstream.DStream * */ @Since("1.1.0") -@DeveloperApi abstract class StreamingLinearAlgorithm[ M <: GeneralizedLinearModel, A <: GeneralizedLinearAlgorithm[M]] extends Logging { diff --git a/mllib/src/main/scala/org/apache/spark/mllib/stat/MultivariateOnlineSummarizer.scala b/mllib/src/main/scala/org/apache/spark/mllib/stat/MultivariateOnlineSummarizer.scala index 00c1da6..829cce6 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/stat/MultivariateOnlineSummarizer.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/stat/MultivariateOnlineSummarizer.scala @@ -17,11 +17,10 @@ package org.apache.spark.mllib.stat -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since import org.apache.spark.mllib.linalg.{Vector, Vectors} /** - * :: DeveloperApi :: * MultivariateOnlineSummarizer implements [[MultivariateStatisticalSummary]] to compute the mean, * variance, minimum, maximum, counts, and nonzero counts for instances in sparse or dense vector * format in an online fashion. @@ -41,7 +40,6 @@ import org.apache.spark.mllib.linalg.{Vector, Vectors} * Reliability weights (Wikipedia)</a>. */ @Since("1.1.0") -@DeveloperApi class MultivariateOnlineSummarizer extends MultivariateStatisticalSummary with Serializable { private var n = 0 diff --git a/mllib/src/main/scala/org/apache/spark/mllib/stat/distribution/MultivariateGaussian.scala b/mllib/src/main/scala/org/apache/spark/mllib/stat/distribution/MultivariateGaussian.scala index f34c229..310eb863 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/stat/distribution/MultivariateGaussian.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/stat/distribution/MultivariateGaussian.scala @@ -19,12 +19,11 @@ package org.apache.spark.mllib.stat.distribution import breeze.linalg.{diag, eigSym, max, DenseMatrix => DBM, DenseVector => DBV, Vector => BV} -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since import org.apache.spark.mllib.linalg.{Matrices, Matrix, Vector, Vectors} import org.apache.spark.mllib.util.MLUtils /** - * :: DeveloperApi :: * This class provides basic functionality for a Multivariate Gaussian (Normal) Distribution. In * the event that the covariance matrix is singular, the density will be computed in a * reduced dimensional subspace under which the distribution is supported. @@ -35,7 +34,6 @@ import org.apache.spark.mllib.util.MLUtils * @param sigma The covariance matrix of the distribution */ @Since("1.3.0") -@DeveloperApi class MultivariateGaussian @Since("1.3.0") ( @Since("1.3.0") val mu: Vector, @Since("1.3.0") val sigma: Matrix) extends Serializable { diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Entropy.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Entropy.scala index d52cb03..6e2732f 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Entropy.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Entropy.scala @@ -17,7 +17,7 @@ package org.apache.spark.mllib.tree.impurity -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since /** * Class for calculating entropy during multiclass classification. @@ -30,14 +30,12 @@ object Entropy extends Impurity { private[tree] def log2(x: Double) = scala.math.log(x) / _log2 /** - * :: DeveloperApi :: * information calculation for multiclass classification * @param counts Array[Double] with counts for each label * @param totalCount sum of counts for all labels * @return information value, or 0 if totalCount = 0 */ @Since("1.1.0") - @DeveloperApi override def calculate(counts: Array[Double], totalCount: Double): Double = { if (totalCount == 0) { return 0 @@ -57,7 +55,6 @@ object Entropy extends Impurity { } /** - * :: DeveloperApi :: * variance calculation * @param count number of instances * @param sum sum of labels @@ -65,7 +62,6 @@ object Entropy extends Impurity { * @return information value, or 0 if count = 0 */ @Since("1.0.0") - @DeveloperApi override def calculate(count: Double, sum: Double, sumSquares: Double): Double = throw new UnsupportedOperationException("Entropy.calculate") diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Gini.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Gini.scala index 913ffbb..5983118 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Gini.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Gini.scala @@ -17,7 +17,7 @@ package org.apache.spark.mllib.tree.impurity -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since /** * Class for calculating the Gini impurity @@ -28,14 +28,12 @@ import org.apache.spark.annotation.{DeveloperApi, Since} object Gini extends Impurity { /** - * :: DeveloperApi :: * information calculation for multiclass classification * @param counts Array[Double] with counts for each label * @param totalCount sum of counts for all labels * @return information value, or 0 if totalCount = 0 */ @Since("1.1.0") - @DeveloperApi override def calculate(counts: Array[Double], totalCount: Double): Double = { if (totalCount == 0) { return 0 @@ -52,7 +50,6 @@ object Gini extends Impurity { } /** - * :: DeveloperApi :: * variance calculation * @param count number of instances * @param sum sum of labels @@ -60,7 +57,6 @@ object Gini extends Impurity { * @return information value, or 0 if count = 0 */ @Since("1.0.0") - @DeveloperApi override def calculate(count: Double, sum: Double, sumSquares: Double): Double = throw new UnsupportedOperationException("Gini.calculate") diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Impurity.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Impurity.scala index 4914734..9b1d8d7 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Impurity.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Impurity.scala @@ -19,7 +19,7 @@ package org.apache.spark.mllib.tree.impurity import java.util.Locale -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since /** * Trait for calculating information gain. @@ -31,18 +31,15 @@ import org.apache.spark.annotation.{DeveloperApi, Since} trait Impurity extends Serializable { /** - * :: DeveloperApi :: * information calculation for multiclass classification * @param counts Array[Double] with counts for each label * @param totalCount sum of counts for all labels * @return information value, or 0 if totalCount = 0 */ @Since("1.1.0") - @DeveloperApi def calculate(counts: Array[Double], totalCount: Double): Double /** - * :: DeveloperApi :: * information calculation for regression * @param count number of instances * @param sum sum of labels @@ -50,7 +47,6 @@ trait Impurity extends Serializable { * @return information value, or 0 if count = 0 */ @Since("1.0.0") - @DeveloperApi def calculate(count: Double, sum: Double, sumSquares: Double): Double } diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Variance.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Variance.scala index a07b919..f5b2f8d 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Variance.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Variance.scala @@ -17,7 +17,7 @@ package org.apache.spark.mllib.tree.impurity -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since /** * Class for calculating variance during regression @@ -26,27 +26,22 @@ import org.apache.spark.annotation.{DeveloperApi, Since} object Variance extends Impurity { /** - * :: DeveloperApi :: * information calculation for multiclass classification * @param counts Array[Double] with counts for each label * @param totalCount sum of counts for all labels * @return information value, or 0 if totalCount = 0 */ @Since("1.1.0") - @DeveloperApi override def calculate(counts: Array[Double], totalCount: Double): Double = throw new UnsupportedOperationException("Variance.calculate") /** - * :: DeveloperApi :: * variance calculation * @param count number of instances * @param sum sum of labels * @param sumSquares summation of squares of the labels * @return information value, or 0 if count = 0 */ - @Since("1.0.0") - @DeveloperApi override def calculate(count: Double, sum: Double, sumSquares: Double): Double = { if (count == 0) { return 0 diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/AbsoluteError.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/AbsoluteError.scala index 9b60d01..48e576a 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/AbsoluteError.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/AbsoluteError.scala @@ -17,11 +17,10 @@ package org.apache.spark.mllib.tree.loss -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since /** - * :: DeveloperApi :: * Class for absolute error loss calculation (for regression). * * The absolute (L1) error is defined as: @@ -29,7 +28,6 @@ import org.apache.spark.annotation.{DeveloperApi, Since} * where y is the label and F(x) is the model prediction for features x. */ @Since("1.2.0") -@DeveloperApi object AbsoluteError extends Loss { /** diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/LogLoss.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/LogLoss.scala index 9339f0a..49f1215 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/LogLoss.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/LogLoss.scala @@ -17,11 +17,10 @@ package org.apache.spark.mllib.tree.loss -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since import org.apache.spark.mllib.util.MLUtils /** - * :: DeveloperApi :: * Class for log loss calculation (for classification). * This uses twice the binomial negative log likelihood, called "deviance" in Friedman (1999). * @@ -30,7 +29,6 @@ import org.apache.spark.mllib.util.MLUtils * where y is a label in {-1, 1} and F(x) is the model prediction for features x. */ @Since("1.2.0") -@DeveloperApi object LogLoss extends ClassificationLoss { /** diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/Loss.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/Loss.scala index e7ffb3f..f589e0c 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/Loss.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/Loss.scala @@ -17,17 +17,15 @@ package org.apache.spark.mllib.tree.loss -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since import org.apache.spark.mllib.regression.LabeledPoint import org.apache.spark.mllib.tree.model.TreeEnsembleModel import org.apache.spark.rdd.RDD /** - * :: DeveloperApi :: * Trait for adding "pluggable" loss functions for the gradient boosting algorithm. */ @Since("1.2.0") -@DeveloperApi trait Loss extends Serializable { /** diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/SquaredError.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/SquaredError.scala index 4eb6810..3ff59fc 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/SquaredError.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/SquaredError.scala @@ -17,11 +17,10 @@ package org.apache.spark.mllib.tree.loss -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since /** - * :: DeveloperApi :: * Class for squared error loss calculation. * * The squared (L2) error is defined as: @@ -29,7 +28,6 @@ import org.apache.spark.annotation.{DeveloperApi, Since} * where y is the label and F(x) is the model prediction for features x. */ @Since("1.2.0") -@DeveloperApi object SquaredError extends Loss { /** diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/InformationGainStats.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/InformationGainStats.scala index f3dbfd9..54d95a3 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/InformationGainStats.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/InformationGainStats.scala @@ -17,11 +17,10 @@ package org.apache.spark.mllib.tree.model -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since import org.apache.spark.mllib.tree.impurity.ImpurityCalculator /** - * :: DeveloperApi :: * Information gain statistics for each split * @param gain information gain value * @param impurity current node impurity @@ -31,7 +30,6 @@ import org.apache.spark.mllib.tree.impurity.ImpurityCalculator * @param rightPredict right node predict */ @Since("1.0.0") -@DeveloperApi class InformationGainStats( val gain: Double, val impurity: Double, diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Node.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Node.scala index 5fd0536..bac36ce 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Node.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Node.scala @@ -17,13 +17,12 @@ package org.apache.spark.mllib.tree.model -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since import org.apache.spark.internal.Logging import org.apache.spark.mllib.linalg.Vector import org.apache.spark.mllib.tree.configuration.FeatureType._ /** - * :: DeveloperApi :: * Node in a decision tree. * * About node indexing: @@ -40,7 +39,6 @@ import org.apache.spark.mllib.tree.configuration.FeatureType._ * @param stats information gain stats */ @Since("1.0.0") -@DeveloperApi class Node @Since("1.2.0") ( @Since("1.0.0") val id: Int, @Since("1.0.0") var predict: Predict, diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Predict.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Predict.scala index 1dbdd2d..44bc625 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Predict.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Predict.scala @@ -17,16 +17,14 @@ package org.apache.spark.mllib.tree.model -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since /** - * :: DeveloperApi :: * Predicted value for a node * @param predict predicted value * @param prob probability of the label (classification only) */ @Since("1.2.0") -@DeveloperApi class Predict @Since("1.2.0") ( @Since("1.2.0") val predict: Double, @Since("1.2.0") val prob: Double = 0.0) extends Serializable { diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Split.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Split.scala index bda5e66..dbada8f 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Split.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Split.scala @@ -17,11 +17,10 @@ package org.apache.spark.mllib.tree.model -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since import org.apache.spark.mllib.tree.configuration.FeatureType.FeatureType /** - * :: DeveloperApi :: * Split applied to a feature * @param feature feature index * @param threshold Threshold for continuous feature. @@ -30,7 +29,6 @@ import org.apache.spark.mllib.tree.configuration.FeatureType.FeatureType * @param categories Split left if categorical feature value is in this set, else right. */ @Since("1.0.0") -@DeveloperApi case class Split( @Since("1.0.0") feature: Int, @Since("1.0.0") threshold: Double, diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/treeEnsembleModels.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/treeEnsembleModels.scala index f7e8d98..c506927 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/treeEnsembleModels.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/treeEnsembleModels.scala @@ -25,7 +25,7 @@ import org.json4s.JsonDSL._ import org.json4s.jackson.JsonMethods._ import org.apache.spark.SparkContext -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since import org.apache.spark.api.java.JavaRDD import org.apache.spark.internal.Logging import org.apache.spark.mllib.linalg.Vector @@ -174,7 +174,6 @@ class GradientBoostedTreesModel @Since("1.2.0") ( object GradientBoostedTreesModel extends Loader[GradientBoostedTreesModel] { /** - * :: DeveloperApi :: * Compute the initial predictions and errors for a dataset for the first * iteration of gradient boosting. * @param data: training data. @@ -185,7 +184,6 @@ object GradientBoostedTreesModel extends Loader[GradientBoostedTreesModel] { * corresponding to every sample. */ @Since("1.4.0") - @DeveloperApi def computeInitialPredictionAndError( data: RDD[LabeledPoint], initTreeWeight: Double, @@ -199,7 +197,6 @@ object GradientBoostedTreesModel extends Loader[GradientBoostedTreesModel] { } /** - * :: DeveloperApi :: * Update a zipped predictionError RDD * (as obtained with computeInitialPredictionAndError) * @param data: training data. @@ -211,7 +208,6 @@ object GradientBoostedTreesModel extends Loader[GradientBoostedTreesModel] { * corresponding to each sample. */ @Since("1.4.0") - @DeveloperApi def updatePredictionError( data: RDD[LabeledPoint], predictionAndError: RDD[(Double, Double)], diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/DataValidators.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/DataValidators.scala index 2c712d8..1a8c880 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/util/DataValidators.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/util/DataValidators.scala @@ -17,16 +17,14 @@ package org.apache.spark.mllib.util -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since import org.apache.spark.internal.Logging import org.apache.spark.mllib.regression.LabeledPoint import org.apache.spark.rdd.RDD /** - * :: DeveloperApi :: * A collection of methods used to validate data before applying ML algorithms. */ -@DeveloperApi @Since("0.8.0") object DataValidators extends Logging { diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/KMeansDataGenerator.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/KMeansDataGenerator.scala index b6eb10e..a409abc 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/util/KMeansDataGenerator.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/util/KMeansDataGenerator.scala @@ -20,16 +20,14 @@ package org.apache.spark.mllib.util import scala.util.Random import org.apache.spark.SparkContext -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since import org.apache.spark.rdd.RDD /** - * :: DeveloperApi :: * Generate test data for KMeans. This class first chooses k cluster centers * from a d-dimensional Gaussian distribution scaled by factor r and then creates a Gaussian * cluster with scale 1 around each center. */ -@DeveloperApi @Since("0.8.0") object KMeansDataGenerator { diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/LinearDataGenerator.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/LinearDataGenerator.scala index de920da..689ed0f 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/util/LinearDataGenerator.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/util/LinearDataGenerator.scala @@ -21,18 +21,16 @@ import scala.collection.JavaConverters._ import scala.util.Random import org.apache.spark.SparkContext -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since import org.apache.spark.mllib.linalg.{BLAS, Vectors} import org.apache.spark.mllib.regression.LabeledPoint import org.apache.spark.rdd.RDD /** - * :: DeveloperApi :: * Generate sample data used for Linear Data. This class generates * uniformly random values for every feature and adds Gaussian noise with mean `eps` to the * response variable `Y`. */ -@DeveloperApi @Since("0.8.0") object LinearDataGenerator { diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/LogisticRegressionDataGenerator.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/LogisticRegressionDataGenerator.scala index 7e9d946..bca4ad4 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/util/LogisticRegressionDataGenerator.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/util/LogisticRegressionDataGenerator.scala @@ -20,17 +20,15 @@ package org.apache.spark.mllib.util import scala.util.Random import org.apache.spark.SparkContext -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since import org.apache.spark.mllib.linalg.Vectors import org.apache.spark.mllib.regression.LabeledPoint import org.apache.spark.rdd.RDD /** - * :: DeveloperApi :: * Generate test data for LogisticRegression. This class chooses positive labels * with probability `probOne` and scales features for positive examples by `eps`. */ -@DeveloperApi @Since("0.8.0") object LogisticRegressionDataGenerator { diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/MFDataGenerator.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/MFDataGenerator.scala index 7a308a5..de21db8 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/util/MFDataGenerator.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/util/MFDataGenerator.scala @@ -22,12 +22,11 @@ import java.{util => ju} import scala.util.Random import org.apache.spark.SparkContext -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since import org.apache.spark.mllib.linalg.{BLAS, DenseMatrix} import org.apache.spark.rdd.RDD /** - * :: DeveloperApi :: * Generate RDD(s) containing data for Matrix Factorization. * * This method samples training entries according to the oversampling factor @@ -50,7 +49,6 @@ import org.apache.spark.rdd.RDD * test (Boolean) Whether to create testing RDD. * testSampFact (Double) Percentage of training data to use as test data. */ -@DeveloperApi @Since("0.8.0") object MFDataGenerator { @Since("0.8.0") diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/SVMDataGenerator.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/SVMDataGenerator.scala index 9f6ba02..9fffa50 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/util/SVMDataGenerator.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/util/SVMDataGenerator.scala @@ -22,17 +22,15 @@ import scala.util.Random import com.github.fommil.netlib.BLAS.{getInstance => blas} import org.apache.spark.SparkContext -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since import org.apache.spark.mllib.linalg.Vectors import org.apache.spark.mllib.regression.LabeledPoint import org.apache.spark.rdd.RDD /** - * :: DeveloperApi :: * Generate sample data used for SVM. This class generates uniform random values * for the features and adds Gaussian noise with weight 0.1 to generate labels. */ -@DeveloperApi @Since("0.8.0") object SVMDataGenerator { diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/modelSaveLoad.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/modelSaveLoad.scala index e8889bf..8f2d8b9 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/util/modelSaveLoad.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/util/modelSaveLoad.scala @@ -24,17 +24,14 @@ import org.json4s._ import org.json4s.jackson.JsonMethods._ import org.apache.spark.SparkContext -import org.apache.spark.annotation.{DeveloperApi, Since} +import org.apache.spark.annotation.Since import org.apache.spark.sql.catalyst.ScalaReflection import org.apache.spark.sql.types.{DataType, StructField, StructType} /** - * :: DeveloperApi :: - * * Trait for models and transformers which may be saved as files. * This should be inherited by the class which implements model instances. */ -@DeveloperApi @Since("1.3.0") trait Saveable { @@ -57,12 +54,9 @@ trait Saveable { } /** - * :: DeveloperApi :: - * * Trait for classes which can load models and transformers from files. * This should be inherited by an object paired with the model class. */ -@DeveloperApi @Since("1.3.0") trait Loader[M <: Saveable] { diff --git a/python/pyspark/ml/base.py b/python/pyspark/ml/base.py index b8df5a3..a24b1ca 100644 --- a/python/pyspark/ml/base.py +++ b/python/pyspark/ml/base.py @@ -96,8 +96,6 @@ class Estimator(Params): :return: A thread safe iterable which contains one model for each param map. Each call to `next(modelIterator)` will return `(index, model)` where model was fit using `paramMaps[index]`. `index` values may not be sequential. - - .. note:: DeveloperApi """ estimator = self.copy() diff --git a/python/pyspark/ml/param/__init__.py b/python/pyspark/ml/param/__init__.py index fe61f9f..1be8755 100644 --- a/python/pyspark/ml/param/__init__.py +++ b/python/pyspark/ml/param/__init__.py @@ -76,8 +76,6 @@ class Param(object): class TypeConverters(object): """ - .. note:: DeveloperApi - Factory methods for common type conversion functions for `Param.typeConverter`. .. versionadded:: 2.0.0 diff --git a/python/pyspark/ml/pipeline.py b/python/pyspark/ml/pipeline.py index 0975302..09e0748 100644 --- a/python/pyspark/ml/pipeline.py +++ b/python/pyspark/ml/pipeline.py @@ -324,8 +324,6 @@ class PipelineModel(Model, MLReadable, MLWritable): @inherit_doc class PipelineSharedReadWrite(): """ - .. note:: DeveloperApi - Functions for :py:class:`MLReader` and :py:class:`MLWriter` shared between :py:class:`Pipeline` and :py:class:`PipelineModel` diff --git a/python/pyspark/ml/util.py b/python/pyspark/ml/util.py index fe0ed56..35ad551 100644 --- a/python/pyspark/ml/util.py +++ b/python/pyspark/ml/util.py @@ -345,8 +345,6 @@ class JavaMLReadable(MLReadable): @inherit_doc class DefaultParamsWritable(MLWritable): """ - .. note:: DeveloperApi - Helper trait for making simple :py:class:`Params` types writable. If a :py:class:`Params` class stores all data as :py:class:`Param` values, then extending this trait will provide a default implementation of writing saved instances of the class. @@ -370,8 +368,6 @@ class DefaultParamsWritable(MLWritable): @inherit_doc class DefaultParamsWriter(MLWriter): """ - .. note:: DeveloperApi - Specialization of :py:class:`MLWriter` for :py:class:`Params` types Class for writing Estimators and Transformers whose parameters are JSON-serializable. @@ -445,8 +441,6 @@ class DefaultParamsWriter(MLWriter): @inherit_doc class DefaultParamsReadable(MLReadable): """ - .. note:: DeveloperApi - Helper trait for making simple :py:class:`Params` types readable. If a :py:class:`Params` class stores all data as :py:class:`Param` values, then extending this trait will provide a default implementation of reading saved @@ -466,8 +460,6 @@ class DefaultParamsReadable(MLReadable): @inherit_doc class DefaultParamsReader(MLReader): """ - .. note:: DeveloperApi - Specialization of :py:class:`MLReader` for :py:class:`Params` types Default :py:class:`MLReader` implementation for transformers and estimators that diff --git a/python/pyspark/mllib/feature.py b/python/pyspark/mllib/feature.py index 905c4da..3efae6ff 100644 --- a/python/pyspark/mllib/feature.py +++ b/python/pyspark/mllib/feature.py @@ -43,8 +43,6 @@ __all__ = ['Normalizer', 'StandardScalerModel', 'StandardScaler', class VectorTransformer(object): """ - .. note:: DeveloperApi - Base class for transformation of a vector or RDD of vector """ def transform(self, vector): --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org