Github user jkbradley commented on a diff in the pull request:
https://github.com/apache/spark/pull/12663#discussion_r60955574
--- Diff:
mllib/src/main/scala/org/apache/spark/ml/classification/Classifier.scala ---
@@ -62,6 +65,76 @@ abstract class Classifier[
def setRawPredictionCol(value: String): E = set(rawPredictionCol,
value).asInstanceOf[E]
// TODO: defaultEvaluator (follow-up PR)
+
+ /**
+ * Extract [[labelCol]] and [[featuresCol]] from the given dataset,
+ * and put it in an RDD with strong types.
+ * @throws SparkException if any label is not an integer >= 0
+ */
+ override protected def extractLabeledPoints(dataset: Dataset[_]):
RDD[LabeledPoint] = {
+ dataset.select(col($(labelCol)).cast(DoubleType),
col($(featuresCol))).rdd.map {
+ case Row(label: Double, features: Vector) =>
+ require(label % 1 == 0 && label >= 0, s"Classifier was given
dataset with invalid label" +
+ s" $label. Labels must be integers in range [0, 1, ...,
numClasses-1]")
+ LabeledPoint(label, features)
+ }
+ }
+
+ /**
+ * Get the number of classes. This looks in column metadata first, and
if that is missing,
+ * then this assumes classes are indexed 0,1,...,numClasses-1 and
computes numClasses
+ * by finding the maximum label value.
+ *
+ * Label validation (ensuring all labels are integers >= 0) needs to be
handled elsewhere,
+ * such as in [[extractLabeledPoints()]].
+ *
+ * @param dataset Dataset which contains a column [[labelCol]]
+ * @param maxNumClasses Maximum number of classes allowed when inferred
from data. If numClasses
+ * is specified in the metadata, then
maxNumClasses is ignored.
+ * @return number of classes
+ * @throws IllegalArgumentException if metadata does not specify
numClasses, and the
+ * actual numClasses exceeds
maxNumClasses
+ */
+ protected def getNumClasses(dataset: Dataset[_], maxNumClasses: Int =
1000): Int = {
+ MetadataUtils.getNumClasses(dataset.schema($(labelCol))) match {
+ case Some(n: Int) => n
+ case None =>
+ // Get number of classes from dataset itself.
+ val maxLabelRow: Array[Row] =
dataset.select(max($(labelCol))).take(1)
+ if (maxLabelRow.isEmpty) {
+ throw new SparkException("ML algorithm was given empty dataset.")
+ }
+ val maxLabel: Int = maxLabelRow.head.getDouble(0).toInt
+ val numClasses = maxLabel + 1
+ require(numClasses <= maxNumClasses, s"Classifier inferred
$numClasses from label values" +
+ s" in column $labelCol since numClasses were not specified in
dataset metadata, but" +
+ s" this exceeded the max numClasses ($maxNumClasses) allowed to
be inferred from" +
+ s" values. To avoid this error, specify numClasses in metadata,
such as by applying" +
+ s" StringIndexer to the label column.")
+ numClasses
+ }
+ }
+}
+
+private[ml] object Classifier {
+ /**
+ * Extract labelCol and featuresCol from the given dataset,
+ * and put it in an RDD with strong types.
+ * @throws SparkException if any label is not an integer >= 0 and <
numClasses
+ */
+ def extractLabeledPoints(
--- End diff --
Note this is ```private[ml]``` instead of protected within class Classifier
since GBTClassifier does not yet implement Classifier.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
