zaleslaw commented on a change in pull request #8466: URL: https://github.com/apache/ignite/pull/8466#discussion_r533216362
########## File path: modules/ml/src/test/java/org/apache/ignite/ml/preprocessing/encoding/TargetEncoderPreprocessorTest.java ########## @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.preprocessing.encoding; + +import java.io.Serializable; +import java.util.HashMap; +import java.util.HashSet; +import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer; +import org.apache.ignite.ml.dataset.feature.extractor.impl.DummyVectorizer; +import org.apache.ignite.ml.math.primitives.vector.Vector; +import org.apache.ignite.ml.math.primitives.vector.impl.DenseVector; +import org.apache.ignite.ml.preprocessing.encoding.target.TargetEncoderPreprocessor; +import org.apache.ignite.ml.preprocessing.encoding.target.TargetEncodingMeta; +import org.junit.Test; +import static org.junit.Assert.assertArrayEquals; + +/** + * Tests for {@link TargetEncoderPreprocessor}. + */ +public class TargetEncoderPreprocessorTest { + /** Tests {@code apply()} method. */ + @Test + public void testApply() { + Vector[] data = new Vector[] { + new DenseVector(new Serializable[] {"1", "Moscow", "A"}), + new DenseVector(new Serializable[] {"2", "Moscow", "B"}), + new DenseVector(new Serializable[] {"3", "Moscow", "B"}), + }; + + Vectorizer<Integer, Vector, Integer, Double> vectorizer = new DummyVectorizer<>(0, 1, 2); + + TargetEncoderPreprocessor<Integer, Vector> preprocessor = new TargetEncoderPreprocessor<Integer, Vector>( + new TargetEncodingMeta[]{ + new TargetEncodingMeta( + 0.5, + new HashMap() { + { + put("1", 1.0); + put("2", 0.0); + } + } + ), + new TargetEncodingMeta( + 0.1, + new HashMap() {} + ), + new TargetEncodingMeta( + 0.1, + new HashMap() { + { + put("A", 1.0); + put("B", 2.0); + } + } + ), + }, + vectorizer, + new HashSet() { + { + add(0); + add(1); + add(2); + } + }); + + double[][] postProcessedData = new double[][] { + {1.0, 0.1, 1.0}, Review comment: Could you explain please numbers in the last columns: why are they 1.0 and 2.0? not 0.33 and 0.66 ########## File path: examples/src/main/java/org/apache/ignite/examples/ml/preprocessing/encoding/TargetEncoderExample.java ########## @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.examples.ml.preprocessing.encoding; + +import java.io.FileNotFoundException; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.Ignition; +import org.apache.ignite.examples.ml.util.MLSandboxDatasets; +import org.apache.ignite.examples.ml.util.SandboxMLCache; +import org.apache.ignite.ml.composition.ModelsComposition; +import org.apache.ignite.ml.composition.boosting.GDBTrainer; +import org.apache.ignite.ml.composition.boosting.convergence.median.MedianOfMedianConvergenceCheckerFactory; +import org.apache.ignite.ml.dataset.feature.extractor.Vectorizer; +import org.apache.ignite.ml.dataset.feature.extractor.impl.ObjectArrayVectorizer; +import org.apache.ignite.ml.preprocessing.Preprocessor; +import org.apache.ignite.ml.preprocessing.encoding.EncoderTrainer; +import org.apache.ignite.ml.preprocessing.encoding.EncoderType; +import org.apache.ignite.ml.selection.scoring.evaluator.Evaluator; +import org.apache.ignite.ml.selection.scoring.metric.classification.Accuracy; +import org.apache.ignite.ml.tree.boosting.GDBBinaryClassifierOnTreesTrainer; + +/** + * Example that shows how to use Target Encoder preprocessor to encode labels presented as a mean target value. + * <p> + * Code in this example launches Ignite grid and fills the cache with test data (based on mushrooms dataset).</p> + * <p> + * After that it defines preprocessors that extract features from an upstream data and encode category with avarage + * target value (categories). </p> + * <p> + * Then, it trains the model based on the processed data using gradient boosing decision tree classification.</p> + * <p> + * Finally, this example uses {@link Evaluator} functionality to compute metrics from predictions.</p> + * + * <p>Daniele Miccii-Barreca (2001). A Preprocessing Scheme for High-Cardinality Categorical + * Attributes in Classification and Prediction Problems. SIGKDD Explor. Newsl. 3, 1. + * From http://dx.doi.org/10.1145/507533.507538</p> + */ +public class TargetEncoderExample { + /** + * Run example. + */ + public static void main(String[] args) { + System.out.println(); + System.out.println(">>> Train Gradient Boosing Decision Tree model on amazon-employee-access-challenge_train.csv dataset."); + + try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) { + try { + IgniteCache<Integer, Object[]> dataCache = new SandboxMLCache(ignite) + .fillObjectCacheWithCategoricalData(MLSandboxDatasets.AMAZON_EMPLOYEE_ACCESS); + + Set<Integer> featuresIndexies = new HashSet<>(Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9)); + Set<Integer> targetEncodedfeaturesIndexies = new HashSet<>(Arrays.asList(1, 5, 6)); + Integer targetIndex = 0; + + final Vectorizer<Integer, Object[], Integer, Object> vectorizer = new ObjectArrayVectorizer<Integer>(featuresIndexies.toArray(new Integer[0])) + .labeled(targetIndex); + + Preprocessor<Integer, Object[]> strEncoderPreprocessor = new EncoderTrainer<Integer, Object[]>() + .withEncoderType(EncoderType.STRING_ENCODER) + .withEncodedFeature(0) + .withEncodedFeatures(featuresIndexies) + .fit(ignite, + dataCache, + vectorizer + ); + + Preprocessor<Integer, Object[]> targetEncoderProcessor = new EncoderTrainer<Integer, Object[]>() + .withEncoderType(EncoderType.TARGET_ENCODER) + .labeled(0) + .withEncodedFeatures(targetEncodedfeaturesIndexies) + .minSamplesLeaf(1) + .minCategorySize(1L) + .smoothing(1d) + .fit(ignite, + dataCache, + strEncoderPreprocessor + ); + + Preprocessor<Integer, Object[]> lbEncoderPreprocessor = new EncoderTrainer<Integer, Object[]>() Review comment: Sorry, but I didn't understand this pipeline? Why those 3 encoders are combined here? Could they work only in this combination? In my opinion, user have a choice what to do with Strings, but he should choose one method (not the chain of methods). Please share your vision here ########## File path: modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/EncoderTrainer.java ########## @@ -343,6 +441,81 @@ else if (lbVal instanceof Double) return categoryFrequencies; } + /** + * Updates frequencies by values and features. + * + * @param row Feature vector. + * @param targetCounters Holds the frequencies of categories by values and features. + * @return target counter. + */ + private TargetCounter[] updateTargetCountersForNextRow(LabeledVector row, + TargetCounter[] targetCounters) { + if (targetCounters == null) + targetCounters = initializeTargetCounters(row); + else + assert targetCounters.length == row.size() : "Base preprocessor must return exactly " + + targetCounters.length + " features"; + + double targetValue = row.features().get(targetLabelIndex); + + for (int i = 0; i < targetCounters.length; i++) { + if (handledIndices.contains(i)) { + String strVal; + Object featureVal = row.features().getRaw(i); + + if (featureVal.equals(Double.NaN)) { + strVal = EncoderPreprocessor.KEY_FOR_NULL_VALUES; + row.features().setRaw(i, strVal); + } + else if (featureVal instanceof String) + strVal = (String)featureVal; + else if (featureVal instanceof Double) Review comment: Maybe add type conversion to Doulbe from another Number types (and boolean) ########## File path: modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/EncoderTrainer.java ########## @@ -116,6 +143,77 @@ } } + /** + * Calculates encoding frequencies as frequency divided on amount of rows in dataset. + * + * NOTE: The amount of rows is calculated as sum of absolute frequencies. + * + * @param dataset Dataset. + * @return Encoding frequency for each feature. + */ + private TargetEncodingMeta[] calculateTargetEncodingFrequencies(Dataset<EmptyContext, EncoderPartitionData> dataset) { + TargetCounter[] targetCounters = dataset.compute( + EncoderPartitionData::targetCounters, + (a, b) -> { + if (a == null) + return b; + + if (b == null) + return a; + + assert a.length == b.length; + + for (int i = 0; i < a.length; i++) { + if (handledIndices.contains(i)) { + int finalI = i; + b[i].setTargetSum(a[i].getTargetSum() + b[i].getTargetSum()); + b[i].setTargetCount(a[i].getTargetCount() + b[i].getTargetCount()); + a[i].getCategoryCounts() + .forEach((k, v) -> b[finalI].getCategoryCounts().merge(k, v, Long::sum)); + a[i].getCategoryTargetSum() + .forEach((k, v) -> b[finalI].getCategoryTargetSum().merge(k, v, Double::sum)); + } + } + return b; + } + ); + + TargetEncodingMeta[] targetEncodingMetas = new TargetEncodingMeta[targetCounters.length]; + for (int i = 0; i < targetCounters.length; i++) { + if (handledIndices.contains(i)) { + int finalI = i; + + targetEncodingMetas[i] = new TargetEncodingMeta( + targetCounters[i].getTargetSum() / targetCounters[i].getTargetCount(), Review comment: I suggest to refactor constructor parameters to separate variables for readability purposes ########## File path: modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/target/TargetEncoderPreprocessor.java ########## @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.preprocessing.encoding.target; + +import java.util.Set; +import org.apache.ignite.ml.math.primitives.vector.VectorUtils; +import org.apache.ignite.ml.preprocessing.Preprocessor; +import org.apache.ignite.ml.preprocessing.encoding.EncoderPreprocessor; +import org.apache.ignite.ml.structures.LabeledVector; + +/** + * Preprocessing function that makes Target encoding. + * + * The Target Encoder Preprocessor encodes string values (categories) to double values + * in range [0.0, 1], where the value will be presented as a regularized mean target value of + * a category. + * + * alpha = 1 / (1 + exp(-(categorySize - min_samples_leaf) / beta)) + * encodedValue = globalTargetMean * (1 - alpha) + categoryTargetMean * alpha + * if categorySize == 1 then use globalTargetMean + * + * min_samples_leaf - minimum samples to take category average into account. Review comment: looks like min_samples_leaf is not used in this class ########## File path: modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/EncoderTrainer.java ########## @@ -116,6 +143,77 @@ } } + /** + * Calculates encoding frequencies as frequency divided on amount of rows in dataset. + * + * NOTE: The amount of rows is calculated as sum of absolute frequencies. + * + * @param dataset Dataset. + * @return Encoding frequency for each feature. + */ + private TargetEncodingMeta[] calculateTargetEncodingFrequencies(Dataset<EmptyContext, EncoderPartitionData> dataset) { + TargetCounter[] targetCounters = dataset.compute( + EncoderPartitionData::targetCounters, + (a, b) -> { + if (a == null) + return b; + + if (b == null) + return a; + + assert a.length == b.length; + + for (int i = 0; i < a.length; i++) { + if (handledIndices.contains(i)) { + int finalI = i; + b[i].setTargetSum(a[i].getTargetSum() + b[i].getTargetSum()); + b[i].setTargetCount(a[i].getTargetCount() + b[i].getTargetCount()); + a[i].getCategoryCounts() + .forEach((k, v) -> b[finalI].getCategoryCounts().merge(k, v, Long::sum)); + a[i].getCategoryTargetSum() + .forEach((k, v) -> b[finalI].getCategoryTargetSum().merge(k, v, Double::sum)); + } + } + return b; + } + ); + + TargetEncodingMeta[] targetEncodingMetas = new TargetEncodingMeta[targetCounters.length]; + for (int i = 0; i < targetCounters.length; i++) { + if (handledIndices.contains(i)) { + int finalI = i; + + targetEncodingMetas[i] = new TargetEncodingMeta( + targetCounters[i].getTargetSum() / targetCounters[i].getTargetCount(), + targetCounters[i].getCategoryTargetSum().entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + value -> { + double prior = targetCounters[finalI].getTargetSum() / Review comment: Also this lambda should be encapsulated and commented separately ########## File path: modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/EncoderTrainer.java ########## @@ -116,6 +143,77 @@ } } + /** + * Calculates encoding frequencies as frequency divided on amount of rows in dataset. + * + * NOTE: The amount of rows is calculated as sum of absolute frequencies. + * + * @param dataset Dataset. + * @return Encoding frequency for each feature. + */ + private TargetEncodingMeta[] calculateTargetEncodingFrequencies(Dataset<EmptyContext, EncoderPartitionData> dataset) { + TargetCounter[] targetCounters = dataset.compute( + EncoderPartitionData::targetCounters, + (a, b) -> { + if (a == null) + return b; + + if (b == null) + return a; + + assert a.length == b.length; + + for (int i = 0; i < a.length; i++) { + if (handledIndices.contains(i)) { + int finalI = i; + b[i].setTargetSum(a[i].getTargetSum() + b[i].getTargetSum()); + b[i].setTargetCount(a[i].getTargetCount() + b[i].getTargetCount()); + a[i].getCategoryCounts() + .forEach((k, v) -> b[finalI].getCategoryCounts().merge(k, v, Long::sum)); + a[i].getCategoryTargetSum() + .forEach((k, v) -> b[finalI].getCategoryTargetSum().merge(k, v, Double::sum)); + } + } + return b; + } + ); + + TargetEncodingMeta[] targetEncodingMetas = new TargetEncodingMeta[targetCounters.length]; + for (int i = 0; i < targetCounters.length; i++) { + if (handledIndices.contains(i)) { + int finalI = i; + + targetEncodingMetas[i] = new TargetEncodingMeta( + targetCounters[i].getTargetSum() / targetCounters[i].getTargetCount(), + targetCounters[i].getCategoryTargetSum().entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + value -> { + double prior = targetCounters[finalI].getTargetSum() / + targetCounters[finalI].getTargetCount(); + double targetSum = targetCounters[finalI].getCategoryTargetSum() + .get(value.getKey()); + long categorySize = targetCounters[finalI].getCategoryCounts() + .get(value.getKey()); + + if (categorySize < minCategorySize) { + return prior; + } else { + double categoryMean = targetSum / categorySize; + + double smoove = 1 / (1 + + Math.exp(-(categorySize - minSamplesLeaf) / smoothing)); + return prior * (1 - smoove) + categoryMean * smoove; + } + } + )) + ); + } + } + Review comment: remove the blank line ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected]
