EBernhardson has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/364639 )
Change subject: Add option to evaluate test set after model training
......................................................................
Add option to evaluate test set after model training
Change-Id: If0a17907bbe56b6f567af5f02b340295b1d16b29
---
M mjolnir/cli/training_pipeline.py
M mjolnir/training/xgboost.py
2 files changed, 40 insertions(+), 11 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/search/MjoLniR
refs/changes/39/364639/1
diff --git a/mjolnir/cli/training_pipeline.py b/mjolnir/cli/training_pipeline.py
index 2304de2..7e91457 100644
--- a/mjolnir/cli/training_pipeline.py
+++ b/mjolnir/cli/training_pipeline.py
@@ -14,13 +14,23 @@
import mjolnir.training.xgboost
import os
import pickle
+import sys
from pyspark import SparkContext
from pyspark.sql import HiveContext
from pyspark.sql import functions as F
def main(sc, sqlContext, input_dir, output_dir, wikis, target_node_evaluations,
- num_workers, num_cv_jobs, num_folds):
+ num_workers, num_cv_jobs, num_folds, test_dir):
+
+ if os.path.exists(output_dir):
+ logging.error('Output directory (%s) already exists' % (output_dir))
+ sys.exit(1)
+
+ # Maybe this is a bit early to create the path ... but should be fine.
+ # The annoyance might be that an error in training requires deleting
+ # this directory to try again.
+ os.mkdir(output_dir)
for wiki in wikis:
print 'Training wiki: %s' % (wiki)
@@ -41,13 +51,6 @@
num_cv_jobs=num_cv_jobs, num_workers=num_workers,
target_node_evaluations=target_node_evaluations)
- # Save the tune results somewhere for later analysis. Use pickle
- # to maintain the hyperopt.Trials objects as is.
- tune_output = os.path.join(output_dir, 'tune_%s.pickle' % (wiki))
- with open(tune_output, 'w') as f:
- f.write(pickle.dumps(tune_results))
- print 'Wrote tuning results to %s' % (tune_output)
-
print 'CV test-ndcg@10: %.4f' % (tune_results['metrics']['test'])
print 'CV train-ndcg@10: %.4f' % (tune_results['metrics']['train'])
@@ -61,7 +64,27 @@
best_params['groupData'] = j_groups
model = mjolnir.training.xgboost.train(df_grouped, best_params)
- print 'train-ndcg@10: %.3f' % (model.eval(df_grouped, j_groups))
+ tune_results['metrics']['train'] = model.eval(df_grouped, j_groups)
+ print 'train-ndcg@10: %.5f' % (tune_results['metrics']['train'])
+
+ if test_dir is not None:
+ try:
+ df_test = sqlContext.read.parquet(test_dir)
+ tune_results['metrics']['test'] = model.eval(df_test)
+ print 'test-ndcg@10: %.5f' % (tune_results['metrics']['test'])
+ except:
+ # It has probably taken some time to get this far. Don't bail
+ # because the user input an invalid test dir.
+ logging.exception('Could not evaluate test_dir: %s' %
(test_dir))
+
+ # Save the tune results somewhere for later analysis. Use pickle
+ # to maintain the hyperopt.Trials objects as is. It might be nice
+ # to write out a json version, but the Trials objects require
+ # some more work before they can be json encoded.
+ tune_output_pickle = os.path.join(output_dir, 'tune_%s.pickle' %
(wiki))
+ with open(tune_output_pickle, 'w') as f:
+ f.write(pickle.dumps(tune_results))
+ print 'Wrote tuning results to %s' % (tune_output_pickle)
# Generate a feature map so xgboost can include feature names in the
dump.
# The final `q` indicates all features are quantitative values
(floats).
@@ -106,6 +129,12 @@
+ 'trees used in the final result. Default uses 100 trees rather '
+ 'than dynamically choosing based on max_depth. (Default: None)')
parser.add_argument(
+ '-t', '--test-path', dest='test_dir', type=str, required=False,
default=None,
+ help='A holdout test set to evaluate the final model against')
+ parser.add_argument(
+ '-z', '--zero-feature', dest='zero_features', type=str, nargs='+',
+ help='Zero out feature in input')
+ parser.add_argument(
'-v', '--verbose', dest='verbose', default=False, action='store_true',
help='Increase logging to INFO')
parser.add_argument(
diff --git a/mjolnir/training/xgboost.py b/mjolnir/training/xgboost.py
index 46b4c9b..5b699c8 100644
--- a/mjolnir/training/xgboost.py
+++ b/mjolnir/training/xgboost.py
@@ -561,7 +561,7 @@
},
'params': space,
'metrics': {
- 'test': -loss,
- 'train': -loss + true_loss
+ 'cv-test': -loss,
+ 'cv-train': -loss + true_loss
}
}
--
To view, visit https://gerrit.wikimedia.org/r/364639
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: If0a17907bbe56b6f567af5f02b340295b1d16b29
Gerrit-PatchSet: 1
Gerrit-Project: search/MjoLniR
Gerrit-Branch: master
Gerrit-Owner: EBernhardson <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
