Repository: systemml Updated Branches: refs/heads/master c00029a7b -> 4d376637a
[MINOR] Fixes for the breast cancer project. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/4d376637 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/4d376637 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/4d376637 Branch: refs/heads/master Commit: 4d376637a99891e53ea93f650fb9341fc19b99f9 Parents: c00029a Author: Mike Dusenberry <mwdus...@us.ibm.com> Authored: Thu Sep 7 16:01:45 2017 -0700 Committer: Mike Dusenberry <mwdus...@us.ibm.com> Committed: Thu Sep 7 16:01:45 2017 -0700 ---------------------------------------------------------------------- projects/breast_cancer/MachineLearning-Keras-ResNet50.ipynb | 3 +-- projects/breast_cancer/breastcancer/preprocessing.py | 4 +++- projects/breast_cancer/preprocess.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/4d376637/projects/breast_cancer/MachineLearning-Keras-ResNet50.ipynb ---------------------------------------------------------------------- diff --git a/projects/breast_cancer/MachineLearning-Keras-ResNet50.ipynb b/projects/breast_cancer/MachineLearning-Keras-ResNet50.ipynb index bafa74a..b99a51d 100644 --- a/projects/breast_cancer/MachineLearning-Keras-ResNet50.ipynb +++ b/projects/breast_cancer/MachineLearning-Keras-ResNet50.ipynb @@ -28,12 +28,11 @@ "from keras.applications.resnet50 import ResNet50\n", "from keras.callbacks import ModelCheckpoint, TensorBoard\n", "from keras.initializers import VarianceScaling\n", - "from keras.layers import Dense, Dropout, Flatten, GlobalAveragePooling2D, Input, Lambda, merge\n", + "from keras.layers import Dense, Dropout, Flatten, GlobalAveragePooling2D, Input, Lambda\n", "from keras.models import Model, load_model\n", "from keras.optimizers import SGD\n", "from keras.preprocessing.image import ImageDataGenerator\n", "from keras.regularizers import l2\n", - "from keras.utils import to_categorical\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import pandas as pd\n", http://git-wip-us.apache.org/repos/asf/systemml/blob/4d376637/projects/breast_cancer/breastcancer/preprocessing.py ---------------------------------------------------------------------- diff --git a/projects/breast_cancer/breastcancer/preprocessing.py b/projects/breast_cancer/breastcancer/preprocessing.py index 763bde5..0e4e91f 100644 --- a/projects/breast_cancer/breastcancer/preprocessing.py +++ b/projects/breast_cancer/breastcancer/preprocessing.py @@ -71,6 +71,8 @@ def open_slide(slide_num, folder, training): slide = openslide.open_slide(filename) except OpenSlideError: slide = None + except FileNotFoundError: + slide = None return slide @@ -586,7 +588,7 @@ def preprocess(spark, slide_nums, folder="data", training=True, tile_size=1024, # Append labels labels_df = get_labels_df(folder) samples_with_labels = (samples.map( - lambda tup: (tup[0], int(labels_df.at[tup[0],"tumor_score"]), + lambda tup: (int(tup[0]), int(labels_df.at[tup[0],"tumor_score"]), float(labels_df.at[tup[0],"molecular_score"]), Vectors.dense(tup[1])))) df = samples_with_labels.toDF(["slide_num", "tumor_score", "molecular_score", "sample"]) df = df.select(df.slide_num.astype("int"), df.tumor_score.astype("int"), http://git-wip-us.apache.org/repos/asf/systemml/blob/4d376637/projects/breast_cancer/preprocess.py ---------------------------------------------------------------------- diff --git a/projects/breast_cancer/preprocess.py b/projects/breast_cancer/preprocess.py index e90fe8c..71055ad 100644 --- a/projects/breast_cancer/preprocess.py +++ b/projects/breast_cancer/preprocess.py @@ -33,7 +33,7 @@ import pandas as pd from sklearn.model_selection import train_test_split from pyspark.sql import SparkSession -from breastcancer.preprocessing import add_row_indices, get_labels_df, preprocess, save +from breastcancer.preprocessing import add_row_indices, get_labels_df, preprocess, save, sample # Create new SparkSession
