This is an automated email from the ASF dual-hosted git repository.
fmcquillan pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/madlib-site.git
The following commit(s) were added to refs/heads/asf-site by this push:
new 388c4b3 updated 3 notebooks with minor changes
388c4b3 is described below
commit 388c4b34a08c96e059baca60f17350db327491ca
Author: Frank McQuillan <[email protected]>
AuthorDate: Tue Aug 27 13:28:24 2019 -0700
updated 3 notebooks with minor changes
---
.../Deep-learning/Load-images-v1.ipynb | 571 +++++++++++----------
.../Deep-learning/Load-model-architecture-v1.ipynb | 228 ++++++--
.../MADlib-Keras-cifar10-cnn-v2.ipynb | 4 +-
3 files changed, 486 insertions(+), 317 deletions(-)
diff --git a/community-artifacts/Deep-learning/Load-images-v1.ipynb
b/community-artifacts/Deep-learning/Load-images-v1.ipynb
index 3209aaf..1750cfc 100644
--- a/community-artifacts/Deep-learning/Load-images-v1.ipynb
+++ b/community-artifacts/Deep-learning/Load-images-v1.ipynb
@@ -134,35 +134,14 @@
"source": [
"import sys\n",
"import os\n",
- "from keras.datasets import cifar10"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 5,
- "metadata": {},
- "outputs": [],
- "source": [
+ "from keras.datasets import cifar10\n",
+ "\n",
"madlib_site_dir =
'/Users/fmcquillan/Documents/Product/MADlib/Demos/data'\n",
- "sys.path.append(madlib_site_dir)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 6,
- "metadata": {},
- "outputs": [],
- "source": [
+ "sys.path.append(madlib_site_dir)\n",
+ "\n",
"# Import image loader module\n",
- "from madlib_image_loader import ImageLoader, DbCredentials"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 7,
- "metadata": {},
- "outputs": [],
- "source": [
+ "from madlib_image_loader import ImageLoader, DbCredentials\n",
+ "\n",
"# Specify database credentials, for connecting to db\n",
"#db_creds = DbCredentials(user='gpadmin',\n",
"# host='35.239.240.26',\n",
@@ -173,15 +152,8 @@
"db_creds = DbCredentials(user='fmcquillan',\n",
" host='localhost',\n",
" port='5432',\n",
- " password='')"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 8,
- "metadata": {},
- "outputs": [],
- "source": [
+ " password='')\n",
+ "\n",
"# Initialize ImageLoader (increase num_workers to run faster)\n",
"iloader = ImageLoader(num_workers=5, db_creds=db_creds)"
]
@@ -201,15 +173,12 @@
"- <em>data_y</em> is a 1D np.array of the image categories (labels).\n",
"\n",
"\n",
- "- If the user passes a <em>table_name</em> while creating ImageLoader
object, it will be used for all further calls to load_dataset_from_np. It can
be changed by passing it as a parameter during the actual call to
load_dataset_from_np, and if so future calls will load to that table name
instead. This avoids needing to pass the table_name again every time, but also
allows it to be changed at any time.\n",
- "\n",
- " \n",
- "- <em>append=False</em> attempts to create a new table, while
<em>append=True</em> appends more images to an existing table."
+ "- If the user passes a <em>table_name</em> while creating ImageLoader
object, it will be used for all further calls to load_dataset_from_np. It can
be changed by passing it as a parameter during the actual call to
load_dataset_from_np, and if so future calls will load to that table name
instead. This avoids needing to pass the table_name again every time, but also
allows it to be changed at any time."
]
},
{
"cell_type": "code",
- "execution_count": 12,
+ "execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
@@ -219,7 +188,7 @@
},
{
"cell_type": "code",
- "execution_count": 13,
+ "execution_count": 6,
"metadata": {},
"outputs": [
{
@@ -232,180 +201,180 @@
"CREATE TABLE\n",
"Created table cifar_10_train_data in madlib db\n",
"Spawning 5 workers...\n",
- "Initializing PoolWorker-1 [pid 32578]\n",
- "PoolWorker-1: Created temporary directory /tmp/madlib_7llkr1iwrB\n",
- "Initializing PoolWorker-2 [pid 32579]\n",
- "PoolWorker-2: Created temporary directory /tmp/madlib_gFBeKWnLLL\n",
- "Initializing PoolWorker-3 [pid 32580]\n",
- "PoolWorker-3: Created temporary directory /tmp/madlib_3EqOi1Elxy\n",
+ "Initializing PoolWorker-1 [pid 82412]\n",
+ "PoolWorker-1: Created temporary directory /tmp/madlib_Bt85aChbv0\n",
+ "Initializing PoolWorker-2 [pid 82413]\n",
+ "PoolWorker-2: Created temporary directory /tmp/madlib_cSyCSiEhHT\n",
+ "Initializing PoolWorker-3 [pid 82414]\n",
+ "PoolWorker-3: Created temporary directory /tmp/madlib_uvtHjGCU5S\n",
"PoolWorker-1: Connected to madlib db.\n",
- "Initializing PoolWorker-4 [pid 32581]\n",
- "PoolWorker-4: Created temporary directory /tmp/madlib_t3HzIGOeAn\n",
+ "Initializing PoolWorker-4 [pid 82415]\n",
+ "PoolWorker-4: Created temporary directory /tmp/madlib_eJmkoDZTr8\n",
"PoolWorker-2: Connected to madlib db.\n",
- "Initializing PoolWorker-5 [pid 32583]\n",
+ "Initializing PoolWorker-5 [pid 82417]\n",
+ "PoolWorker-5: Created temporary directory /tmp/madlib_websbk05x2\n",
"PoolWorker-3: Connected to madlib db.\n",
- "PoolWorker-5: Created temporary directory /tmp/madlib_I9wQ0HGhrx\n",
"PoolWorker-4: Connected to madlib db.\n",
"PoolWorker-5: Connected to madlib db.\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_7llkr1iwrB/cifar_10_train_data0000.tmp\n",
- "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_gFBeKWnLLL/cifar_10_train_data0000.tmp\n",
- "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_3EqOi1Elxy/cifar_10_train_data0000.tmp\n",
- "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_t3HzIGOeAn/cifar_10_train_data0000.tmp\n",
- "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_I9wQ0HGhrx/cifar_10_train_data0000.tmp\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_Bt85aChbv0/cifar_10_train_data0000.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_cSyCSiEhHT/cifar_10_train_data0000.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_uvtHjGCU5S/cifar_10_train_data0000.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_eJmkoDZTr8/cifar_10_train_data0000.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_websbk05x2/cifar_10_train_data0000.tmp\n",
"PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_7llkr1iwrB/cifar_10_train_data0001.tmp\n",
- "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_gFBeKWnLLL/cifar_10_train_data0001.tmp\n",
- "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_3EqOi1Elxy/cifar_10_train_data0001.tmp\n",
- "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_t3HzIGOeAn/cifar_10_train_data0001.tmp\n",
- "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_I9wQ0HGhrx/cifar_10_train_data0001.tmp\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_Bt85aChbv0/cifar_10_train_data0001.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_cSyCSiEhHT/cifar_10_train_data0001.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_uvtHjGCU5S/cifar_10_train_data0001.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_eJmkoDZTr8/cifar_10_train_data0001.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_websbk05x2/cifar_10_train_data0001.tmp\n",
"PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_7llkr1iwrB/cifar_10_train_data0002.tmp\n",
- "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_gFBeKWnLLL/cifar_10_train_data0002.tmp\n",
- "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_3EqOi1Elxy/cifar_10_train_data0002.tmp\n",
- "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_t3HzIGOeAn/cifar_10_train_data0002.tmp\n",
- "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_I9wQ0HGhrx/cifar_10_train_data0002.tmp\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_Bt85aChbv0/cifar_10_train_data0002.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_cSyCSiEhHT/cifar_10_train_data0002.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_uvtHjGCU5S/cifar_10_train_data0002.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_eJmkoDZTr8/cifar_10_train_data0002.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_websbk05x2/cifar_10_train_data0002.tmp\n",
"PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_7llkr1iwrB/cifar_10_train_data0003.tmp\n",
- "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_gFBeKWnLLL/cifar_10_train_data0003.tmp\n",
- "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_3EqOi1Elxy/cifar_10_train_data0003.tmp\n",
- "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_t3HzIGOeAn/cifar_10_train_data0003.tmp\n",
- "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_I9wQ0HGhrx/cifar_10_train_data0003.tmp\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_Bt85aChbv0/cifar_10_train_data0003.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_cSyCSiEhHT/cifar_10_train_data0003.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_uvtHjGCU5S/cifar_10_train_data0003.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_eJmkoDZTr8/cifar_10_train_data0003.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_websbk05x2/cifar_10_train_data0003.tmp\n",
"PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_7llkr1iwrB/cifar_10_train_data0004.tmp\n",
- "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_gFBeKWnLLL/cifar_10_train_data0004.tmp\n",
- "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_3EqOi1Elxy/cifar_10_train_data0004.tmp\n",
- "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_t3HzIGOeAn/cifar_10_train_data0004.tmp\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_Bt85aChbv0/cifar_10_train_data0004.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_cSyCSiEhHT/cifar_10_train_data0004.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_uvtHjGCU5S/cifar_10_train_data0004.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_eJmkoDZTr8/cifar_10_train_data0004.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_websbk05x2/cifar_10_train_data0004.tmp\n",
"PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_I9wQ0HGhrx/cifar_10_train_data0004.tmp\n",
"PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_7llkr1iwrB/cifar_10_train_data0005.tmp\n",
- "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_gFBeKWnLLL/cifar_10_train_data0005.tmp\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_Bt85aChbv0/cifar_10_train_data0005.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_cSyCSiEhHT/cifar_10_train_data0005.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_uvtHjGCU5S/cifar_10_train_data0005.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_eJmkoDZTr8/cifar_10_train_data0005.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_websbk05x2/cifar_10_train_data0005.tmp\n",
"PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_t3HzIGOeAn/cifar_10_train_data0005.tmp\n",
- "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_3EqOi1Elxy/cifar_10_train_data0005.tmp\n",
- "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_I9wQ0HGhrx/cifar_10_train_data0005.tmp\n",
"PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_7llkr1iwrB/cifar_10_train_data0006.tmp\n",
- "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_gFBeKWnLLL/cifar_10_train_data0006.tmp\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_Bt85aChbv0/cifar_10_train_data0006.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_cSyCSiEhHT/cifar_10_train_data0006.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_uvtHjGCU5S/cifar_10_train_data0006.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_eJmkoDZTr8/cifar_10_train_data0006.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_websbk05x2/cifar_10_train_data0006.tmp\n",
"PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_t3HzIGOeAn/cifar_10_train_data0006.tmp\n",
- "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_3EqOi1Elxy/cifar_10_train_data0006.tmp\n",
- "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_I9wQ0HGhrx/cifar_10_train_data0006.tmp\n",
"PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_7llkr1iwrB/cifar_10_train_data0007.tmp\n",
- "PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_Bt85aChbv0/cifar_10_train_data0007.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_cSyCSiEhHT/cifar_10_train_data0007.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_uvtHjGCU5S/cifar_10_train_data0007.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_eJmkoDZTr8/cifar_10_train_data0007.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_websbk05x2/cifar_10_train_data0007.tmp\n",
"PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_gFBeKWnLLL/cifar_10_train_data0007.tmp\n",
- "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_3EqOi1Elxy/cifar_10_train_data0007.tmp\n",
- "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_t3HzIGOeAn/cifar_10_train_data0007.tmp\n",
- "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_I9wQ0HGhrx/cifar_10_train_data0007.tmp\n",
"PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_7llkr1iwrB/cifar_10_train_data0008.tmp\n",
"PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_Bt85aChbv0/cifar_10_train_data0008.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_cSyCSiEhHT/cifar_10_train_data0008.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_uvtHjGCU5S/cifar_10_train_data0008.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_eJmkoDZTr8/cifar_10_train_data0008.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_websbk05x2/cifar_10_train_data0008.tmp\n",
"PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_gFBeKWnLLL/cifar_10_train_data0008.tmp\n",
- "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_3EqOi1Elxy/cifar_10_train_data0008.tmp\n",
- "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_t3HzIGOeAn/cifar_10_train_data0008.tmp\n",
- "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_I9wQ0HGhrx/cifar_10_train_data0008.tmp\n",
"PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_7llkr1iwrB/cifar_10_train_data0009.tmp\n",
"PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_Bt85aChbv0/cifar_10_train_data0009.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_cSyCSiEhHT/cifar_10_train_data0009.tmp\n",
"PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_gFBeKWnLLL/cifar_10_train_data0009.tmp\n",
"PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_7llkr1iwrB/cifar_10_train_data0010.tmp\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_Bt85aChbv0/cifar_10_train_data0010.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_cSyCSiEhHT/cifar_10_train_data0010.tmp\n",
"PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_gFBeKWnLLL/cifar_10_train_data0010.tmp\n",
"PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_7llkr1iwrB/cifar_10_train_data0011.tmp\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_Bt85aChbv0/cifar_10_train_data0011.tmp\n",
"PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-5: Removed temporary directory /tmp/madlib_I9wQ0HGhrx\n",
- "PoolWorker-2: Removed temporary directory /tmp/madlib_gFBeKWnLLL\n"
+ "PoolWorker-2: Removed temporary directory /tmp/madlib_cSyCSiEhHT\n",
+ "PoolWorker-3: Removed temporary directory /tmp/madlib_uvtHjGCU5S\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
- "PoolWorker-3: Removed temporary directory /tmp/madlib_3EqOi1Elxy\n",
- "PoolWorker-1: Removed temporary directory /tmp/madlib_7llkr1iwrB\n",
- "PoolWorker-4: Removed temporary directory /tmp/madlib_t3HzIGOeAn\n",
- "Done! Loaded 50000 images in 24.7632198334s\n",
+ "PoolWorker-5: Removed temporary directory /tmp/madlib_websbk05x2\n",
+ "PoolWorker-4: Removed temporary directory /tmp/madlib_eJmkoDZTr8\n",
+ "PoolWorker-1: Removed temporary directory /tmp/madlib_Bt85aChbv0\n",
+ "Done! Loaded 50000 images in 24.2222080231s\n",
"5 workers terminated.\n",
"MainProcess: Connected to madlib db.\n",
"Executing: CREATE TABLE cifar_10_test_data (id SERIAL, x REAL[], y
TEXT)\n",
"CREATE TABLE\n",
"Created table cifar_10_test_data in madlib db\n",
"Spawning 5 workers...\n",
- "Initializing PoolWorker-6 [pid 32590]\n",
- "PoolWorker-6: Created temporary directory /tmp/madlib_ukKJGqIRI7\n",
- "Initializing PoolWorker-7 [pid 32591]\n",
- "PoolWorker-7: Created temporary directory /tmp/madlib_FiXPZxXK47\n",
- "Initializing PoolWorker-8 [pid 32592]\n",
- "Initializing PoolWorker-9 [pid 32593]\n",
- "PoolWorker-8: Created temporary directory /tmp/madlib_EVmSLqZ6Od\n",
+ "Initializing PoolWorker-6 [pid 82423]\n",
+ "PoolWorker-6: Created temporary directory /tmp/madlib_e615zVgkaE\n",
+ "Initializing PoolWorker-7 [pid 82424]\n",
+ "PoolWorker-7: Created temporary directory /tmp/madlib_iRi2oMNIFA\n",
+ "Initializing PoolWorker-8 [pid 82425]\n",
+ "PoolWorker-8: Created temporary directory /tmp/madlib_kkSktVCq3n\n",
"PoolWorker-6: Connected to madlib db.\n",
- "PoolWorker-9: Created temporary directory /tmp/madlib_b4uua9iT9p\n",
+ "Initializing PoolWorker-9 [pid 82426]\n",
"PoolWorker-7: Connected to madlib db.\n",
- "Initializing PoolWorker-10 [pid 32595]\n",
+ "PoolWorker-9: Created temporary directory /tmp/madlib_0To3XX96yI\n",
+ "Initializing PoolWorker-10 [pid 82428]\n",
"PoolWorker-8: Connected to madlib db.\n",
- "PoolWorker-10: Created temporary directory /tmp/madlib_um0N6zuuKM\n",
+ "PoolWorker-10: Created temporary directory /tmp/madlib_8zwK04IJsc\n",
"PoolWorker-9: Connected to madlib db.\n",
"PoolWorker-10: Connected to madlib db.\n",
- "PoolWorker-6: Wrote 1000 images to
/tmp/madlib_ukKJGqIRI7/cifar_10_test_data0000.tmp\n",
- "PoolWorker-7: Wrote 1000 images to
/tmp/madlib_FiXPZxXK47/cifar_10_test_data0000.tmp\n",
- "PoolWorker-10: Wrote 1000 images to
/tmp/madlib_um0N6zuuKM/cifar_10_test_data0000.tmp\n",
- "PoolWorker-8: Wrote 1000 images to
/tmp/madlib_EVmSLqZ6Od/cifar_10_test_data0000.tmp\n",
- "PoolWorker-9: Wrote 1000 images to
/tmp/madlib_b4uua9iT9p/cifar_10_test_data0000.tmp\n",
+ "PoolWorker-6: Wrote 1000 images to
/tmp/madlib_e615zVgkaE/cifar_10_test_data0000.tmp\n",
+ "PoolWorker-7: Wrote 1000 images to
/tmp/madlib_iRi2oMNIFA/cifar_10_test_data0000.tmp\n",
+ "PoolWorker-8: Wrote 1000 images to
/tmp/madlib_kkSktVCq3n/cifar_10_test_data0000.tmp\n",
+ "PoolWorker-9: Wrote 1000 images to
/tmp/madlib_0To3XX96yI/cifar_10_test_data0000.tmp\n",
+ "PoolWorker-10: Wrote 1000 images to
/tmp/madlib_8zwK04IJsc/cifar_10_test_data0000.tmp\n",
"PoolWorker-6: Loaded 1000 images into cifar_10_test_data\n",
"PoolWorker-7: Loaded 1000 images into cifar_10_test_data\n",
- "PoolWorker-10: Loaded 1000 images into cifar_10_test_data\n",
"PoolWorker-8: Loaded 1000 images into cifar_10_test_data\n",
+ "PoolWorker-10: Loaded 1000 images into cifar_10_test_data\n",
"PoolWorker-9: Loaded 1000 images into cifar_10_test_data\n",
- "PoolWorker-6: Wrote 1000 images to
/tmp/madlib_ukKJGqIRI7/cifar_10_test_data0001.tmp\n",
- "PoolWorker-7: Wrote 1000 images to
/tmp/madlib_FiXPZxXK47/cifar_10_test_data0001.tmp\n",
- "PoolWorker-10: Wrote 1000 images to
/tmp/madlib_um0N6zuuKM/cifar_10_test_data0001.tmp\n",
- "PoolWorker-9: Wrote 1000 images to
/tmp/madlib_b4uua9iT9p/cifar_10_test_data0001.tmp\n",
- "PoolWorker-8: Wrote 1000 images to
/tmp/madlib_EVmSLqZ6Od/cifar_10_test_data0001.tmp\n",
+ "PoolWorker-6: Wrote 1000 images to
/tmp/madlib_e615zVgkaE/cifar_10_test_data0001.tmp\n",
+ "PoolWorker-7: Wrote 1000 images to
/tmp/madlib_iRi2oMNIFA/cifar_10_test_data0001.tmp\n",
+ "PoolWorker-8: Wrote 1000 images to
/tmp/madlib_kkSktVCq3n/cifar_10_test_data0001.tmp\n",
+ "PoolWorker-9: Wrote 1000 images to
/tmp/madlib_0To3XX96yI/cifar_10_test_data0001.tmp\n",
+ "PoolWorker-10: Wrote 1000 images to
/tmp/madlib_8zwK04IJsc/cifar_10_test_data0001.tmp\n",
"PoolWorker-6: Loaded 1000 images into cifar_10_test_data\n",
"PoolWorker-7: Loaded 1000 images into cifar_10_test_data\n",
- "PoolWorker-10: Loaded 1000 images into cifar_10_test_data\n",
- "PoolWorker-9: Loaded 1000 images into cifar_10_test_data\n",
"PoolWorker-8: Loaded 1000 images into cifar_10_test_data\n",
- "PoolWorker-10: Removed temporary directory /tmp/madlib_um0N6zuuKM\n",
- "PoolWorker-7: Removed temporary directory /tmp/madlib_FiXPZxXK47\n",
- "PoolWorker-8: Removed temporary directory /tmp/madlib_EVmSLqZ6Od\n",
- "PoolWorker-6: Removed temporary directory /tmp/madlib_ukKJGqIRI7\n",
- "PoolWorker-9: Removed temporary directory /tmp/madlib_b4uua9iT9p\n",
- "Done! Loaded 10000 images in 5.86952185631s\n",
+ "PoolWorker-9: Loaded 1000 images into cifar_10_test_data\n",
+ "PoolWorker-10: Loaded 1000 images into cifar_10_test_data\n",
+ "PoolWorker-10: Removed temporary directory /tmp/madlib_8zwK04IJsc\n",
+ "PoolWorker-8: Removed temporary directory /tmp/madlib_kkSktVCq3n\n",
+ "PoolWorker-7: Removed temporary directory /tmp/madlib_iRi2oMNIFA\n",
+ "PoolWorker-6: Removed temporary directory /tmp/madlib_e615zVgkaE\n",
+ "PoolWorker-9: Removed temporary directory /tmp/madlib_0To3XX96yI\n",
+ "Done! Loaded 10000 images in 4.6932258606s\n",
"5 workers terminated.\n"
]
}
@@ -419,6 +388,82 @@
]
},
{
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>count</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>50000</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(50000L,)]"
+ ]
+ },
+ "execution_count": 12,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT COUNT(*) FROM cifar_10_train_data;"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>count</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>10000</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(10000L,)]"
+ ]
+ },
+ "execution_count": 13,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT COUNT(*) FROM cifar_10_test_data;"
+ ]
+ },
+ {
"cell_type": "markdown",
"metadata": {},
"source": [
@@ -440,7 +485,7 @@
},
{
"cell_type": "code",
- "execution_count": 9,
+ "execution_count": 14,
"metadata": {},
"outputs": [
{
@@ -456,133 +501,133 @@
"number of labels = 10\n",
"Found 10 image labels in /Users/fmcquillan/tmp/cifar10\n",
"Spawning 5 workers...\n",
- "Initializing PoolWorker-1 [pid 64905]\n",
- "PoolWorker-1: Created temporary directory /tmp/madlib_lFwfdAIfkq\n",
- "Initializing PoolWorker-2 [pid 64906]\n",
- "PoolWorker-2: Created temporary directory /tmp/madlib_PAZRj5SNyn\n",
- "Initializing PoolWorker-3 [pid 64907]\n",
- "PoolWorker-3: Created temporary directory /tmp/madlib_x0attqv2T6\n",
- "PoolWorker-1: Connected to madlib db.\n",
- "Initializing PoolWorker-4 [pid 64908]\n",
- "PoolWorker-4: Created temporary directory /tmp/madlib_lyhVMiyvdL\n",
- "Initializing PoolWorker-5 [pid 64909]\n",
- "PoolWorker-5: Created temporary directory /tmp/madlib_8IpNResiVI\n",
- "PoolWorker-3: Connected to madlib db.\n",
- "PoolWorker-2: Connected to madlib db.\n",
- "PoolWorker-4: Connected to madlib db.\n",
- "PoolWorker-5: Connected to madlib db.\n",
- "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_x0attqv2T6/cifar_10_train_data_filesystem0000.tmp\n",
- "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_8IpNResiVI/cifar_10_train_data_filesystem0000.tmp\n",
- "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_lyhVMiyvdL/cifar_10_train_data_filesystem0000.tmp\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_lFwfdAIfkq/cifar_10_train_data_filesystem0000.tmp\n",
- "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_PAZRj5SNyn/cifar_10_train_data_filesystem0000.tmp\n",
- "PoolWorker-3: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-1: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-4: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-5: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-2: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_x0attqv2T6/cifar_10_train_data_filesystem0001.tmp\n",
- "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_lyhVMiyvdL/cifar_10_train_data_filesystem0001.tmp\n",
- "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_8IpNResiVI/cifar_10_train_data_filesystem0001.tmp\n",
- "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_PAZRj5SNyn/cifar_10_train_data_filesystem0001.tmp\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_lFwfdAIfkq/cifar_10_train_data_filesystem0001.tmp\n",
- "PoolWorker-3: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-4: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-2: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-5: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-1: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_x0attqv2T6/cifar_10_train_data_filesystem0002.tmp\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_lFwfdAIfkq/cifar_10_train_data_filesystem0002.tmp\n",
- "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_PAZRj5SNyn/cifar_10_train_data_filesystem0002.tmp\n",
- "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_lyhVMiyvdL/cifar_10_train_data_filesystem0002.tmp\n",
- "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_8IpNResiVI/cifar_10_train_data_filesystem0002.tmp\n",
- "PoolWorker-1: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-3: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-2: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-4: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-5: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_x0attqv2T6/cifar_10_train_data_filesystem0003.tmp\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_lFwfdAIfkq/cifar_10_train_data_filesystem0003.tmp\n",
- "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_PAZRj5SNyn/cifar_10_train_data_filesystem0003.tmp\n",
- "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_lyhVMiyvdL/cifar_10_train_data_filesystem0003.tmp\n",
- "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_8IpNResiVI/cifar_10_train_data_filesystem0003.tmp\n",
- "PoolWorker-3: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-1: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-2: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-5: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-4: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_lFwfdAIfkq/cifar_10_train_data_filesystem0004.tmp\n",
- "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_x0attqv2T6/cifar_10_train_data_filesystem0004.tmp\n",
- "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_PAZRj5SNyn/cifar_10_train_data_filesystem0004.tmp\n",
- "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_8IpNResiVI/cifar_10_train_data_filesystem0004.tmp\n",
- "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_lyhVMiyvdL/cifar_10_train_data_filesystem0004.tmp\n",
- "PoolWorker-1: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-3: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-2: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-5: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-4: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_lFwfdAIfkq/cifar_10_train_data_filesystem0005.tmp\n",
- "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_x0attqv2T6/cifar_10_train_data_filesystem0005.tmp\n",
- "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_8IpNResiVI/cifar_10_train_data_filesystem0005.tmp\n",
- "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_PAZRj5SNyn/cifar_10_train_data_filesystem0005.tmp\n",
- "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_lyhVMiyvdL/cifar_10_train_data_filesystem0005.tmp\n",
- "PoolWorker-1: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-3: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-5: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-4: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-2: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_x0attqv2T6/cifar_10_train_data_filesystem0006.tmp\n",
- "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_8IpNResiVI/cifar_10_train_data_filesystem0006.tmp\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_lFwfdAIfkq/cifar_10_train_data_filesystem0006.tmp\n",
- "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_PAZRj5SNyn/cifar_10_train_data_filesystem0006.tmp\n",
- "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_lyhVMiyvdL/cifar_10_train_data_filesystem0006.tmp\n",
- "PoolWorker-5: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-3: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-1: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-2: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-4: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_lFwfdAIfkq/cifar_10_train_data_filesystem0007.tmp\n",
- "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_8IpNResiVI/cifar_10_train_data_filesystem0007.tmp\n",
- "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_x0attqv2T6/cifar_10_train_data_filesystem0007.tmp\n",
- "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_lyhVMiyvdL/cifar_10_train_data_filesystem0007.tmp\n",
- "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_PAZRj5SNyn/cifar_10_train_data_filesystem0007.tmp\n",
- "PoolWorker-5: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-1: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-3: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-4: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-2: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_8IpNResiVI/cifar_10_train_data_filesystem0008.tmp\n",
- "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_x0attqv2T6/cifar_10_train_data_filesystem0008.tmp\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_lFwfdAIfkq/cifar_10_train_data_filesystem0008.tmp\n",
- "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_lyhVMiyvdL/cifar_10_train_data_filesystem0008.tmp\n",
- "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_PAZRj5SNyn/cifar_10_train_data_filesystem0008.tmp\n"
+ "Initializing PoolWorker-11 [pid 82438]\n",
+ "PoolWorker-11: Created temporary directory /tmp/madlib_aEC1lF2HqL\n",
+ "Initializing PoolWorker-12 [pid 82439]\n",
+ "PoolWorker-12: Created temporary directory /tmp/madlib_70qpwFzzqW\n",
+ "Initializing PoolWorker-13 [pid 82440]\n",
+ "PoolWorker-13: Created temporary directory /tmp/madlib_r2u4Zo5bPt\n",
+ "PoolWorker-11: Connected to madlib db.\n",
+ "Initializing PoolWorker-14 [pid 82441]\n",
+ "PoolWorker-12: Connected to madlib db.\n",
+ "PoolWorker-14: Created temporary directory /tmp/madlib_aTPESoNjVi\n",
+ "Initializing PoolWorker-15 [pid 82443]\n",
+ "PoolWorker-13: Connected to madlib db.\n",
+ "PoolWorker-15: Created temporary directory /tmp/madlib_rhVwjLTbWI\n",
+ "PoolWorker-14: Connected to madlib db.\n",
+ "PoolWorker-15: Connected to madlib db.\n",
+ "PoolWorker-13: Wrote 1000 images to
/tmp/madlib_r2u4Zo5bPt/cifar_10_train_data_filesystem0000.tmp\n",
+ "PoolWorker-14: Wrote 1000 images to
/tmp/madlib_aTPESoNjVi/cifar_10_train_data_filesystem0000.tmp\n",
+ "PoolWorker-12: Wrote 1000 images to
/tmp/madlib_70qpwFzzqW/cifar_10_train_data_filesystem0000.tmp\n",
+ "PoolWorker-11: Wrote 1000 images to
/tmp/madlib_aEC1lF2HqL/cifar_10_train_data_filesystem0000.tmp\n",
+ "PoolWorker-15: Wrote 1000 images to
/tmp/madlib_rhVwjLTbWI/cifar_10_train_data_filesystem0000.tmp\n",
+ "PoolWorker-13: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-14: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-11: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-12: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-15: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-14: Wrote 1000 images to
/tmp/madlib_aTPESoNjVi/cifar_10_train_data_filesystem0001.tmp\n",
+ "PoolWorker-11: Wrote 1000 images to
/tmp/madlib_aEC1lF2HqL/cifar_10_train_data_filesystem0001.tmp\n",
+ "PoolWorker-13: Wrote 1000 images to
/tmp/madlib_r2u4Zo5bPt/cifar_10_train_data_filesystem0001.tmp\n",
+ "PoolWorker-15: Wrote 1000 images to
/tmp/madlib_rhVwjLTbWI/cifar_10_train_data_filesystem0001.tmp\n",
+ "PoolWorker-12: Wrote 1000 images to
/tmp/madlib_70qpwFzzqW/cifar_10_train_data_filesystem0001.tmp\n",
+ "PoolWorker-14: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-11: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-13: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-15: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-12: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-14: Wrote 1000 images to
/tmp/madlib_aTPESoNjVi/cifar_10_train_data_filesystem0002.tmp\n",
+ "PoolWorker-15: Wrote 1000 images to
/tmp/madlib_rhVwjLTbWI/cifar_10_train_data_filesystem0002.tmp\n",
+ "PoolWorker-13: Wrote 1000 images to
/tmp/madlib_r2u4Zo5bPt/cifar_10_train_data_filesystem0002.tmp\n",
+ "PoolWorker-11: Wrote 1000 images to
/tmp/madlib_aEC1lF2HqL/cifar_10_train_data_filesystem0002.tmp\n",
+ "PoolWorker-12: Wrote 1000 images to
/tmp/madlib_70qpwFzzqW/cifar_10_train_data_filesystem0002.tmp\n",
+ "PoolWorker-14: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-11: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-15: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-13: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-12: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-15: Wrote 1000 images to
/tmp/madlib_rhVwjLTbWI/cifar_10_train_data_filesystem0003.tmp\n",
+ "PoolWorker-14: Wrote 1000 images to
/tmp/madlib_aTPESoNjVi/cifar_10_train_data_filesystem0003.tmp\n",
+ "PoolWorker-13: Wrote 1000 images to
/tmp/madlib_r2u4Zo5bPt/cifar_10_train_data_filesystem0003.tmp\n",
+ "PoolWorker-12: Wrote 1000 images to
/tmp/madlib_70qpwFzzqW/cifar_10_train_data_filesystem0003.tmp\n",
+ "PoolWorker-11: Wrote 1000 images to
/tmp/madlib_aEC1lF2HqL/cifar_10_train_data_filesystem0003.tmp\n",
+ "PoolWorker-15: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-14: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-13: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-11: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-12: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-15: Wrote 1000 images to
/tmp/madlib_rhVwjLTbWI/cifar_10_train_data_filesystem0004.tmp\n",
+ "PoolWorker-14: Wrote 1000 images to
/tmp/madlib_aTPESoNjVi/cifar_10_train_data_filesystem0004.tmp\n",
+ "PoolWorker-12: Wrote 1000 images to
/tmp/madlib_70qpwFzzqW/cifar_10_train_data_filesystem0004.tmp\n",
+ "PoolWorker-13: Wrote 1000 images to
/tmp/madlib_r2u4Zo5bPt/cifar_10_train_data_filesystem0004.tmp\n",
+ "PoolWorker-11: Wrote 1000 images to
/tmp/madlib_aEC1lF2HqL/cifar_10_train_data_filesystem0004.tmp\n",
+ "PoolWorker-14: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-15: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-12: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-11: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-13: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-15: Wrote 1000 images to
/tmp/madlib_rhVwjLTbWI/cifar_10_train_data_filesystem0005.tmp\n",
+ "PoolWorker-14: Wrote 1000 images to
/tmp/madlib_aTPESoNjVi/cifar_10_train_data_filesystem0005.tmp\n",
+ "PoolWorker-12: Wrote 1000 images to
/tmp/madlib_70qpwFzzqW/cifar_10_train_data_filesystem0005.tmp\n",
+ "PoolWorker-11: Wrote 1000 images to
/tmp/madlib_aEC1lF2HqL/cifar_10_train_data_filesystem0005.tmp\n",
+ "PoolWorker-13: Wrote 1000 images to
/tmp/madlib_r2u4Zo5bPt/cifar_10_train_data_filesystem0005.tmp\n",
+ "PoolWorker-14: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-15: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-12: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-13: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-11: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-15: Wrote 1000 images to
/tmp/madlib_rhVwjLTbWI/cifar_10_train_data_filesystem0006.tmp\n",
+ "PoolWorker-14: Wrote 1000 images to
/tmp/madlib_aTPESoNjVi/cifar_10_train_data_filesystem0006.tmp\n",
+ "PoolWorker-12: Wrote 1000 images to
/tmp/madlib_70qpwFzzqW/cifar_10_train_data_filesystem0006.tmp\n",
+ "PoolWorker-13: Wrote 1000 images to
/tmp/madlib_r2u4Zo5bPt/cifar_10_train_data_filesystem0006.tmp\n",
+ "PoolWorker-11: Wrote 1000 images to
/tmp/madlib_aEC1lF2HqL/cifar_10_train_data_filesystem0006.tmp\n",
+ "PoolWorker-15: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-14: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-12: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-11: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-13: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-15: Wrote 1000 images to
/tmp/madlib_rhVwjLTbWI/cifar_10_train_data_filesystem0007.tmp\n",
+ "PoolWorker-14: Wrote 1000 images to
/tmp/madlib_aTPESoNjVi/cifar_10_train_data_filesystem0007.tmp\n",
+ "PoolWorker-12: Wrote 1000 images to
/tmp/madlib_70qpwFzzqW/cifar_10_train_data_filesystem0007.tmp\n",
+ "PoolWorker-13: Wrote 1000 images to
/tmp/madlib_r2u4Zo5bPt/cifar_10_train_data_filesystem0007.tmp\n",
+ "PoolWorker-11: Wrote 1000 images to
/tmp/madlib_aEC1lF2HqL/cifar_10_train_data_filesystem0007.tmp\n",
+ "PoolWorker-15: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-14: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-12: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-13: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-11: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-14: Wrote 1000 images to
/tmp/madlib_aTPESoNjVi/cifar_10_train_data_filesystem0008.tmp\n",
+ "PoolWorker-15: Wrote 1000 images to
/tmp/madlib_rhVwjLTbWI/cifar_10_train_data_filesystem0008.tmp\n",
+ "PoolWorker-12: Wrote 1000 images to
/tmp/madlib_70qpwFzzqW/cifar_10_train_data_filesystem0008.tmp\n",
+ "PoolWorker-13: Wrote 1000 images to
/tmp/madlib_r2u4Zo5bPt/cifar_10_train_data_filesystem0008.tmp\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
- "PoolWorker-1: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-4: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-5: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-2: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-3: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_8IpNResiVI/cifar_10_train_data_filesystem0009.tmp\n",
- "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_PAZRj5SNyn/cifar_10_train_data_filesystem0009.tmp\n",
- "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_lyhVMiyvdL/cifar_10_train_data_filesystem0009.tmp\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_lFwfdAIfkq/cifar_10_train_data_filesystem0009.tmp\n",
- "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_x0attqv2T6/cifar_10_train_data_filesystem0009.tmp\n",
- "PoolWorker-5: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-2: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-4: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-1: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-3: Loaded 1000 images into cifar_10_train_data_filesystem\n",
- "PoolWorker-3: Removed temporary directory /tmp/madlib_x0attqv2T6\n",
- "PoolWorker-1: Removed temporary directory /tmp/madlib_lFwfdAIfkq\n",
- "PoolWorker-5: Removed temporary directory /tmp/madlib_8IpNResiVI\n",
- "PoolWorker-2: Removed temporary directory /tmp/madlib_PAZRj5SNyn\n",
- "PoolWorker-4: Removed temporary directory /tmp/madlib_lyhVMiyvdL\n",
- "Done! Loaded 10 image categories in 27.8543739319s\n",
+ "PoolWorker-11: Wrote 1000 images to
/tmp/madlib_aEC1lF2HqL/cifar_10_train_data_filesystem0008.tmp\n",
+ "PoolWorker-14: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-15: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-12: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-13: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-11: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-14: Wrote 1000 images to
/tmp/madlib_aTPESoNjVi/cifar_10_train_data_filesystem0009.tmp\n",
+ "PoolWorker-15: Wrote 1000 images to
/tmp/madlib_rhVwjLTbWI/cifar_10_train_data_filesystem0009.tmp\n",
+ "PoolWorker-12: Wrote 1000 images to
/tmp/madlib_70qpwFzzqW/cifar_10_train_data_filesystem0009.tmp\n",
+ "PoolWorker-13: Wrote 1000 images to
/tmp/madlib_r2u4Zo5bPt/cifar_10_train_data_filesystem0009.tmp\n",
+ "PoolWorker-11: Wrote 1000 images to
/tmp/madlib_aEC1lF2HqL/cifar_10_train_data_filesystem0009.tmp\n",
+ "PoolWorker-14: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-12: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-15: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-13: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-11: Loaded 1000 images into
cifar_10_train_data_filesystem\n",
+ "PoolWorker-12: Removed temporary directory /tmp/madlib_70qpwFzzqW\n",
+ "PoolWorker-13: Removed temporary directory /tmp/madlib_r2u4Zo5bPt\n",
+ "PoolWorker-15: Removed temporary directory /tmp/madlib_rhVwjLTbWI\n",
+ "PoolWorker-11: Removed temporary directory /tmp/madlib_aEC1lF2HqL\n",
+ "PoolWorker-14: Removed temporary directory /tmp/madlib_aTPESoNjVi\n",
+ "Done! Loaded 10 image categories in 27.9927430153s\n",
"5 workers terminated.\n"
]
}
@@ -595,7 +640,7 @@
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": 15,
"metadata": {},
"outputs": [
{
@@ -621,7 +666,7 @@
"[(50000L,)]"
]
},
- "execution_count": 10,
+ "execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
diff --git a/community-artifacts/Deep-learning/Load-model-architecture-v1.ipynb
b/community-artifacts/Deep-learning/Load-model-architecture-v1.ipynb
index 0cb9fa6..8e04b3c 100644
--- a/community-artifacts/Deep-learning/Load-model-architecture-v1.ipynb
+++ b/community-artifacts/Deep-learning/Load-model-architecture-v1.ipynb
@@ -25,15 +25,17 @@
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": 1,
"metadata": {},
"outputs": [
{
- "name": "stdout",
+ "name": "stderr",
"output_type": "stream",
"text": [
- "The sql extension is already loaded. To reload it, use:\n",
- " %reload_ext sql\n"
+
"/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/config.py:13:
ShimWarning: The `IPython.config` package has been deprecated since IPython
4.0. You should import from traitlets.config instead.\n",
+ " \"You should import from traitlets.config instead.\", ShimWarning)\n",
+
"/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/utils/traitlets.py:5:
UserWarning: IPython.utils.traitlets has moved to a top-level traitlets
package.\n",
+ " warn(\"IPython.utils.traitlets has moved to a top-level traitlets
package.\")\n"
]
}
],
@@ -43,34 +45,31 @@
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": 2,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
- "u'Connected: gpadmin@madlib'"
+ "u'Connected: fmcquillan@madlib'"
]
},
- "execution_count": 3,
+ "execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
- "# Greenplum Database 5.x on GCP (PM demo machine)\n",
- "#%sql postgresql://[email protected]:5432/madlib\n",
- " \n",
"# Greenplum Database 5.x on GCP for deep learning (PM demo machine)\n",
- "%sql postgresql://[email protected]:5432/madlib\n",
+ "#%sql postgresql://[email protected]:5432/madlib\n",
" \n",
"# PostgreSQL local\n",
- "#%sql postgresql://fmcquillan@localhost:5432/madlib"
+ "%sql postgresql://fmcquillan@localhost:5432/madlib"
]
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": 3,
"metadata": {},
"outputs": [
{
@@ -88,15 +87,15 @@
" <th>version</th>\n",
" </tr>\n",
" <tr>\n",
- " <td>MADlib version: 1.17-dev, git revision:
rel/v1.16-10-g205bdba, cmake configuration time: Thu Aug 15 17:53:15 UTC 2019,
build type: release, build system: Linux-3.10.0-957.21.3.el7.x86_64, C
compiler: gcc 4.8.5, C++ compiler: g++ 4.8.5</td>\n",
+ " <td>MADlib version: 1.16, git revision: rc/1.16-rc1, cmake
configuration time: Mon Jul 1 17:45:09 UTC 2019, build type: Release, build
system: Darwin-16.7.0, C compiler: Clang, C++ compiler: Clang</td>\n",
" </tr>\n",
"</table>"
],
"text/plain": [
- "[(u'MADlib version: 1.17-dev, git revision: rel/v1.16-10-g205bdba,
cmake configuration time: Thu Aug 15 17:53:15 UTC 2019, build type: release,
build system: Linux-3.10.0-957.21.3.el7.x86_64, C compiler: gcc 4.8.5, C++
compiler: g++ 4.8.5',)]"
+ "[(u'MADlib version: 1.16, git revision: rc/1.16-rc1, cmake
configuration time: Mon Jul 1 17:45:09 UTC 2019, build type: Release, build
system: Darwin-16.7.0, C compiler: Clang, C++ compiler: Clang',)]"
]
},
- "execution_count": 4,
+ "execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
@@ -118,7 +117,7 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": 4,
"metadata": {},
"outputs": [
{
@@ -151,7 +150,7 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 5,
"metadata": {},
"outputs": [
{
@@ -185,7 +184,7 @@
},
{
"cell_type": "code",
- "execution_count": 7,
+ "execution_count": 6,
"metadata": {},
"outputs": [
{
@@ -194,7 +193,7 @@
"'{\"class_name\": \"Sequential\", \"keras_version\": \"2.1.6\",
\"config\": [{\"class_name\": \"Dense\", \"config\": {\"kernel_initializer\":
{\"class_name\": \"VarianceScaling\", \"config\": {\"distribution\":
\"uniform\", \"scale\": 1.0, \"seed\": null, \"mode\": \"fan_avg\"}}, \"name\":
\"dense_1\", \"kernel_constraint\": null, \"bias_regularizer\": null,
\"bias_constraint\": null, \"dtype\": \"float32\", \"activation\": \"relu\",
\"trainable\": true, \"kernel_regularizer\": n [...]
]
},
- "execution_count": 7,
+ "execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
@@ -223,7 +222,7 @@
},
{
"cell_type": "code",
- "execution_count": 8,
+ "execution_count": 7,
"metadata": {},
"outputs": [
{
@@ -253,15 +252,15 @@
" <td>None</td>\n",
" <td>Sophie</td>\n",
" <td>A simple model</td>\n",
- " <td>__madlib_temp_76404198_1566343068_55991799__</td>\n",
+ " <td>__madlib_temp_86175082_1566932966_2506754__</td>\n",
" </tr>\n",
"</table>"
],
"text/plain": [
- "[(1, {u'class_name': u'Sequential', u'keras_version': u'2.1.6',
u'config': [{u'class_name': u'Dense', u'config': {u'kernel_initializer':
{u'class_name': u'VarianceScaling', u'config': {u'distribution': u'uniform',
u'scale': 1.0, u'seed': None, u'mode': u'fan_avg'}}, u'name': u'dense_1',
u'kernel_constraint': None, u'bias_regularizer': None, u'bias_constraint':
None, u'dtype': u'float32', u'activation': u'relu', u'trainable': True,
u'kernel_regularizer': None, u'bias_initializer': [...]
+ "[(1, {u'class_name': u'Sequential', u'keras_version': u'2.1.6',
u'config': [{u'class_name': u'Dense', u'config': {u'kernel_initializer':
{u'class_name': u'VarianceScaling', u'config': {u'distribution': u'uniform',
u'scale': 1.0, u'seed': None, u'mode': u'fan_avg'}}, u'name': u'dense_1',
u'kernel_constraint': None, u'bias_regularizer': None, u'bias_constraint':
None, u'dtype': u'float32', u'activation': u'relu', u'trainable': True,
u'kernel_regularizer': None, u'bias_initializer': [...]
]
},
- "execution_count": 8,
+ "execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
@@ -292,7 +291,7 @@
},
{
"cell_type": "code",
- "execution_count": 9,
+ "execution_count": 8,
"metadata": {},
"outputs": [
{
@@ -321,7 +320,7 @@
" <td>None</td>\n",
" <td>Sophie</td>\n",
" <td>A simple model</td>\n",
- " <td>__madlib_temp_76404198_1566343068_55991799__</td>\n",
+ " <td>__madlib_temp_86175082_1566932966_2506754__</td>\n",
" </tr>\n",
" <tr>\n",
" <td>2</td>\n",
@@ -329,16 +328,16 @@
" <td>None</td>\n",
" <td>Maria</td>\n",
" <td>Also a simple model</td>\n",
- " <td>__madlib_temp_9313391_1566343069_44328817__</td>\n",
+ " <td>__madlib_temp_82870028_1566932968_1094668__</td>\n",
" </tr>\n",
"</table>"
],
"text/plain": [
- "[(1, {u'class_name': u'Sequential', u'keras_version': u'2.1.6',
u'config': [{u'class_name': u'Dense', u'config': {u'kernel_initializer':
{u'class_name': u'VarianceScaling', u'config': {u'distribution': u'uniform',
u'scale': 1.0, u'seed': None, u'mode': u'fan_avg'}}, u'name': u'dense_1',
u'kernel_constraint': None, u'bias_regularizer': None, u'bias_constraint':
None, u'dtype': u'float32', u'activation': u'relu', u'trainable': True,
u'kernel_regularizer': None, u'bias_initializer': [...]
- " (2, {u'class_name': u'Sequential', u'keras_version': u'2.1.6',
u'config': [{u'class_name': u'Dense', u'config': {u'kernel_initializer':
{u'class_name': u'VarianceScaling', u'config': {u'distribution': u'uniform',
u'scale': 1.0, u'seed': None, u'mode': u'fan_avg'}}, u'name': u'dense_1',
u'kernel_constraint': None, u'bias_regularizer': None, u'bias_constraint':
None, u'dtype': u'float32', u'activation': u'relu', u'trainable': True,
u'kernel_regularizer': None, u'bias_initializer': [...]
+ "[(1, {u'class_name': u'Sequential', u'keras_version': u'2.1.6',
u'config': [{u'class_name': u'Dense', u'config': {u'kernel_initializer':
{u'class_name': u'VarianceScaling', u'config': {u'distribution': u'uniform',
u'scale': 1.0, u'seed': None, u'mode': u'fan_avg'}}, u'name': u'dense_1',
u'kernel_constraint': None, u'bias_regularizer': None, u'bias_constraint':
None, u'dtype': u'float32', u'activation': u'relu', u'trainable': True,
u'kernel_regularizer': None, u'bias_initializer': [...]
+ " (2, {u'class_name': u'Sequential', u'keras_version': u'2.1.6',
u'config': [{u'class_name': u'Dense', u'config': {u'kernel_initializer':
{u'class_name': u'VarianceScaling', u'config': {u'distribution': u'uniform',
u'scale': 1.0, u'seed': None, u'mode': u'fan_avg'}}, u'name': u'dense_1',
u'kernel_constraint': None, u'bias_regularizer': None, u'bias_constraint':
None, u'dtype': u'float32', u'activation': u'relu', u'trainable': True,
u'kernel_regularizer': None, u'bias_initializer': [...]
]
},
- "execution_count": 9,
+ "execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
@@ -374,30 +373,44 @@
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": 9,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
+ "1 rows affected.\n",
"1 rows affected.\n"
]
},
{
"data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>count</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
"text/plain": [
- "[]"
+ "[(1L,)]"
]
},
- "execution_count": 10,
+ "execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"%%sql\n",
- "UPDATE model_arch_library SET model_weights = model_data FROM iris_model
WHERE model_id = 2;"
+ "UPDATE model_arch_library SET model_weights = model_data FROM iris_model
WHERE model_id = 2;\n",
+ "\n",
+ "-- Check weights loaded OK\n",
+ "SELECT COUNT(*) FROM model_arch_library WHERE model_weights IS NOT NULL;"
]
},
{
@@ -411,7 +424,7 @@
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": 13,
"metadata": {},
"outputs": [
{
@@ -439,7 +452,7 @@
"[(2L,)]"
]
},
- "execution_count": 10,
+ "execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
@@ -468,9 +481,9 @@
"# load query\n",
"load_query = plpy.prepare(\"\"\"SELECT madlib.load_keras_model(\n",
" 'model_arch_library',\n",
- " $1, $2)\n",
- " \"\"\", ['json','bytea'])\n",
- "plpy.execute(load_query, [model.to_json(), weights_bytea])\n",
+ " $1, $2, $3, $4)\n",
+ " \"\"\", ['json','bytea', 'text', 'text'])\n",
+ "plpy.execute(load_query, [model.to_json(), weights_bytea, \"Ella\",
\"Model x\"])\n",
"$$ language plpythonu;\n",
"\n",
"-- Call load function\n",
@@ -481,6 +494,54 @@
]
},
{
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "3 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>model_id</th>\n",
+ " <th>name</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>Sophie</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>Maria</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " <td>Ella</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(1, u'Sophie'), (2, u'Maria'), (3, u'Ella')]"
+ ]
+ },
+ "execution_count": 14,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT model_id, name from model_arch_library ORDER BY model_id;"
+ ]
+ },
+ {
"cell_type": "markdown",
"metadata": {},
"source": [
@@ -493,7 +554,7 @@
},
{
"cell_type": "code",
- "execution_count": 11,
+ "execution_count": 15,
"metadata": {},
"outputs": [
{
@@ -511,22 +572,23 @@
" <th>count</th>\n",
" </tr>\n",
" <tr>\n",
- " <td>2</td>\n",
+ " <td>3</td>\n",
" </tr>\n",
"</table>"
],
"text/plain": [
- "[(2L,)]"
+ "[(3L,)]"
]
},
- "execution_count": 11,
+ "execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import psycopg2 as p2\n",
- "conn = p2.connect('postgresql://[email protected]:5432/madlib')\n",
+ "#conn = p2.connect('postgresql://[email protected]:5432/madlib')\n",
+ "conn = p2.connect('postgresql://fmcquillan@localhost:5432/madlib')\n",
"cur = conn.cursor()\n",
"\n",
"from keras.layers import *\n",
@@ -545,8 +607,8 @@
"weights1d = np.concatenate(weights_flat).ravel()\n",
"weights_bytea = p2.Binary(weights1d.tostring())\n",
"\n",
- "query = \"SELECT madlib.load_keras_model('model_arch_library',
%s,%s)\"\n",
- "cur.execute(query,[model.to_json(),weights_bytea])\n",
+ "query = \"SELECT madlib.load_keras_model('model_arch_library',
%s,%s,%s,%s)\"\n",
+ "cur.execute(query,[model.to_json(), weights_bytea, \"Grace\", \"Model
y\"])\n",
"conn.commit()\n",
"\n",
"# check weights loaded OK\n",
@@ -554,6 +616,58 @@
]
},
{
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "4 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>model_id</th>\n",
+ " <th>name</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>1</td>\n",
+ " <td>Sophie</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>2</td>\n",
+ " <td>Maria</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " <td>Ella</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>4</td>\n",
+ " <td>Grace</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(1, u'Sophie'), (2, u'Maria'), (3, u'Ella'), (4, u'Grace')]"
+ ]
+ },
+ "execution_count": 16,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT model_id, name from model_arch_library ORDER BY model_id;"
+ ]
+ },
+ {
"cell_type": "markdown",
"metadata": {},
"source": [
@@ -563,7 +677,7 @@
},
{
"cell_type": "code",
- "execution_count": 12,
+ "execution_count": 17,
"metadata": {},
"outputs": [
{
@@ -571,7 +685,7 @@
"output_type": "stream",
"text": [
"1 rows affected.\n",
- "1 rows affected.\n"
+ "3 rows affected.\n"
]
},
{
@@ -579,18 +693,28 @@
"text/html": [
"<table>\n",
" <tr>\n",
- " <th>count</th>\n",
+ " <th>model_id</th>\n",
+ " <th>name</th>\n",
" </tr>\n",
" <tr>\n",
" <td>2</td>\n",
+ " <td>Maria</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>3</td>\n",
+ " <td>Ella</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>4</td>\n",
+ " <td>Grace</td>\n",
" </tr>\n",
"</table>"
],
"text/plain": [
- "[(2L,)]"
+ "[(2, u'Maria'), (3, u'Ella'), (4, u'Grace')]"
]
},
- "execution_count": 12,
+ "execution_count": 17,
"metadata": {},
"output_type": "execute_result"
}
@@ -600,7 +724,7 @@
"SELECT madlib.delete_keras_model('model_arch_library', -- Output
table\n",
" 1 -- Model id\n",
" );\n",
- "SELECT COUNT(*) FROM model_arch_library;"
+ "SELECT model_id, name from model_arch_library ORDER BY model_id;"
]
}
],
diff --git
a/community-artifacts/Deep-learning/MADlib-Keras-cifar10-cnn-v2.ipynb
b/community-artifacts/Deep-learning/MADlib-Keras-cifar10-cnn-v2.ipynb
index 55232d9..faa322e 100644
--- a/community-artifacts/Deep-learning/MADlib-Keras-cifar10-cnn-v2.ipynb
+++ b/community-artifacts/Deep-learning/MADlib-Keras-cifar10-cnn-v2.ipynb
@@ -420,8 +420,8 @@
"%sql DROP TABLE IF EXISTS cifar_10_train_data, cifar_10_test_data;\n",
"\n",
"# Save images to temporary directories and load into database\n",
- "iloader.load_dataset_from_np(x_train, y_train, 'cifar_10_train_data',
append=False, no_temp_files=False)\n",
- "iloader.load_dataset_from_np(x_test, y_test, 'cifar_10_test_data',
append=False, no_temp_files=False)"
+ "iloader.load_dataset_from_np(x_train, y_train, 'cifar_10_train_data',
append=False)\n",
+ "iloader.load_dataset_from_np(x_test, y_test, 'cifar_10_test_data',
append=False)"
]
},
{
