This is an automated email from the ASF dual-hosted git repository.
fmcquillan pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/madlib-site.git
The following commit(s) were added to refs/heads/asf-site by this push:
new b53d4bd update jupyter notebooks for new image loader script
b53d4bd is described below
commit b53d4bd6b95373c5b05db554d52c3ef5fd094f0c
Author: Frank McQuillan <fmcquil...@pivotal.io>
AuthorDate: Tue Jul 23 10:41:42 2019 -0700
update jupyter notebooks for new image loader script
---
.../Deep-learning/Load-images-v1.ipynb | 662 +++++++++++++++++++++
...-v1.ipynb => MADlib-Keras-cifar10-cnn-v2.ipynb} | 348 ++++++-----
...ynb => MADlib-Keras-transfer-learning-v2.ipynb} | 516 ++++++++--------
.../Deep-learning/Madlib Image Loader Demo.ipynb | 488 ---------------
4 files changed, 1088 insertions(+), 926 deletions(-)
diff --git a/community-artifacts/Deep-learning/Load-images-v1.ipynb
b/community-artifacts/Deep-learning/Load-images-v1.ipynb
new file mode 100644
index 0000000..15aa948
--- /dev/null
+++ b/community-artifacts/Deep-learning/Load-images-v1.ipynb
@@ -0,0 +1,662 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Load images into table\n",
+ "\n",
+ "This demonstrates different ways to load images into a database table.\n",
+ "\n",
+ "We use the script called <em>madlib_image_loader.py</em> located at
https://github.com/apache/madlib-site/tree/asf-site/community-artifacts/Deep-learning
which uses the Python Imaging Library so supports multiple formats
http://www.pythonware.com/products/pil/\n";,
+ "\n",
+ "## Table of contents\n",
+ "\n",
+ "<a href=\"#setup\">1. Setup image loader</a>\n",
+ "\n",
+ "<a href=\"#fetch_numpy\">2. Fetch images then load NumPy array into
table</a>\n",
+ "\n",
+ "<a href=\"#file_system\">3. Load from file system into table</a>"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+
"/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/config.py:13:
ShimWarning: The `IPython.config` package has been deprecated since IPython
4.0. You should import from traitlets.config instead.\n",
+ " \"You should import from traitlets.config instead.\", ShimWarning)\n",
+
"/Users/fmcquillan/anaconda/lib/python2.7/site-packages/IPython/utils/traitlets.py:5:
UserWarning: IPython.utils.traitlets has moved to a top-level traitlets
package.\n",
+ " warn(\"IPython.utils.traitlets has moved to a top-level traitlets
package.\")\n"
+ ]
+ }
+ ],
+ "source": [
+ "%load_ext sql"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "u'Connected: fmcquillan@madlib'"
+ ]
+ },
+ "execution_count": 2,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# Greenplum Database 5.x on GCP for deep learning (PM demo machine)\n",
+ "#%sql postgresql://gpadmin@35.239.240.26:5432/madlib\n",
+ " \n",
+ "# PostgreSQL local\n",
+ "%sql postgresql://fmcquillan@localhost:5432/madlib"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>version</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>MADlib version: 1.16, git revision: rc/1.16-rc1, cmake
configuration time: Mon Jul 1 17:45:09 UTC 2019, build type: Release, build
system: Darwin-16.7.0, C compiler: Clang, C++ compiler: Clang</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(u'MADlib version: 1.16, git revision: rc/1.16-rc1, cmake
configuration time: Mon Jul 1 17:45:09 UTC 2019, build type: Release, build
system: Darwin-16.7.0, C compiler: Clang, C++ compiler: Clang',)]"
+ ]
+ },
+ "execution_count": 3,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%sql select madlib.version();\n",
+ "#%sql select version();"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"setup\"></a>\n",
+ "# 1. Set up image loader\n",
+ "\n",
+ "We use the script called <em>madlib_image_loader.py</em> located at
https://github.com/apache/madlib-site/tree/asf-site/community-artifacts/Deep-learning";
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Using TensorFlow backend.\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Couldn't import dot_parser, loading of dot files will not be
possible.\n"
+ ]
+ }
+ ],
+ "source": [
+ "import sys\n",
+ "import os\n",
+ "from keras.datasets import cifar10"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "madlib_site_dir =
'/Users/fmcquillan/Documents/Product/MADlib/Demos/data'\n",
+ "sys.path.append(madlib_site_dir)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Import image loader module\n",
+ "from madlib_image_loader import ImageLoader, DbCredentials"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Specify database credentials, for connecting to db\n",
+ "#db_creds = DbCredentials(user='gpadmin',\n",
+ "# host='35.239.240.26',\n",
+ "# port='5432',\n",
+ "# password='')\n",
+ "\n",
+ "# Specify database credentials, for connecting to db\n",
+ "db_creds = DbCredentials(user='fmcquillan',\n",
+ " host='localhost',\n",
+ " port='5432',\n",
+ " password='')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Initialize ImageLoader (increase num_workers to run faster)\n",
+ "iloader = ImageLoader(num_workers=5, db_creds=db_creds)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"fetch_numpy\"></a>\n",
+ "# 2. Fetch images then load NumPy array into table\n",
+ "\n",
+ "<em>iloader.load_dataset_from_np(data_x, data_y, table_name,
append=False, no_temp_files=False)</em>\n",
+ "\n",
+ "- <em>data_x</em> contains image data in np.array format\n",
+ "\n",
+ "\n",
+ "- <em>data_y</em> is a 1D np.array of the image categories (labels).\n",
+ "\n",
+ "\n",
+ "- If the user passes a <em>table_name</em> while creating ImageLoader
object, it will be used for all further calls to load_dataset_from_np. It can
be changed by passing it as a parameter during the actual call to
load_dataset_from_np, and if so future calls will load to that table name
instead. This avoids needing to pass the table_name again every time, but also
allows it to be changed at any time.\n",
+ "\n",
+ " \n",
+ "- <em>append=False</em> attempts to create a new table, while
<em>append=True</em> appends more images to an existing table.\n",
+ "\n",
+ "\n",
+ "- EXPERIMENTAL: If <em>no_temp_files=True</em>, the operation will
happen without\n",
+ " writing out the tables to temporary files before
loading them.\n",
+ " Instead, an in-memory filelike buffer (StringIO)
will be used\n",
+ " to build the tables before loading."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Load dataset into np array\n",
+ "(x_train, y_train), (x_test, y_test) = cifar10.load_data()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "MainProcess: Connected to madlib db.\n",
+ "Executing: CREATE TABLE cifar_10_train_data (id SERIAL, x REAL[], y
TEXT)\n",
+ "CREATE TABLE\n",
+ "Created table cifar_10_train_data in madlib db\n",
+ "Spawning 5 workers...\n",
+ "Initializing PoolWorker-1 [pid 32578]\n",
+ "PoolWorker-1: Created temporary directory /tmp/madlib_7llkr1iwrB\n",
+ "Initializing PoolWorker-2 [pid 32579]\n",
+ "PoolWorker-2: Created temporary directory /tmp/madlib_gFBeKWnLLL\n",
+ "Initializing PoolWorker-3 [pid 32580]\n",
+ "PoolWorker-3: Created temporary directory /tmp/madlib_3EqOi1Elxy\n",
+ "PoolWorker-1: Connected to madlib db.\n",
+ "Initializing PoolWorker-4 [pid 32581]\n",
+ "PoolWorker-4: Created temporary directory /tmp/madlib_t3HzIGOeAn\n",
+ "PoolWorker-2: Connected to madlib db.\n",
+ "Initializing PoolWorker-5 [pid 32583]\n",
+ "PoolWorker-3: Connected to madlib db.\n",
+ "PoolWorker-5: Created temporary directory /tmp/madlib_I9wQ0HGhrx\n",
+ "PoolWorker-4: Connected to madlib db.\n",
+ "PoolWorker-5: Connected to madlib db.\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_7llkr1iwrB/cifar_10_train_data0000.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_gFBeKWnLLL/cifar_10_train_data0000.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_3EqOi1Elxy/cifar_10_train_data0000.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_t3HzIGOeAn/cifar_10_train_data0000.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_I9wQ0HGhrx/cifar_10_train_data0000.tmp\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_7llkr1iwrB/cifar_10_train_data0001.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_gFBeKWnLLL/cifar_10_train_data0001.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_3EqOi1Elxy/cifar_10_train_data0001.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_t3HzIGOeAn/cifar_10_train_data0001.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_I9wQ0HGhrx/cifar_10_train_data0001.tmp\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_7llkr1iwrB/cifar_10_train_data0002.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_gFBeKWnLLL/cifar_10_train_data0002.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_3EqOi1Elxy/cifar_10_train_data0002.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_t3HzIGOeAn/cifar_10_train_data0002.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_I9wQ0HGhrx/cifar_10_train_data0002.tmp\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_7llkr1iwrB/cifar_10_train_data0003.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_gFBeKWnLLL/cifar_10_train_data0003.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_3EqOi1Elxy/cifar_10_train_data0003.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_t3HzIGOeAn/cifar_10_train_data0003.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_I9wQ0HGhrx/cifar_10_train_data0003.tmp\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_7llkr1iwrB/cifar_10_train_data0004.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_gFBeKWnLLL/cifar_10_train_data0004.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_3EqOi1Elxy/cifar_10_train_data0004.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_t3HzIGOeAn/cifar_10_train_data0004.tmp\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_I9wQ0HGhrx/cifar_10_train_data0004.tmp\n",
+ "PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_7llkr1iwrB/cifar_10_train_data0005.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_gFBeKWnLLL/cifar_10_train_data0005.tmp\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_t3HzIGOeAn/cifar_10_train_data0005.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_3EqOi1Elxy/cifar_10_train_data0005.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_I9wQ0HGhrx/cifar_10_train_data0005.tmp\n",
+ "PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_7llkr1iwrB/cifar_10_train_data0006.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_gFBeKWnLLL/cifar_10_train_data0006.tmp\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_t3HzIGOeAn/cifar_10_train_data0006.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_3EqOi1Elxy/cifar_10_train_data0006.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_I9wQ0HGhrx/cifar_10_train_data0006.tmp\n",
+ "PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_7llkr1iwrB/cifar_10_train_data0007.tmp\n",
+ "PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_gFBeKWnLLL/cifar_10_train_data0007.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_3EqOi1Elxy/cifar_10_train_data0007.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_t3HzIGOeAn/cifar_10_train_data0007.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_I9wQ0HGhrx/cifar_10_train_data0007.tmp\n",
+ "PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_7llkr1iwrB/cifar_10_train_data0008.tmp\n",
+ "PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_gFBeKWnLLL/cifar_10_train_data0008.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_3EqOi1Elxy/cifar_10_train_data0008.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_t3HzIGOeAn/cifar_10_train_data0008.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_I9wQ0HGhrx/cifar_10_train_data0008.tmp\n",
+ "PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_7llkr1iwrB/cifar_10_train_data0009.tmp\n",
+ "PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_gFBeKWnLLL/cifar_10_train_data0009.tmp\n",
+ "PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_7llkr1iwrB/cifar_10_train_data0010.tmp\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_gFBeKWnLLL/cifar_10_train_data0010.tmp\n",
+ "PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_7llkr1iwrB/cifar_10_train_data0011.tmp\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-5: Removed temporary directory /tmp/madlib_I9wQ0HGhrx\n",
+ "PoolWorker-2: Removed temporary directory /tmp/madlib_gFBeKWnLLL\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "PoolWorker-3: Removed temporary directory /tmp/madlib_3EqOi1Elxy\n",
+ "PoolWorker-1: Removed temporary directory /tmp/madlib_7llkr1iwrB\n",
+ "PoolWorker-4: Removed temporary directory /tmp/madlib_t3HzIGOeAn\n",
+ "Done! Loaded 50000 images in 24.7632198334s\n",
+ "5 workers terminated.\n",
+ "MainProcess: Connected to madlib db.\n",
+ "Executing: CREATE TABLE cifar_10_test_data (id SERIAL, x REAL[], y
TEXT)\n",
+ "CREATE TABLE\n",
+ "Created table cifar_10_test_data in madlib db\n",
+ "Spawning 5 workers...\n",
+ "Initializing PoolWorker-6 [pid 32590]\n",
+ "PoolWorker-6: Created temporary directory /tmp/madlib_ukKJGqIRI7\n",
+ "Initializing PoolWorker-7 [pid 32591]\n",
+ "PoolWorker-7: Created temporary directory /tmp/madlib_FiXPZxXK47\n",
+ "Initializing PoolWorker-8 [pid 32592]\n",
+ "Initializing PoolWorker-9 [pid 32593]\n",
+ "PoolWorker-8: Created temporary directory /tmp/madlib_EVmSLqZ6Od\n",
+ "PoolWorker-6: Connected to madlib db.\n",
+ "PoolWorker-9: Created temporary directory /tmp/madlib_b4uua9iT9p\n",
+ "PoolWorker-7: Connected to madlib db.\n",
+ "Initializing PoolWorker-10 [pid 32595]\n",
+ "PoolWorker-8: Connected to madlib db.\n",
+ "PoolWorker-10: Created temporary directory /tmp/madlib_um0N6zuuKM\n",
+ "PoolWorker-9: Connected to madlib db.\n",
+ "PoolWorker-10: Connected to madlib db.\n",
+ "PoolWorker-6: Wrote 1000 images to
/tmp/madlib_ukKJGqIRI7/cifar_10_test_data0000.tmp\n",
+ "PoolWorker-7: Wrote 1000 images to
/tmp/madlib_FiXPZxXK47/cifar_10_test_data0000.tmp\n",
+ "PoolWorker-10: Wrote 1000 images to
/tmp/madlib_um0N6zuuKM/cifar_10_test_data0000.tmp\n",
+ "PoolWorker-8: Wrote 1000 images to
/tmp/madlib_EVmSLqZ6Od/cifar_10_test_data0000.tmp\n",
+ "PoolWorker-9: Wrote 1000 images to
/tmp/madlib_b4uua9iT9p/cifar_10_test_data0000.tmp\n",
+ "PoolWorker-6: Loaded 1000 images into cifar_10_test_data\n",
+ "PoolWorker-7: Loaded 1000 images into cifar_10_test_data\n",
+ "PoolWorker-10: Loaded 1000 images into cifar_10_test_data\n",
+ "PoolWorker-8: Loaded 1000 images into cifar_10_test_data\n",
+ "PoolWorker-9: Loaded 1000 images into cifar_10_test_data\n",
+ "PoolWorker-6: Wrote 1000 images to
/tmp/madlib_ukKJGqIRI7/cifar_10_test_data0001.tmp\n",
+ "PoolWorker-7: Wrote 1000 images to
/tmp/madlib_FiXPZxXK47/cifar_10_test_data0001.tmp\n",
+ "PoolWorker-10: Wrote 1000 images to
/tmp/madlib_um0N6zuuKM/cifar_10_test_data0001.tmp\n",
+ "PoolWorker-9: Wrote 1000 images to
/tmp/madlib_b4uua9iT9p/cifar_10_test_data0001.tmp\n",
+ "PoolWorker-8: Wrote 1000 images to
/tmp/madlib_EVmSLqZ6Od/cifar_10_test_data0001.tmp\n",
+ "PoolWorker-6: Loaded 1000 images into cifar_10_test_data\n",
+ "PoolWorker-7: Loaded 1000 images into cifar_10_test_data\n",
+ "PoolWorker-10: Loaded 1000 images into cifar_10_test_data\n",
+ "PoolWorker-9: Loaded 1000 images into cifar_10_test_data\n",
+ "PoolWorker-8: Loaded 1000 images into cifar_10_test_data\n",
+ "PoolWorker-10: Removed temporary directory /tmp/madlib_um0N6zuuKM\n",
+ "PoolWorker-7: Removed temporary directory /tmp/madlib_FiXPZxXK47\n",
+ "PoolWorker-8: Removed temporary directory /tmp/madlib_EVmSLqZ6Od\n",
+ "PoolWorker-6: Removed temporary directory /tmp/madlib_ukKJGqIRI7\n",
+ "PoolWorker-9: Removed temporary directory /tmp/madlib_b4uua9iT9p\n",
+ "Done! Loaded 10000 images in 5.86952185631s\n",
+ "5 workers terminated.\n"
+ ]
+ }
+ ],
+ "source": [
+ "%sql DROP TABLE IF EXISTS cifar_10_train_data, cifar_10_test_data;\n",
+ "\n",
+ "# Save images to temporary directories and load into database\n",
+ "iloader.load_dataset_from_np(x_train, y_train, 'cifar_10_train_data',
append=False, no_temp_files=False)\n",
+ "iloader.load_dataset_from_np(x_test, y_test, 'cifar_10_test_data',
append=False, no_temp_files=False)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<a id=\"file_system\"></a>\n",
+ "# 3. Load from file system\n",
+ "\n",
+ "Uses the Python Imaging Library so supports multiple formats\n",
+ "http://www.pythonware.com/products/pil/\n";,
+ "\n",
+ "<em>load_dataset_from_disk(root_dir, table_name, num_labels='all',
append=False, no_temp_files=False)</em>\n",
+ "\n",
+ "- Calling this function will look in <em>root_dir</em> on the local disk
of wherever this is being run. It will skip over any files in that directory,
but will load images contained in each of its subdirectories. The images
should be organized by category/class, where the name of each subdirectory is
the label for the images contained within it.\n",
+ "\n",
+ "\n",
+ "- The <em>table_name</em>, <em>append</em>, and <em>no_temp_files</em>
parameters are the same as above The parameter <em>num_labels</em> is an
optional parameter which can be used to restrict the number of labels (image
classes) loaded, even if more are found in <em>root_dir</em>. For example, for
a large dataset you may have hundreds of labels, but only wish to use a subset
of that containing a few dozen.\n",
+ "\n",
+ "For example, if we put the CIFAR-10 training data is in 10 subdirectories
under directory <em>cifar10</em>, with one subdirectory for each class:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Done.\n",
+ "MainProcess: Connected to madlib db.\n",
+ "Executing: CREATE TABLE cifar_10_train_data_filesystem (id SERIAL, x
REAL[], y TEXT, img_name TEXT)\n",
+ "CREATE TABLE\n",
+ "Created table cifar_10_train_data_filesystem in madlib db\n",
+ ".DS_Store is not a directory, skipping\n",
+ "number of labels = 10\n",
+ "Found 10 image labels in /Users/fmcquillan/tmp/cifar10\n",
+ "Spawning 5 workers...\n",
+ "Initializing PoolWorker-1 [pid 64905]\n",
+ "PoolWorker-1: Created temporary directory /tmp/madlib_lFwfdAIfkq\n",
+ "Initializing PoolWorker-2 [pid 64906]\n",
+ "PoolWorker-2: Created temporary directory /tmp/madlib_PAZRj5SNyn\n",
+ "Initializing PoolWorker-3 [pid 64907]\n",
+ "PoolWorker-3: Created temporary directory /tmp/madlib_x0attqv2T6\n",
+ "PoolWorker-1: Connected to madlib db.\n",
+ "Initializing PoolWorker-4 [pid 64908]\n",
+ "PoolWorker-4: Created temporary directory /tmp/madlib_lyhVMiyvdL\n",
+ "Initializing PoolWorker-5 [pid 64909]\n",
+ "PoolWorker-5: Created temporary directory /tmp/madlib_8IpNResiVI\n",
+ "PoolWorker-3: Connected to madlib db.\n",
+ "PoolWorker-2: Connected to madlib db.\n",
+ "PoolWorker-4: Connected to madlib db.\n",
+ "PoolWorker-5: Connected to madlib db.\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_x0attqv2T6/cifar_10_train_data_filesystem0000.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_8IpNResiVI/cifar_10_train_data_filesystem0000.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_lyhVMiyvdL/cifar_10_train_data_filesystem0000.tmp\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_lFwfdAIfkq/cifar_10_train_data_filesystem0000.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_PAZRj5SNyn/cifar_10_train_data_filesystem0000.tmp\n",
+ "PoolWorker-3: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-4: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-5: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-2: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_x0attqv2T6/cifar_10_train_data_filesystem0001.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_lyhVMiyvdL/cifar_10_train_data_filesystem0001.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_8IpNResiVI/cifar_10_train_data_filesystem0001.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_PAZRj5SNyn/cifar_10_train_data_filesystem0001.tmp\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_lFwfdAIfkq/cifar_10_train_data_filesystem0001.tmp\n",
+ "PoolWorker-3: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-4: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-2: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-5: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_x0attqv2T6/cifar_10_train_data_filesystem0002.tmp\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_lFwfdAIfkq/cifar_10_train_data_filesystem0002.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_PAZRj5SNyn/cifar_10_train_data_filesystem0002.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_lyhVMiyvdL/cifar_10_train_data_filesystem0002.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_8IpNResiVI/cifar_10_train_data_filesystem0002.tmp\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-3: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-2: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-4: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-5: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_x0attqv2T6/cifar_10_train_data_filesystem0003.tmp\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_lFwfdAIfkq/cifar_10_train_data_filesystem0003.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_PAZRj5SNyn/cifar_10_train_data_filesystem0003.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_lyhVMiyvdL/cifar_10_train_data_filesystem0003.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_8IpNResiVI/cifar_10_train_data_filesystem0003.tmp\n",
+ "PoolWorker-3: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-2: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-5: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-4: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_lFwfdAIfkq/cifar_10_train_data_filesystem0004.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_x0attqv2T6/cifar_10_train_data_filesystem0004.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_PAZRj5SNyn/cifar_10_train_data_filesystem0004.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_8IpNResiVI/cifar_10_train_data_filesystem0004.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_lyhVMiyvdL/cifar_10_train_data_filesystem0004.tmp\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-3: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-2: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-5: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-4: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_lFwfdAIfkq/cifar_10_train_data_filesystem0005.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_x0attqv2T6/cifar_10_train_data_filesystem0005.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_8IpNResiVI/cifar_10_train_data_filesystem0005.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_PAZRj5SNyn/cifar_10_train_data_filesystem0005.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_lyhVMiyvdL/cifar_10_train_data_filesystem0005.tmp\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-3: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-5: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-4: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-2: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_x0attqv2T6/cifar_10_train_data_filesystem0006.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_8IpNResiVI/cifar_10_train_data_filesystem0006.tmp\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_lFwfdAIfkq/cifar_10_train_data_filesystem0006.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_PAZRj5SNyn/cifar_10_train_data_filesystem0006.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_lyhVMiyvdL/cifar_10_train_data_filesystem0006.tmp\n",
+ "PoolWorker-5: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-3: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-2: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-4: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_lFwfdAIfkq/cifar_10_train_data_filesystem0007.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_8IpNResiVI/cifar_10_train_data_filesystem0007.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_x0attqv2T6/cifar_10_train_data_filesystem0007.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_lyhVMiyvdL/cifar_10_train_data_filesystem0007.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_PAZRj5SNyn/cifar_10_train_data_filesystem0007.tmp\n",
+ "PoolWorker-5: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-3: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-4: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-2: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_8IpNResiVI/cifar_10_train_data_filesystem0008.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_x0attqv2T6/cifar_10_train_data_filesystem0008.tmp\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_lFwfdAIfkq/cifar_10_train_data_filesystem0008.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_lyhVMiyvdL/cifar_10_train_data_filesystem0008.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_PAZRj5SNyn/cifar_10_train_data_filesystem0008.tmp\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-4: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-5: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-2: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-3: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_8IpNResiVI/cifar_10_train_data_filesystem0009.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_PAZRj5SNyn/cifar_10_train_data_filesystem0009.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_lyhVMiyvdL/cifar_10_train_data_filesystem0009.tmp\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_lFwfdAIfkq/cifar_10_train_data_filesystem0009.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_x0attqv2T6/cifar_10_train_data_filesystem0009.tmp\n",
+ "PoolWorker-5: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-2: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-4: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-3: Loaded 1000 images into cifar_10_train_data_filesystem\n",
+ "PoolWorker-3: Removed temporary directory /tmp/madlib_x0attqv2T6\n",
+ "PoolWorker-1: Removed temporary directory /tmp/madlib_lFwfdAIfkq\n",
+ "PoolWorker-5: Removed temporary directory /tmp/madlib_8IpNResiVI\n",
+ "PoolWorker-2: Removed temporary directory /tmp/madlib_PAZRj5SNyn\n",
+ "PoolWorker-4: Removed temporary directory /tmp/madlib_lyhVMiyvdL\n",
+ "Done! Loaded 10 image categories in 27.8543739319s\n",
+ "5 workers terminated.\n"
+ ]
+ }
+ ],
+ "source": [
+ "%sql drop table if exists cifar_10_train_data_filesystem;\n",
+ "# Load images from file system\n",
+ "iloader.load_dataset_from_disk('/Users/fmcquillan/tmp/cifar10',
'cifar_10_train_data_filesystem', num_labels='all', append=False,
no_temp_files=False)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "<table>\n",
+ " <tr>\n",
+ " <th>count</th>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <td>50000</td>\n",
+ " </tr>\n",
+ "</table>"
+ ],
+ "text/plain": [
+ "[(50000L,)]"
+ ]
+ },
+ "execution_count": 10,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "%%sql\n",
+ "SELECT COUNT(*) FROM cifar_10_train_data_filesystem;"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 2",
+ "language": "python",
+ "name": "python2"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 2
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython2",
+ "version": "2.7.10"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git
a/community-artifacts/Deep-learning/MADlib-Keras-cifar10-cnn-v1.ipynb
b/community-artifacts/Deep-learning/MADlib-Keras-cifar10-cnn-v2.ipynb
similarity index 93%
rename from community-artifacts/Deep-learning/MADlib-Keras-cifar10-cnn-v1.ipynb
rename to community-artifacts/Deep-learning/MADlib-Keras-cifar10-cnn-v2.ipynb
index 6ba5acc..55232d9 100644
--- a/community-artifacts/Deep-learning/MADlib-Keras-cifar10-cnn-v1.ipynb
+++ b/community-artifacts/Deep-learning/MADlib-Keras-cifar10-cnn-v2.ipynb
@@ -8,6 +8,10 @@
"\n",
"E2E classification example using MADlib calling a Keras CNN. Based on
model architecture in https://keras.io/examples/cifar10_cnn/\n";,
"\n",
+ "To load images into tables we use the script called
<em>madlib_image_loader.py</em> located at
https://github.com/apache/madlib-site/tree/asf-site/community-artifacts/Deep-learning
which uses the Python Imaging Library so supports multiple formats\n",
+ "http://www.pythonware.com/products/pil/\n";,
+ "\n",
+ "\n",
"## Table of contents\n",
"<a href=\"#import_libraries\">1. Import libraries</a>\n",
"\n",
@@ -52,7 +56,7 @@
{
"data": {
"text/plain": [
- "u'Connected: gpadmin@cifar10'"
+ "u'Connected: fmcquillan@madlib'"
]
},
"execution_count": 2,
@@ -62,18 +66,15 @@
],
"source": [
"# Greenplum Database 5.x on GCP for deep learning (PM demo machine)\n",
- "%sql postgresql://gpadmin@35.239.240.26:5432/madlib\n",
- " \n",
- "# Greenplum Database 5.x on GCP (dev machine)\n",
- "#%sql postgresql://gpadmin@35.247.88.252:5432/cifar10\n",
+ "#%sql postgresql://gpadmin@35.239.240.26:5432/madlib\n",
" \n",
"# PostgreSQL local\n",
- "#%sql postgresql://fmcquillan@localhost:5432/madlib"
+ "%sql postgresql://fmcquillan@localhost:5432/madlib"
]
},
{
"cell_type": "code",
- "execution_count": 7,
+ "execution_count": 3,
"metadata": {},
"outputs": [
{
@@ -91,15 +92,15 @@
" <th>version</th>\n",
" </tr>\n",
" <tr>\n",
- " <td>PostgreSQL 8.3.23 (Greenplum Database 5.7.0 build
commit:f7c6eb5cc61b25a7ff9c5a657d6f903befbae013) on x86_64-pc-linux-gnu,
compiled by GCC gcc (GCC) 6.2.0, 64-bit compiled on Mar 30 2018
14:19:06</td>\n",
+ " <td>MADlib version: 1.16, git revision: rc/1.16-rc1, cmake
configuration time: Mon Jul 1 17:45:09 UTC 2019, build type: Release, build
system: Darwin-16.7.0, C compiler: Clang, C++ compiler: Clang</td>\n",
" </tr>\n",
"</table>"
],
"text/plain": [
- "[(u'PostgreSQL 8.3.23 (Greenplum Database 5.7.0 build
commit:f7c6eb5cc61b25a7ff9c5a657d6f903befbae013) on x86_64-pc-linux-gnu,
compiled by GCC gcc (GCC) 6.2.0, 64-bit compiled on Mar 30 2018 14:19:06',)]"
+ "[(u'MADlib version: 1.16, git revision: rc/1.16-rc1, cmake
configuration time: Mon Jul 1 17:45:09 UTC 2019, build type: Release, build
system: Darwin-16.7.0, C compiler: Clang, C++ compiler: Clang',)]"
]
},
- "execution_count": 7,
+ "execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
@@ -120,7 +121,7 @@
},
{
"cell_type": "code",
- "execution_count": 8,
+ "execution_count": 4,
"metadata": {},
"outputs": [
{
@@ -162,7 +163,7 @@
},
{
"cell_type": "code",
- "execution_count": 9,
+ "execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
@@ -180,12 +181,12 @@
"<a id=\"load_and_prepare_data\"></a>\n",
"# 2. Set up image loader and load dataset into table\n",
"\n",
- "First set up image loader."
+ "First set up image loader using the script called
<em>madlib_image_loader.py</em> located at
https://github.com/apache/madlib-site/tree/asf-site/community-artifacts/Deep-learning";
]
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
@@ -196,15 +197,15 @@
"from madlib_image_loader import ImageLoader, DbCredentials\n",
"\n",
"# Specify database credentials, for connecting to db\n",
- "db_creds = DbCredentials(user='gpadmin',\n",
- " host='35.239.240.26',\n",
+ "db_creds = DbCredentials(user='fmcquillan',\n",
+ " host='localhost',\n",
" port='5432',\n",
" password='')\n",
"\n",
"# Specify database credentials, for connecting to db\n",
"#db_creds = DbCredentials(user='gpadmin', \n",
- "# db_name='cifar10',\n",
- "# host='35.247.88.252',\n",
+ "# db_name='madlib',\n",
+ "# host='35.239.240.26',\n",
"# port='5432',\n",
"# password='')\n",
"\n",
@@ -221,191 +222,194 @@
},
{
"cell_type": "code",
- "execution_count": 11,
+ "execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
- "MainProcess: Connected to cifar10 db.\n",
+ "Done.\n",
+ "MainProcess: Connected to madlib db.\n",
"Executing: CREATE TABLE cifar_10_train_data (id SERIAL, x REAL[], y
TEXT)\n",
"CREATE TABLE\n",
- "Created table cifar_10_train_data in cifar10 db\n",
+ "Created table cifar_10_train_data in madlib db\n",
"Spawning 5 workers...\n",
- "Initializing PoolWorker-1 [pid 38560]\n",
- "PoolWorker-1: Created temporary directory PoolWorker-1\n",
- "Initializing PoolWorker-2 [pid 38561]\n",
- "PoolWorker-2: Created temporary directory PoolWorker-2\n",
- "Initializing PoolWorker-3 [pid 38562]\n",
- "PoolWorker-3: Created temporary directory PoolWorker-3\n",
- "Initializing PoolWorker-4 [pid 38563]\n",
- "PoolWorker-4: Created temporary directory PoolWorker-4\n",
- "Initializing PoolWorker-5 [pid 38564]\n",
- "PoolWorker-5: Created temporary directory PoolWorker-5\n",
- "PoolWorker-1: Connected to cifar10 db.\n",
- "PoolWorker-2: Connected to cifar10 db.\n",
- "PoolWorker-3: Connected to cifar10 db.\n",
- "PoolWorker-4: Connected to cifar10 db.\n",
- "PoolWorker-5: Connected to cifar10 db.\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_ASclSSNtOB/cifar_10_train_data0000.tmp\n",
- "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_va8z5K5H7g/cifar_10_train_data0000.tmp\n",
- "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_JcEAyIUK2N/cifar_10_train_data0000.tmp\n",
- "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_x6Z6qduBSV/cifar_10_train_data0000.tmp\n",
- "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_MwcNPSdCVQ/cifar_10_train_data0000.tmp\n",
- "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_ASclSSNtOB/cifar_10_train_data0001.tmp\n",
- "PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
+ "Initializing PoolWorker-1 [pid 28054]\n",
+ "PoolWorker-1: Created temporary directory /tmp/madlib_tdv3zEFPL1\n",
+ "Initializing PoolWorker-2 [pid 28055]\n",
+ "PoolWorker-2: Created temporary directory /tmp/madlib_bWb3jWWKsY\n",
+ "Initializing PoolWorker-3 [pid 28056]\n",
+ "PoolWorker-1: Connected to madlib db.\n",
+ "PoolWorker-3: Created temporary directory /tmp/madlib_KetBMAbjq5\n",
+ "Initializing PoolWorker-4 [pid 28057]\n",
+ "PoolWorker-2: Connected to madlib db.\n",
+ "PoolWorker-4: Created temporary directory /tmp/madlib_sME12BQHb1\n",
+ "Initializing PoolWorker-5 [pid 28059]\n",
+ "PoolWorker-3: Connected to madlib db.\n",
+ "PoolWorker-5: Created temporary directory /tmp/madlib_i6LP0aJJDY\n",
+ "PoolWorker-4: Connected to madlib db.\n",
+ "PoolWorker-5: Connected to madlib db.\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_tdv3zEFPL1/cifar_10_train_data0000.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_bWb3jWWKsY/cifar_10_train_data0000.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_KetBMAbjq5/cifar_10_train_data0000.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_sME12BQHb1/cifar_10_train_data0000.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_i6LP0aJJDY/cifar_10_train_data0000.tmp\n",
"PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_JcEAyIUK2N/cifar_10_train_data0001.tmp\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_ASclSSNtOB/cifar_10_train_data0002.tmp\n",
- "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_va8z5K5H7g/cifar_10_train_data0001.tmp\n",
- "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_ASclSSNtOB/cifar_10_train_data0003.tmp\n",
- "PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_JcEAyIUK2N/cifar_10_train_data0002.tmp\n",
- "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_x6Z6qduBSV/cifar_10_train_data0001.tmp\n",
+ "PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_tdv3zEFPL1/cifar_10_train_data0001.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_bWb3jWWKsY/cifar_10_train_data0001.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_KetBMAbjq5/cifar_10_train_data0001.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_sME12BQHb1/cifar_10_train_data0001.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_i6LP0aJJDY/cifar_10_train_data0001.tmp\n",
"PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_va8z5K5H7g/cifar_10_train_data0002.tmp\n",
- "PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_ASclSSNtOB/cifar_10_train_data0004.tmp\n",
"PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_JcEAyIUK2N/cifar_10_train_data0003.tmp\n",
"PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_va8z5K5H7g/cifar_10_train_data0003.tmp\n",
- "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_x6Z6qduBSV/cifar_10_train_data0002.tmp\n",
- "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_MwcNPSdCVQ/cifar_10_train_data0001.tmp\n",
- "PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_ASclSSNtOB/cifar_10_train_data0005.tmp\n",
- "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_JcEAyIUK2N/cifar_10_train_data0004.tmp\n",
- "PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_tdv3zEFPL1/cifar_10_train_data0002.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_bWb3jWWKsY/cifar_10_train_data0002.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_sME12BQHb1/cifar_10_train_data0002.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_i6LP0aJJDY/cifar_10_train_data0002.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_KetBMAbjq5/cifar_10_train_data0002.tmp\n",
"PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_va8z5K5H7g/cifar_10_train_data0004.tmp\n",
- "PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_x6Z6qduBSV/cifar_10_train_data0003.tmp\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_ASclSSNtOB/cifar_10_train_data0006.tmp\n",
- "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_JcEAyIUK2N/cifar_10_train_data0005.tmp\n",
- "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_MwcNPSdCVQ/cifar_10_train_data0002.tmp\n",
"PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_va8z5K5H7g/cifar_10_train_data0005.tmp\n",
+ "PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_tdv3zEFPL1/cifar_10_train_data0003.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_bWb3jWWKsY/cifar_10_train_data0003.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_sME12BQHb1/cifar_10_train_data0003.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_i6LP0aJJDY/cifar_10_train_data0003.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_KetBMAbjq5/cifar_10_train_data0003.tmp\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_x6Z6qduBSV/cifar_10_train_data0004.tmp\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_ASclSSNtOB/cifar_10_train_data0007.tmp\n",
- "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_JcEAyIUK2N/cifar_10_train_data0006.tmp\n",
"PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_va8z5K5H7g/cifar_10_train_data0006.tmp\n",
- "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_MwcNPSdCVQ/cifar_10_train_data0003.tmp\n",
"PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_tdv3zEFPL1/cifar_10_train_data0004.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_bWb3jWWKsY/cifar_10_train_data0004.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_sME12BQHb1/cifar_10_train_data0004.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_i6LP0aJJDY/cifar_10_train_data0004.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_KetBMAbjq5/cifar_10_train_data0004.tmp\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_x6Z6qduBSV/cifar_10_train_data0005.tmp\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_ASclSSNtOB/cifar_10_train_data0008.tmp\n",
- "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_JcEAyIUK2N/cifar_10_train_data0007.tmp\n",
"PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_va8z5K5H7g/cifar_10_train_data0007.tmp\n",
- "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_MwcNPSdCVQ/cifar_10_train_data0004.tmp\n",
"PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_tdv3zEFPL1/cifar_10_train_data0005.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_bWb3jWWKsY/cifar_10_train_data0005.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_sME12BQHb1/cifar_10_train_data0005.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_i6LP0aJJDY/cifar_10_train_data0005.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_KetBMAbjq5/cifar_10_train_data0005.tmp\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_x6Z6qduBSV/cifar_10_train_data0006.tmp\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_ASclSSNtOB/cifar_10_train_data0009.tmp\n",
- "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_JcEAyIUK2N/cifar_10_train_data0008.tmp\n",
"PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_tdv3zEFPL1/cifar_10_train_data0006.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_bWb3jWWKsY/cifar_10_train_data0006.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_sME12BQHb1/cifar_10_train_data0006.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_i6LP0aJJDY/cifar_10_train_data0006.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_KetBMAbjq5/cifar_10_train_data0006.tmp\n",
"PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_va8z5K5H7g/cifar_10_train_data0008.tmp\n",
- "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_MwcNPSdCVQ/cifar_10_train_data0005.tmp\n",
+ "PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_tdv3zEFPL1/cifar_10_train_data0007.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_i6LP0aJJDY/cifar_10_train_data0007.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_bWb3jWWKsY/cifar_10_train_data0007.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_sME12BQHb1/cifar_10_train_data0007.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_KetBMAbjq5/cifar_10_train_data0007.tmp\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_x6Z6qduBSV/cifar_10_train_data0007.tmp\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_ASclSSNtOB/cifar_10_train_data0010.tmp\n",
- "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_JcEAyIUK2N/cifar_10_train_data0009.tmp\n",
- "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_MwcNPSdCVQ/cifar_10_train_data0006.tmp\n",
- "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_ASclSSNtOB/cifar_10_train_data0011.tmp\n",
- "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_x6Z6qduBSV/cifar_10_train_data0008.tmp\n",
- "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_JcEAyIUK2N/cifar_10_train_data0010.tmp\n",
- "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_MwcNPSdCVQ/cifar_10_train_data0007.tmp\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_tdv3zEFPL1/cifar_10_train_data0008.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_sME12BQHb1/cifar_10_train_data0008.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_i6LP0aJJDY/cifar_10_train_data0008.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_bWb3jWWKsY/cifar_10_train_data0008.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_KetBMAbjq5/cifar_10_train_data0008.tmp\n",
"PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-5: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_JcEAyIUK2N/cifar_10_train_data0011.tmp\n",
+ "PoolWorker-2: Loaded 1000 images into cifar_10_train_data\n",
"PoolWorker-3: Loaded 1000 images into cifar_10_train_data\n",
- "PoolWorker-5: Removed temporary directory PoolWorker-5\n",
- "PoolWorker-2: Removed temporary directory PoolWorker-2\n",
- "PoolWorker-4: Removed temporary directory PoolWorker-4\n"
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_tdv3zEFPL1/cifar_10_train_data0009.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_sME12BQHb1/cifar_10_train_data0009.tmp\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_tdv3zEFPL1/cifar_10_train_data0010.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_sME12BQHb1/cifar_10_train_data0010.tmp\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-4: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_tdv3zEFPL1/cifar_10_train_data0011.tmp\n",
+ "PoolWorker-1: Loaded 1000 images into cifar_10_train_data\n",
+ "PoolWorker-3: Removed temporary directory /tmp/madlib_KetBMAbjq5\n",
+ "PoolWorker-2: Removed temporary directory /tmp/madlib_bWb3jWWKsY\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
- "PoolWorker-1: Removed temporary directory PoolWorker-1\n",
- "PoolWorker-3: Removed temporary directory PoolWorker-3\n",
- "Done! Loaded 50000 images in 48.6765198708s\n",
- "MainProcess: Connected to cifar10 db.\n",
+ "PoolWorker-4: Removed temporary directory /tmp/madlib_sME12BQHb1\n",
+ "PoolWorker-1: Removed temporary directory /tmp/madlib_tdv3zEFPL1\n",
+ "PoolWorker-5: Removed temporary directory /tmp/madlib_i6LP0aJJDY\n",
+ "Done! Loaded 50000 images in 24.227011919s\n",
+ "5 workers terminated.\n",
+ "MainProcess: Connected to madlib db.\n",
"Executing: CREATE TABLE cifar_10_test_data (id SERIAL, x REAL[], y
TEXT)\n",
"CREATE TABLE\n",
- "Created table cifar_10_test_data in cifar10 db\n",
+ "Created table cifar_10_test_data in madlib db\n",
"Spawning 5 workers...\n",
- "Initializing PoolWorker-6 [pid 38571]\n",
- "PoolWorker-6: Created temporary directory PoolWorker-6\n",
- "Initializing PoolWorker-7 [pid 38572]\n",
- "PoolWorker-7: Created temporary directory PoolWorker-7\n",
- "Initializing PoolWorker-8 [pid 38573]\n",
- "PoolWorker-8: Created temporary directory PoolWorker-8\n",
- "Initializing PoolWorker-9 [pid 38574]\n",
- "PoolWorker-9: Created temporary directory PoolWorker-9\n",
- "Initializing PoolWorker-10 [pid 38575]\n",
- "PoolWorker-10: Created temporary directory PoolWorker-10\n",
- "PoolWorker-6: Connected to cifar10 db.\n",
- "PoolWorker-7: Connected to cifar10 db.\n",
- "PoolWorker-8: Connected to cifar10 db.\n",
- "PoolWorker-9: Connected to cifar10 db.\n",
- "PoolWorker-10: Connected to cifar10 db.\n",
- "PoolWorker-6: Wrote 1000 images to
/tmp/madlib_aDPItemTNl/cifar_10_test_data0000.tmp\n",
- "PoolWorker-7: Wrote 1000 images to
/tmp/madlib_pvthRdGxuQ/cifar_10_test_data0000.tmp\n",
- "PoolWorker-8: Wrote 1000 images to
/tmp/madlib_wDeaZQyk8B/cifar_10_test_data0000.tmp\n",
- "PoolWorker-9: Wrote 1000 images to
/tmp/madlib_C43d97TTQR/cifar_10_test_data0000.tmp\n",
- "PoolWorker-10: Wrote 1000 images to
/tmp/madlib_z9IdD2HoMV/cifar_10_test_data0000.tmp\n",
+ "Initializing PoolWorker-6 [pid 28066]\n",
+ "PoolWorker-6: Created temporary directory /tmp/madlib_yKNKBHEc3G\n",
+ "Initializing PoolWorker-7 [pid 28067]\n",
+ "PoolWorker-7: Created temporary directory /tmp/madlib_hb8ESuQLva\n",
+ "Initializing PoolWorker-8 [pid 28068]\n",
+ "PoolWorker-8: Created temporary directory /tmp/madlib_PmtDmYhSBj\n",
+ "PoolWorker-6: Connected to madlib db.\n",
+ "Initializing PoolWorker-9 [pid 28069]\n",
+ "PoolWorker-7: Connected to madlib db.\n",
+ "PoolWorker-9: Created temporary directory /tmp/madlib_h7oUVpBwyZ\n",
+ "Initializing PoolWorker-10 [pid 28071]\n",
+ "PoolWorker-8: Connected to madlib db.\n",
+ "PoolWorker-10: Created temporary directory /tmp/madlib_9TZoE98hbn\n",
+ "PoolWorker-9: Connected to madlib db.\n",
+ "PoolWorker-10: Connected to madlib db.\n",
+ "PoolWorker-8: Wrote 1000 images to
/tmp/madlib_PmtDmYhSBj/cifar_10_test_data0000.tmp\n",
+ "PoolWorker-6: Wrote 1000 images to
/tmp/madlib_yKNKBHEc3G/cifar_10_test_data0000.tmp\n",
+ "PoolWorker-7: Wrote 1000 images to
/tmp/madlib_hb8ESuQLva/cifar_10_test_data0000.tmp\n",
+ "PoolWorker-9: Wrote 1000 images to
/tmp/madlib_h7oUVpBwyZ/cifar_10_test_data0000.tmp\n",
+ "PoolWorker-10: Wrote 1000 images to
/tmp/madlib_9TZoE98hbn/cifar_10_test_data0000.tmp\n",
+ "PoolWorker-8: Loaded 1000 images into cifar_10_test_data\n",
"PoolWorker-7: Loaded 1000 images into cifar_10_test_data\n",
- "PoolWorker-9: Loaded 1000 images into cifar_10_test_data\n",
"PoolWorker-6: Loaded 1000 images into cifar_10_test_data\n",
- "PoolWorker-7: Wrote 1000 images to
/tmp/madlib_pvthRdGxuQ/cifar_10_test_data0001.tmp\n",
- "PoolWorker-9: Wrote 1000 images to
/tmp/madlib_C43d97TTQR/cifar_10_test_data0001.tmp\n",
"PoolWorker-10: Loaded 1000 images into cifar_10_test_data\n",
- "PoolWorker-6: Wrote 1000 images to
/tmp/madlib_aDPItemTNl/cifar_10_test_data0001.tmp\n",
+ "PoolWorker-9: Loaded 1000 images into cifar_10_test_data\n",
+ "PoolWorker-8: Wrote 1000 images to
/tmp/madlib_PmtDmYhSBj/cifar_10_test_data0001.tmp\n",
+ "PoolWorker-7: Wrote 1000 images to
/tmp/madlib_hb8ESuQLva/cifar_10_test_data0001.tmp\n",
+ "PoolWorker-6: Wrote 1000 images to
/tmp/madlib_yKNKBHEc3G/cifar_10_test_data0001.tmp\n",
+ "PoolWorker-10: Wrote 1000 images to
/tmp/madlib_9TZoE98hbn/cifar_10_test_data0001.tmp\n",
+ "PoolWorker-9: Wrote 1000 images to
/tmp/madlib_h7oUVpBwyZ/cifar_10_test_data0001.tmp\n",
"PoolWorker-8: Loaded 1000 images into cifar_10_test_data\n",
"PoolWorker-7: Loaded 1000 images into cifar_10_test_data\n",
- "PoolWorker-9: Loaded 1000 images into cifar_10_test_data\n",
- "PoolWorker-10: Wrote 1000 images to
/tmp/madlib_z9IdD2HoMV/cifar_10_test_data0001.tmp\n",
"PoolWorker-6: Loaded 1000 images into cifar_10_test_data\n",
- "PoolWorker-8: Wrote 1000 images to
/tmp/madlib_wDeaZQyk8B/cifar_10_test_data0001.tmp\n",
"PoolWorker-10: Loaded 1000 images into cifar_10_test_data\n",
- "PoolWorker-8: Loaded 1000 images into cifar_10_test_data\n",
- "PoolWorker-9: Removed temporary directory PoolWorker-9\n",
- "PoolWorker-6: Removed temporary directory PoolWorker-6\n",
- "PoolWorker-10: Removed temporary directory PoolWorker-10\n",
- "PoolWorker-8: Removed temporary directory PoolWorker-8\n",
- "PoolWorker-7: Removed temporary directory PoolWorker-7\n",
- "Done! Loaded 10000 images in 11.5516760349s\n"
+ "PoolWorker-9: Loaded 1000 images into cifar_10_test_data\n",
+ "PoolWorker-8: Removed temporary directory /tmp/madlib_PmtDmYhSBj\n",
+ "PoolWorker-7: Removed temporary directory /tmp/madlib_hb8ESuQLva\n",
+ "PoolWorker-6: Removed temporary directory /tmp/madlib_yKNKBHEc3G\n",
+ "PoolWorker-9: Removed temporary directory /tmp/madlib_h7oUVpBwyZ\n",
+ "PoolWorker-10: Removed temporary directory /tmp/madlib_9TZoE98hbn\n",
+ "Done! Loaded 10000 images in 4.54620194435s\n",
+ "5 workers terminated.\n"
]
}
],
@@ -413,16 +417,16 @@
"# Load dataset into np array\n",
"(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n",
"\n",
- "#%sql DROP TABLE IF EXISTS cifar_10_train_data, cifar_10_test_data;\n",
+ "%sql DROP TABLE IF EXISTS cifar_10_train_data, cifar_10_test_data;\n",
"\n",
"# Save images to temporary directories and load into database\n",
- "iloader.load_np_array_to_table(x_train, y_train, 'cifar_10_train_data',
append=False, img_names=None)\n",
- "iloader.load_np_array_to_table(x_test, y_test, 'cifar_10_test_data',
append=False, img_names=None)"
+ "iloader.load_dataset_from_np(x_train, y_train, 'cifar_10_train_data',
append=False, no_temp_files=False)\n",
+ "iloader.load_dataset_from_np(x_test, y_test, 'cifar_10_test_data',
append=False, no_temp_files=False)"
]
},
{
"cell_type": "code",
- "execution_count": 14,
+ "execution_count": 10,
"metadata": {},
"outputs": [
{
@@ -448,7 +452,7 @@
"[(50000L,)]"
]
},
- "execution_count": 14,
+ "execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
@@ -471,7 +475,7 @@
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": 9,
"metadata": {
"scrolled": true
},
@@ -507,17 +511,17 @@
" <td>x</td>\n",
" <td>text</td>\n",
" <td>[u'0', u'1', u'2', u'3', u'4', u'5', u'6', u'7', u'8',
u'9']</td>\n",
- " <td>24415</td>\n",
+ " <td>1000</td>\n",
" <td>255.0</td>\n",
" <td>10</td>\n",
" </tr>\n",
"</table>"
],
"text/plain": [
- "[(u'cifar_10_train_data', u'cifar_10_train_data_packed', u'y', u'x',
u'text', [u'0', u'1', u'2', u'3', u'4', u'5', u'6', u'7', u'8', u'9'], 24415,
255.0, 10)]"
+ "[(u'cifar_10_train_data', u'cifar_10_train_data_packed', u'y', u'x',
u'text', [u'0', u'1', u'2', u'3', u'4', u'5', u'6', u'7', u'8', u'9'], 1000,
255.0, 10)]"
]
},
- "execution_count": 10,
+ "execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
@@ -580,14 +584,14 @@
" <td>x</td>\n",
" <td>text</td>\n",
" <td>[u'0', u'1', u'2', u'3', u'4', u'5', u'6', u'7', u'8',
u'9']</td>\n",
- " <td>5000</td>\n",
+ " <td>1000</td>\n",
" <td>255.0</td>\n",
" <td>10</td>\n",
" </tr>\n",
"</table>"
],
"text/plain": [
- "[(u'cifar_10_test_data', u'cifar_10_test_data_packed', u'y', u'x',
u'text', [u'0', u'1', u'2', u'3', u'4', u'5', u'6', u'7', u'8', u'9'], 5000,
255.0, 10)]"
+ "[(u'cifar_10_test_data', u'cifar_10_test_data_packed', u'y', u'x',
u'text', [u'0', u'1', u'2', u'3', u'4', u'5', u'6', u'7', u'8', u'9'], 1000,
255.0, 10)]"
]
},
"execution_count": 11,
@@ -747,7 +751,8 @@
],
"source": [
"import psycopg2 as p2\n",
- "conn = p2.connect('postgresql://gpadmin@35.239.240.26:5432/madlib')\n",
+ "#conn = p2.connect('postgresql://gpadmin@35.239.240.26:5432/madlib')\n",
+ "conn = p2.connect('postgresql://fmcquillan@localhost:5432/madlib')\n",
"cur = conn.cursor()\n",
"\n",
"%sql DROP TABLE IF EXISTS model_arch_library;\n",
@@ -769,36 +774,15 @@
},
{
"cell_type": "code",
- "execution_count": 22,
+ "execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
- "Done.\n",
- "1 rows affected.\n"
+ "Done.\n"
]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>madlib_keras_fit</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td></td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[('',)]"
- ]
- },
- "execution_count": 22,
- "metadata": {},
- "output_type": "execute_result"
}
],
"source": [
@@ -811,7 +795,7 @@
" 1, -- model
arch id\n",
" $$ loss='categorical_crossentropy',
optimizer='rmsprop(lr=0.0001, decay=1e-6)', metrics=['accuracy']$$, --
compile_params\n",
" $$ batch_size=32, epochs=3 $$, --
fit_params\n",
- " 20, --
num_iterations\n",
+ " 3, --
num_iterations\n",
" 0, -- GPUs
per host\n",
" 'cifar_10_test_data_packed', --
validation dataset\n",
" 2 --
metrics compute frequency\n",
@@ -1241,7 +1225,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
- "version": "2.7.12"
+ "version": "2.7.10"
}
},
"nbformat": 4,
diff --git
a/community-artifacts/Deep-learning/MADlib-Keras-transfer-learning-v1.ipynb
b/community-artifacts/Deep-learning/MADlib-Keras-transfer-learning-v2.ipynb
similarity index 70%
rename from
community-artifacts/Deep-learning/MADlib-Keras-transfer-learning-v1.ipynb
rename to
community-artifacts/Deep-learning/MADlib-Keras-transfer-learning-v2.ipynb
index faaad68..d8f46ed 100644
--- a/community-artifacts/Deep-learning/MADlib-Keras-transfer-learning-v1.ipynb
+++ b/community-artifacts/Deep-learning/MADlib-Keras-transfer-learning-v2.ipynb
@@ -8,6 +8,8 @@
"\n",
"This is a transfer learning example based on
https://keras.io/examples/mnist_transfer_cnn/ \n",
"\n",
+ "To load images into tables we use the script called
<em>madlib_image_loader.py</em> located at
https://github.com/apache/madlib-site/tree/asf-site/community-artifacts/Deep-learning
which uses the Python Imaging Library so supports multiple formats
http://www.pythonware.com/products/pil/\n";,
+ "\n",
"## Table of contents\n",
"<a href=\"#import_libraries\">1. Import libraries</a>\n",
"\n",
@@ -61,9 +63,6 @@
}
],
"source": [
- "# Greenplum Database 5.x on GCP (PM demo machine)\n",
- "#%sql postgresql://gpadmin@35.184.232.200:5432/madlib\n",
- " \n",
"# Greenplum Database 5.x on GCP for deep learning (PM demo machine)\n",
"%sql postgresql://gpadmin@35.239.240.26:5432/madlib\n",
" \n",
@@ -91,12 +90,12 @@
" <th>version</th>\n",
" </tr>\n",
" <tr>\n",
- " <td>MADlib version: 1.16-dev, git revision:
rel/v1.15.1-129-g954609a, cmake configuration time: Fri Jun 21 18:18:35 UTC
2019, build type: release, build system: Linux-3.10.0-957.12.1.el7.x86_64, C
compiler: gcc 4.8.5, C++ compiler: g++ 4.8.5</td>\n",
+ " <td>MADlib version: 1.17-dev, git revision:
rel/v1.16-2-g8a612fe, cmake configuration time: Wed Jul 17 18:49:47 UTC 2019,
build type: release, build system: Linux-3.10.0-957.21.3.el7.x86_64, C
compiler: gcc 4.8.5, C++ compiler: g++ 4.8.5</td>\n",
" </tr>\n",
"</table>"
],
"text/plain": [
- "[(u'MADlib version: 1.16-dev, git revision: rel/v1.15.1-129-g954609a,
cmake configuration time: Fri Jun 21 18:18:35 UTC 2019, build type: release,
build system: Linux-3.10.0-957.12.1.el7.x86_64, C compiler: gcc 4.8.5, C++
compiler: g++ 4.8.5',)]"
+ "[(u'MADlib version: 1.17-dev, git revision: rel/v1.16-2-g8a612fe,
cmake configuration time: Wed Jul 17 18:49:47 UTC 2019, build type: release,
build system: Linux-3.10.0-957.21.3.el7.x86_64, C compiler: gcc 4.8.5, C++
compiler: g++ 4.8.5',)]"
]
},
"execution_count": 3,
@@ -199,7 +198,7 @@
},
{
"cell_type": "code",
- "execution_count": 14,
+ "execution_count": 6,
"metadata": {},
"outputs": [
{
@@ -239,7 +238,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "Load datasets into tables using image loader scripts"
+ "Load datasets into tables using image loader scripts called
<em>madlib_image_loader.py</em> located at
https://github.com/apache/madlib-site/tree/asf-site/community-artifacts/Deep-learning";
]
},
{
@@ -283,7 +282,7 @@
},
{
"cell_type": "code",
- "execution_count": 16,
+ "execution_count": 10,
"metadata": {},
"outputs": [
{
@@ -291,259 +290,264 @@
"output_type": "stream",
"text": [
"Done.\n",
+ "MainProcess: Connected to madlib db.\n",
"Executing: CREATE TABLE train_lt5 (id SERIAL, x REAL[], y TEXT)\n",
"CREATE TABLE\n",
"Created table train_lt5 in madlib db\n",
"Spawning 5 workers...\n",
- "Initializing PoolWorker-31 [pid 36504]\n",
- "PoolWorker-31: Created temporary directory PoolWorker-31\n",
- "Initializing PoolWorker-32 [pid 36505]\n",
- "PoolWorker-32: Created temporary directory PoolWorker-32\n",
- "Initializing PoolWorker-33 [pid 36506]\n",
- "PoolWorker-33: Created temporary directory PoolWorker-33\n",
- "Initializing PoolWorker-34 [pid 36507]\n",
- "PoolWorker-34: Created temporary directory PoolWorker-34\n",
- "Initializing PoolWorker-35 [pid 36508]\n",
- "PoolWorker-35: Created temporary directory PoolWorker-35\n",
- "PoolWorker-31: Connected to madlib db.\n",
- "PoolWorker-32: Connected to madlib db.\n",
- "PoolWorker-33: Connected to madlib db.\n",
- "PoolWorker-34: Connected to madlib db.\n",
- "PoolWorker-35: Connected to madlib db.\n",
- "PoolWorker-32: Wrote 1000 images to
/tmp/madlib_BaxJbXtAOV/train_lt50000.tmp\n",
- "PoolWorker-31: Wrote 1000 images to
/tmp/madlib_gijRoaCfJc/train_lt50000.tmp\n",
- "PoolWorker-33: Wrote 1000 images to
/tmp/madlib_OpZNic57mi/train_lt50000.tmp\n",
- "PoolWorker-35: Wrote 1000 images to
/tmp/madlib_l9HyKE46Lp/train_lt50000.tmp\n",
- "PoolWorker-34: Wrote 1000 images to
/tmp/madlib_92vUGUI7Kg/train_lt50000.tmp\n",
- "PoolWorker-31: Loaded 1000 images into train_lt5\n",
- "PoolWorker-32: Loaded 1000 images into train_lt5\n",
- "PoolWorker-34: Loaded 1000 images into train_lt5\n",
- "PoolWorker-33: Loaded 1000 images into train_lt5\n",
- "PoolWorker-31: Wrote 1000 images to
/tmp/madlib_gijRoaCfJc/train_lt50001.tmp\n",
- "PoolWorker-35: Loaded 1000 images into train_lt5\n",
- "PoolWorker-32: Wrote 1000 images to
/tmp/madlib_BaxJbXtAOV/train_lt50001.tmp\n",
- "PoolWorker-34: Wrote 1000 images to
/tmp/madlib_92vUGUI7Kg/train_lt50001.tmp\n",
- "PoolWorker-33: Wrote 1000 images to
/tmp/madlib_OpZNic57mi/train_lt50001.tmp\n",
- "PoolWorker-35: Wrote 1000 images to
/tmp/madlib_l9HyKE46Lp/train_lt50001.tmp\n",
- "PoolWorker-31: Loaded 1000 images into train_lt5\n",
- "PoolWorker-32: Loaded 1000 images into train_lt5\n",
- "PoolWorker-34: Loaded 1000 images into train_lt5\n",
- "PoolWorker-31: Wrote 1000 images to
/tmp/madlib_gijRoaCfJc/train_lt50002.tmp\n",
- "PoolWorker-32: Wrote 1000 images to
/tmp/madlib_BaxJbXtAOV/train_lt50002.tmp\n",
- "PoolWorker-33: Loaded 1000 images into train_lt5\n",
- "PoolWorker-34: Wrote 1000 images to
/tmp/madlib_92vUGUI7Kg/train_lt50002.tmp\n",
- "PoolWorker-35: Loaded 1000 images into train_lt5\n",
- "PoolWorker-33: Wrote 1000 images to
/tmp/madlib_OpZNic57mi/train_lt50002.tmp\n",
- "PoolWorker-31: Loaded 1000 images into train_lt5\n",
- "PoolWorker-35: Wrote 1000 images to
/tmp/madlib_l9HyKE46Lp/train_lt50002.tmp\n",
- "PoolWorker-34: Loaded 1000 images into train_lt5\n",
- "PoolWorker-32: Loaded 1000 images into train_lt5\n",
- "PoolWorker-31: Wrote 1000 images to
/tmp/madlib_gijRoaCfJc/train_lt50003.tmp\n",
- "PoolWorker-34: Wrote 1000 images to
/tmp/madlib_92vUGUI7Kg/train_lt50003.tmp\n",
- "PoolWorker-32: Wrote 1000 images to
/tmp/madlib_BaxJbXtAOV/train_lt50003.tmp\n",
- "PoolWorker-33: Loaded 1000 images into train_lt5\n",
- "PoolWorker-31: Loaded 1000 images into train_lt5\n",
- "PoolWorker-33: Wrote 1000 images to
/tmp/madlib_OpZNic57mi/train_lt50003.tmp\n",
- "PoolWorker-35: Loaded 1000 images into train_lt5\n",
- "PoolWorker-31: Wrote 1000 images to
/tmp/madlib_gijRoaCfJc/train_lt50004.tmp\n",
- "PoolWorker-34: Loaded 1000 images into train_lt5\n",
- "PoolWorker-35: Wrote 1000 images to
/tmp/madlib_l9HyKE46Lp/train_lt50003.tmp\n",
- "PoolWorker-32: Loaded 1000 images into train_lt5\n",
- "PoolWorker-34: Wrote 1000 images to
/tmp/madlib_92vUGUI7Kg/train_lt50004.tmp\n",
- "PoolWorker-31: Loaded 1000 images into train_lt5\n",
- "PoolWorker-33: Loaded 1000 images into train_lt5\n",
- "PoolWorker-32: Wrote 1000 images to
/tmp/madlib_BaxJbXtAOV/train_lt50004.tmp\n",
- "PoolWorker-31: Wrote 1000 images to
/tmp/madlib_gijRoaCfJc/train_lt50005.tmp\n",
- "PoolWorker-33: Wrote 1000 images to
/tmp/madlib_OpZNic57mi/train_lt50004.tmp\n",
- "PoolWorker-34: Loaded 1000 images into train_lt5\n",
- "PoolWorker-35: Loaded 1000 images into train_lt5\n",
- "PoolWorker-32: Loaded 1000 images into train_lt5\n",
- "PoolWorker-31: Loaded 1000 images into train_lt5\n",
- "PoolWorker-35: Wrote 1000 images to
/tmp/madlib_l9HyKE46Lp/train_lt50004.tmp\n",
- "PoolWorker-34: Wrote 1000 images to
/tmp/madlib_92vUGUI7Kg/train_lt50005.tmp\n",
- "PoolWorker-31: Wrote 596 images to
/tmp/madlib_gijRoaCfJc/train_lt50006.tmp\n",
- "PoolWorker-33: Loaded 1000 images into train_lt5\n",
- "PoolWorker-32: Wrote 1000 images to
/tmp/madlib_BaxJbXtAOV/train_lt50005.tmp\n",
- "PoolWorker-33: Wrote 1000 images to
/tmp/madlib_OpZNic57mi/train_lt50005.tmp\n",
- "PoolWorker-31: Loaded 596 images into train_lt5\n",
- "PoolWorker-34: Loaded 1000 images into train_lt5\n",
- "PoolWorker-32: Loaded 1000 images into train_lt5\n",
- "PoolWorker-35: Loaded 1000 images into train_lt5\n",
- "PoolWorker-33: Loaded 1000 images into train_lt5\n",
- "PoolWorker-35: Wrote 1000 images to
/tmp/madlib_l9HyKE46Lp/train_lt50005.tmp\n",
- "PoolWorker-35: Loaded 1000 images into train_lt5\n",
- "PoolWorker-35: Removed temporary directory PoolWorker-35\n",
- "PoolWorker-34: Removed temporary directory PoolWorker-34\n",
- "PoolWorker-33: Removed temporary directory PoolWorker-33\n",
- "PoolWorker-31: Removed temporary directory PoolWorker-31\n",
- "PoolWorker-32: Removed temporary directory PoolWorker-32\n",
- "Done! Loaded 30596 images in 13.3909709454s\n",
+ "Initializing PoolWorker-1 [pid 32452]\n",
+ "PoolWorker-1: Created temporary directory /tmp/madlib_RqdNj6bF0W\n",
+ "Initializing PoolWorker-2 [pid 32453]\n",
+ "PoolWorker-2: Created temporary directory /tmp/madlib_2dVRSzAit5\n",
+ "Initializing PoolWorker-3 [pid 32454]\n",
+ "PoolWorker-3: Created temporary directory /tmp/madlib_wAZd4zRnM5\n",
+ "Initializing PoolWorker-4 [pid 32455]\n",
+ "PoolWorker-4: Created temporary directory /tmp/madlib_fGfGIO2M6v\n",
+ "Initializing PoolWorker-5 [pid 32456]\n",
+ "PoolWorker-5: Created temporary directory /tmp/madlib_wJYreK1smG\n",
+ "PoolWorker-1: Connected to madlib db.\n",
+ "PoolWorker-2: Connected to madlib db.\n",
+ "PoolWorker-3: Connected to madlib db.\n",
+ "PoolWorker-4: Connected to madlib db.\n",
+ "PoolWorker-5: Connected to madlib db.\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_RqdNj6bF0W/train_lt50000.tmp\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_2dVRSzAit5/train_lt50000.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_fGfGIO2M6v/train_lt50000.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_wAZd4zRnM5/train_lt50000.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_wJYreK1smG/train_lt50000.tmp\n",
+ "PoolWorker-1: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-4: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-5: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-3: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_RqdNj6bF0W/train_lt50001.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_wJYreK1smG/train_lt50001.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_fGfGIO2M6v/train_lt50001.tmp\n",
+ "PoolWorker-2: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_wAZd4zRnM5/train_lt50001.tmp\n",
+ "PoolWorker-1: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_2dVRSzAit5/train_lt50001.tmp\n",
+ "PoolWorker-5: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-3: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-4: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_RqdNj6bF0W/train_lt50002.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_wJYreK1smG/train_lt50002.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_wAZd4zRnM5/train_lt50002.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_fGfGIO2M6v/train_lt50002.tmp\n",
+ "PoolWorker-2: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-1: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-5: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-4: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-3: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_2dVRSzAit5/train_lt50002.tmp\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_RqdNj6bF0W/train_lt50003.tmp\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_wJYreK1smG/train_lt50003.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_fGfGIO2M6v/train_lt50003.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_wAZd4zRnM5/train_lt50003.tmp\n",
+ "PoolWorker-2: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-1: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-4: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_2dVRSzAit5/train_lt50003.tmp\n",
+ "PoolWorker-3: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_RqdNj6bF0W/train_lt50004.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_fGfGIO2M6v/train_lt50004.tmp\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_wAZd4zRnM5/train_lt50004.tmp\n",
+ "PoolWorker-2: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-4: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-1: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-3: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_2dVRSzAit5/train_lt50004.tmp\n",
+ "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_fGfGIO2M6v/train_lt50005.tmp\n",
+ "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_RqdNj6bF0W/train_lt50005.tmp\n",
+ "PoolWorker-5: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_wAZd4zRnM5/train_lt50005.tmp\n",
+ "PoolWorker-4: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_wJYreK1smG/train_lt50004.tmp\n",
+ "PoolWorker-2: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-1: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-4: Wrote 596 images to
/tmp/madlib_fGfGIO2M6v/train_lt50006.tmp\n",
+ "PoolWorker-3: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_2dVRSzAit5/train_lt50005.tmp\n",
+ "PoolWorker-4: Loaded 596 images into train_lt5\n",
+ "PoolWorker-2: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-5: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_wJYreK1smG/train_lt50005.tmp\n",
+ "PoolWorker-5: Loaded 1000 images into train_lt5\n",
+ "PoolWorker-1: Removed temporary directory /tmp/madlib_RqdNj6bF0W\n",
+ "PoolWorker-5: Removed temporary directory /tmp/madlib_wJYreK1smG\n",
+ "PoolWorker-4: Removed temporary directory /tmp/madlib_fGfGIO2M6v\n",
+ "PoolWorker-3: Removed temporary directory /tmp/madlib_wAZd4zRnM5\n",
+ "PoolWorker-2: Removed temporary directory /tmp/madlib_2dVRSzAit5\n",
+ "Done! Loaded 30596 images in 14.2171461582s\n",
+ "5 workers terminated.\n",
"MainProcess: Connected to madlib db.\n",
"Executing: CREATE TABLE test_lt5 (id SERIAL, x REAL[], y TEXT)\n",
"CREATE TABLE\n",
"Created table test_lt5 in madlib db\n",
"Spawning 5 workers...\n",
- "Initializing PoolWorker-36 [pid 36510]\n",
- "PoolWorker-36: Created temporary directory PoolWorker-36\n",
- "Initializing PoolWorker-37 [pid 36511]\n",
- "PoolWorker-37: Created temporary directory PoolWorker-37\n",
- "Initializing PoolWorker-38 [pid 36512]\n",
- "PoolWorker-38: Created temporary directory PoolWorker-38\n",
- "Initializing PoolWorker-39 [pid 36513]\n",
- "PoolWorker-39: Created temporary directory PoolWorker-39\n",
- "Initializing PoolWorker-40 [pid 36514]\n",
- "PoolWorker-40: Created temporary directory PoolWorker-40\n",
- "PoolWorker-36: Connected to madlib db.\n",
- "PoolWorker-37: Connected to madlib db.\n",
- "PoolWorker-38: Connected to madlib db.\n",
- "PoolWorker-39: Connected to madlib db.\n",
- "PoolWorker-40: Connected to madlib db.\n",
- "PoolWorker-37: Wrote 1000 images to
/tmp/madlib_TPjU7lz34X/test_lt50000.tmp\n",
- "PoolWorker-38: Wrote 1000 images to
/tmp/madlib_PNUF8ABUjF/test_lt50000.tmp\n",
- "PoolWorker-39: Wrote 1000 images to
/tmp/madlib_4X2ZzvvwFm/test_lt50000.tmp\n",
- "PoolWorker-40: Wrote 1000 images to
/tmp/madlib_neDGAgy83W/test_lt50000.tmp\n",
- "PoolWorker-36: Wrote 1000 images to
/tmp/madlib_L6JjsmxTxZ/test_lt50000.tmp\n",
- "PoolWorker-36: Loaded 1000 images into test_lt5\n",
- "PoolWorker-37: Loaded 1000 images into test_lt5\n",
- "PoolWorker-36: Wrote 139 images to
/tmp/madlib_L6JjsmxTxZ/test_lt50001.tmp\n",
- "PoolWorker-38: Loaded 1000 images into test_lt5\n",
- "PoolWorker-40: Loaded 1000 images into test_lt5\n",
- "PoolWorker-36: Loaded 139 images into test_lt5\n",
- "PoolWorker-39: Loaded 1000 images into test_lt5\n",
- "PoolWorker-38: Removed temporary directory PoolWorker-38\n",
- "PoolWorker-37: Removed temporary directory PoolWorker-37\n",
- "PoolWorker-36: Removed temporary directory PoolWorker-36\n",
- "PoolWorker-40: Removed temporary directory PoolWorker-40\n",
- "PoolWorker-39: Removed temporary directory PoolWorker-39\n",
- "Done! Loaded 5139 images in 3.34779882431s\n",
+ "Initializing PoolWorker-6 [pid 32457]\n",
+ "PoolWorker-6: Created temporary directory /tmp/madlib_4n6hb9xfIi\n",
+ "Initializing PoolWorker-7 [pid 32458]\n",
+ "PoolWorker-7: Created temporary directory /tmp/madlib_F1kcKRJNq0\n",
+ "Initializing PoolWorker-8 [pid 32459]\n",
+ "PoolWorker-8: Created temporary directory /tmp/madlib_XpsmAPt8MP\n",
+ "Initializing PoolWorker-9 [pid 32460]\n",
+ "PoolWorker-10: Created temporary directory /tmp/madlib_ZcWfxD9Vqm\n",
+ "PoolWorker-9: Created temporary directory /tmp/madlib_7TQeRq7vm6\n",
+ "Initializing PoolWorker-10 [pid 32461]\n",
+ "PoolWorker-6: Connected to madlib db.\n",
+ "PoolWorker-7: Connected to madlib db.\n",
+ "PoolWorker-8: Connected to madlib db.\n",
+ "PoolWorker-9: Connected to madlib db.\n",
+ "PoolWorker-10: Connected to madlib db.\n",
+ "PoolWorker-9: Wrote 1000 images to
/tmp/madlib_7TQeRq7vm6/test_lt50000.tmp\n",
+ "PoolWorker-6: Wrote 1000 images to
/tmp/madlib_4n6hb9xfIi/test_lt50000.tmp\n",
+ "PoolWorker-7: Wrote 1000 images to
/tmp/madlib_F1kcKRJNq0/test_lt50000.tmp\n",
+ "PoolWorker-8: Wrote 1000 images to
/tmp/madlib_XpsmAPt8MP/test_lt50000.tmp\n",
+ "PoolWorker-10: Wrote 1000 images to
/tmp/madlib_ZcWfxD9Vqm/test_lt50000.tmp\n",
+ "PoolWorker-6: Loaded 1000 images into test_lt5\n",
+ "PoolWorker-6: Wrote 139 images to
/tmp/madlib_4n6hb9xfIi/test_lt50001.tmp\n",
+ "PoolWorker-7: Loaded 1000 images into test_lt5\n",
+ "PoolWorker-8: Loaded 1000 images into test_lt5\n",
+ "PoolWorker-9: Loaded 1000 images into test_lt5\n",
+ "PoolWorker-10: Loaded 1000 images into test_lt5\n",
+ "PoolWorker-6: Loaded 139 images into test_lt5\n",
+ "PoolWorker-9: Removed temporary directory /tmp/madlib_7TQeRq7vm6\n",
+ "PoolWorker-10: Removed temporary directory /tmp/madlib_ZcWfxD9Vqm\n",
+ "PoolWorker-8: Removed temporary directory /tmp/madlib_XpsmAPt8MP\n",
+ "PoolWorker-7: Removed temporary directory /tmp/madlib_F1kcKRJNq0\n",
+ "PoolWorker-6: Removed temporary directory /tmp/madlib_4n6hb9xfIi\n",
+ "Done! Loaded 5139 images in 3.46985602379s\n",
+ "5 workers terminated.\n",
"MainProcess: Connected to madlib db.\n",
"Executing: CREATE TABLE train_gte5 (id SERIAL, x REAL[], y TEXT)\n",
"CREATE TABLE\n",
"Created table train_gte5 in madlib db\n",
"Spawning 5 workers...\n",
- "Initializing PoolWorker-41 [pid 36515]\n",
- "PoolWorker-41: Created temporary directory PoolWorker-41\n",
- "Initializing PoolWorker-42 [pid 36516]\n",
- "PoolWorker-42: Created temporary directory PoolWorker-42\n",
- "Initializing PoolWorker-43 [pid 36517]\n",
- "PoolWorker-43: Created temporary directory PoolWorker-43\n",
- "Initializing PoolWorker-44 [pid 36518]\n",
- "PoolWorker-44: Created temporary directory PoolWorker-44\n",
- "Initializing PoolWorker-45 [pid 36519]\n",
- "PoolWorker-45: Created temporary directory PoolWorker-45\n",
- "PoolWorker-41: Connected to madlib db.\n",
- "PoolWorker-42: Connected to madlib db.\n",
- "PoolWorker-43: Connected to madlib db.\n",
- "PoolWorker-44: Connected to madlib db.\n",
- "PoolWorker-45: Connected to madlib db.\n",
- "PoolWorker-41: Wrote 1000 images to
/tmp/madlib_4culpKfGHi/train_gte50000.tmp\n",
- "PoolWorker-42: Wrote 1000 images to
/tmp/madlib_V6Nqwp374Q/train_gte50000.tmp\n",
- "PoolWorker-43: Wrote 1000 images to
/tmp/madlib_fHsDrEhTBh/train_gte50000.tmp\n",
- "PoolWorker-44: Wrote 1000 images to
/tmp/madlib_CU2CNf1Yxz/train_gte50000.tmp\n",
- "PoolWorker-45: Wrote 1000 images to
/tmp/madlib_1ZkWBU7pyz/train_gte50000.tmp\n"
+ "Initializing PoolWorker-11 [pid 32462]\n",
+ "PoolWorker-11: Created temporary directory /tmp/madlib_F1VrR8QcQB\n",
+ "Initializing PoolWorker-12 [pid 32463]\n",
+ "PoolWorker-12: Created temporary directory /tmp/madlib_8dZVe2Lowc\n",
+ "Initializing PoolWorker-13 [pid 32464]\n",
+ "PoolWorker-13: Created temporary directory /tmp/madlib_V3jNZmDpr3\n",
+ "Initializing PoolWorker-14 [pid 32465]\n",
+ "PoolWorker-14: Created temporary directory /tmp/madlib_EtHaTZ0PtA\n",
+ "Initializing PoolWorker-15 [pid 32466]\n",
+ "PoolWorker-15: Created temporary directory /tmp/madlib_XtW6GudAE0\n",
+ "PoolWorker-11: Connected to madlib db.\n",
+ "PoolWorker-12: Connected to madlib db.\n",
+ "PoolWorker-13: Connected to madlib db.\n",
+ "PoolWorker-14: Connected to madlib db.\n",
+ "PoolWorker-15: Connected to madlib db.\n",
+ "PoolWorker-11: Wrote 1000 images to
/tmp/madlib_F1VrR8QcQB/train_gte50000.tmp\n",
+ "PoolWorker-12: Wrote 1000 images to
/tmp/madlib_8dZVe2Lowc/train_gte50000.tmp\n",
+ "PoolWorker-13: Wrote 1000 images to
/tmp/madlib_V3jNZmDpr3/train_gte50000.tmp\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
- "PoolWorker-43: Loaded 1000 images into train_gte5\n",
- "PoolWorker-44: Loaded 1000 images into train_gte5\n",
- "PoolWorker-41: Loaded 1000 images into train_gte5\n",
- "PoolWorker-43: Wrote 1000 images to
/tmp/madlib_fHsDrEhTBh/train_gte50001.tmp\n",
- "PoolWorker-44: Wrote 1000 images to
/tmp/madlib_CU2CNf1Yxz/train_gte50001.tmp\n",
- "PoolWorker-41: Wrote 1000 images to
/tmp/madlib_4culpKfGHi/train_gte50001.tmp\n",
- "PoolWorker-43: Loaded 1000 images into train_gte5\n",
- "PoolWorker-42: Loaded 1000 images into train_gte5\n",
- "PoolWorker-45: Loaded 1000 images into train_gte5\n",
- "PoolWorker-43: Wrote 1000 images to
/tmp/madlib_fHsDrEhTBh/train_gte50002.tmp\n",
- "PoolWorker-41: Loaded 1000 images into train_gte5\n",
- "PoolWorker-42: Wrote 1000 images to
/tmp/madlib_V6Nqwp374Q/train_gte50001.tmp\n",
- "PoolWorker-45: Wrote 1000 images to
/tmp/madlib_1ZkWBU7pyz/train_gte50001.tmp\n",
- "PoolWorker-41: Wrote 1000 images to
/tmp/madlib_4culpKfGHi/train_gte50002.tmp\n",
- "PoolWorker-43: Loaded 1000 images into train_gte5\n",
- "PoolWorker-44: Loaded 1000 images into train_gte5\n",
- "PoolWorker-43: Wrote 1000 images to
/tmp/madlib_fHsDrEhTBh/train_gte50003.tmp\n",
- "PoolWorker-44: Wrote 1000 images to
/tmp/madlib_CU2CNf1Yxz/train_gte50002.tmp\n",
- "PoolWorker-41: Loaded 1000 images into train_gte5\n",
- "PoolWorker-42: Loaded 1000 images into train_gte5\n",
- "PoolWorker-43: Loaded 1000 images into train_gte5\n",
- "PoolWorker-41: Wrote 1000 images to
/tmp/madlib_4culpKfGHi/train_gte50003.tmp\n",
- "PoolWorker-45: Loaded 1000 images into train_gte5\n",
- "PoolWorker-42: Wrote 1000 images to
/tmp/madlib_V6Nqwp374Q/train_gte50002.tmp\n",
- "PoolWorker-43: Wrote 1000 images to
/tmp/madlib_fHsDrEhTBh/train_gte50004.tmp\n",
- "PoolWorker-44: Loaded 1000 images into train_gte5\n",
- "PoolWorker-45: Wrote 1000 images to
/tmp/madlib_1ZkWBU7pyz/train_gte50002.tmp\n",
- "PoolWorker-41: Loaded 1000 images into train_gte5\n",
- "PoolWorker-43: Loaded 1000 images into train_gte5\n",
- "PoolWorker-44: Wrote 1000 images to
/tmp/madlib_CU2CNf1Yxz/train_gte50003.tmp\n",
- "PoolWorker-42: Loaded 1000 images into train_gte5\n",
- "PoolWorker-41: Wrote 1000 images to
/tmp/madlib_4culpKfGHi/train_gte50004.tmp\n",
- "PoolWorker-43: Wrote 1000 images to
/tmp/madlib_fHsDrEhTBh/train_gte50005.tmp\n",
- "PoolWorker-42: Wrote 1000 images to
/tmp/madlib_V6Nqwp374Q/train_gte50003.tmp\n",
- "PoolWorker-43: Loaded 1000 images into train_gte5\n",
- "PoolWorker-41: Loaded 1000 images into train_gte5\n",
- "PoolWorker-45: Loaded 1000 images into train_gte5\n",
- "PoolWorker-44: Loaded 1000 images into train_gte5\n",
- "PoolWorker-42: Loaded 1000 images into train_gte5\n",
- "PoolWorker-43: Wrote 1000 images to
/tmp/madlib_fHsDrEhTBh/train_gte50006.tmp\n",
- "PoolWorker-41: Wrote 1000 images to
/tmp/madlib_4culpKfGHi/train_gte50005.tmp\n",
- "PoolWorker-45: Wrote 1000 images to
/tmp/madlib_1ZkWBU7pyz/train_gte50003.tmp\n",
- "PoolWorker-44: Wrote 1000 images to
/tmp/madlib_CU2CNf1Yxz/train_gte50004.tmp\n",
- "PoolWorker-42: Wrote 1000 images to
/tmp/madlib_V6Nqwp374Q/train_gte50004.tmp\n",
- "PoolWorker-43: Loaded 1000 images into train_gte5\n",
- "PoolWorker-41: Loaded 1000 images into train_gte5\n",
- "PoolWorker-43: Wrote 1000 images to
/tmp/madlib_fHsDrEhTBh/train_gte50007.tmp\n",
- "PoolWorker-42: Loaded 1000 images into train_gte5\n",
- "PoolWorker-44: Loaded 1000 images into train_gte5\n",
- "PoolWorker-45: Loaded 1000 images into train_gte5\n",
- "PoolWorker-42: Wrote 404 images to
/tmp/madlib_V6Nqwp374Q/train_gte50005.tmp\n",
- "PoolWorker-44: Wrote 1000 images to
/tmp/madlib_CU2CNf1Yxz/train_gte50005.tmp\n",
- "PoolWorker-42: Loaded 404 images into train_gte5\n",
- "PoolWorker-43: Loaded 1000 images into train_gte5\n",
- "PoolWorker-44: Loaded 1000 images into train_gte5\n",
- "PoolWorker-45: Removed temporary directory PoolWorker-45\n",
- "PoolWorker-44: Removed temporary directory PoolWorker-44\n",
- "PoolWorker-42: Removed temporary directory PoolWorker-42\n",
- "PoolWorker-41: Removed temporary directory PoolWorker-41\n",
- "PoolWorker-43: Removed temporary directory PoolWorker-43\n",
- "Done! Loaded 29404 images in 12.6191689968s\n",
+ "PoolWorker-15: Wrote 1000 images to
/tmp/madlib_XtW6GudAE0/train_gte50000.tmp\n",
+ "PoolWorker-14: Wrote 1000 images to
/tmp/madlib_EtHaTZ0PtA/train_gte50000.tmp\n",
+ "PoolWorker-15: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-14: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-15: Wrote 1000 images to
/tmp/madlib_XtW6GudAE0/train_gte50001.tmp\n",
+ "PoolWorker-14: Wrote 1000 images to
/tmp/madlib_EtHaTZ0PtA/train_gte50001.tmp\n",
+ "PoolWorker-15: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-15: Wrote 1000 images to
/tmp/madlib_XtW6GudAE0/train_gte50002.tmp\n",
+ "PoolWorker-14: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-14: Wrote 1000 images to
/tmp/madlib_EtHaTZ0PtA/train_gte50002.tmp\n",
+ "PoolWorker-15: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-11: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-12: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-14: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-15: Wrote 1000 images to
/tmp/madlib_XtW6GudAE0/train_gte50003.tmp\n",
+ "PoolWorker-11: Wrote 1000 images to
/tmp/madlib_F1VrR8QcQB/train_gte50001.tmp\n",
+ "PoolWorker-13: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-12: Wrote 1000 images to
/tmp/madlib_8dZVe2Lowc/train_gte50001.tmp\n",
+ "PoolWorker-14: Wrote 1000 images to
/tmp/madlib_EtHaTZ0PtA/train_gte50003.tmp\n",
+ "PoolWorker-13: Wrote 1000 images to
/tmp/madlib_V3jNZmDpr3/train_gte50001.tmp\n",
+ "PoolWorker-14: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-15: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-14: Wrote 1000 images to
/tmp/madlib_EtHaTZ0PtA/train_gte50004.tmp\n",
+ "PoolWorker-15: Wrote 1000 images to
/tmp/madlib_XtW6GudAE0/train_gte50004.tmp\n",
+ "PoolWorker-12: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-13: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-14: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-12: Wrote 1000 images to
/tmp/madlib_8dZVe2Lowc/train_gte50002.tmp\n",
+ "PoolWorker-13: Wrote 1000 images to
/tmp/madlib_V3jNZmDpr3/train_gte50002.tmp\n",
+ "PoolWorker-14: Wrote 1000 images to
/tmp/madlib_EtHaTZ0PtA/train_gte50005.tmp\n",
+ "PoolWorker-11: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-11: Wrote 1000 images to
/tmp/madlib_F1VrR8QcQB/train_gte50002.tmp\n",
+ "PoolWorker-13: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-14: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-13: Wrote 1000 images to
/tmp/madlib_V3jNZmDpr3/train_gte50003.tmp\n",
+ "PoolWorker-14: Wrote 1000 images to
/tmp/madlib_EtHaTZ0PtA/train_gte50006.tmp\n",
+ "PoolWorker-11: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-12: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-11: Wrote 1000 images to
/tmp/madlib_F1VrR8QcQB/train_gte50003.tmp\n",
+ "PoolWorker-12: Wrote 1000 images to
/tmp/madlib_8dZVe2Lowc/train_gte50003.tmp\n",
+ "PoolWorker-15: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-13: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-14: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-15: Wrote 1000 images to
/tmp/madlib_XtW6GudAE0/train_gte50005.tmp\n",
+ "PoolWorker-13: Wrote 1000 images to
/tmp/madlib_V3jNZmDpr3/train_gte50004.tmp\n",
+ "PoolWorker-14: Wrote 1000 images to
/tmp/madlib_EtHaTZ0PtA/train_gte50007.tmp\n",
+ "PoolWorker-11: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-14: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-13: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-12: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-11: Wrote 1000 images to
/tmp/madlib_F1VrR8QcQB/train_gte50004.tmp\n",
+ "PoolWorker-13: Wrote 1000 images to
/tmp/madlib_V3jNZmDpr3/train_gte50005.tmp\n",
+ "PoolWorker-15: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-11: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-13: Loaded 1000 images into train_gte5\n",
+ "PoolWorker-11: Wrote 404 images to
/tmp/madlib_F1VrR8QcQB/train_gte50005.tmp\n",
+ "PoolWorker-11: Loaded 404 images into train_gte5\n",
+ "PoolWorker-13: Removed temporary directory /tmp/madlib_V3jNZmDpr3\n",
+ "PoolWorker-14: Removed temporary directory /tmp/madlib_EtHaTZ0PtA\n",
+ "PoolWorker-12: Removed temporary directory /tmp/madlib_8dZVe2Lowc\n",
+ "PoolWorker-15: Removed temporary directory /tmp/madlib_XtW6GudAE0\n",
+ "PoolWorker-11: Removed temporary directory /tmp/madlib_F1VrR8QcQB\n",
+ "Done! Loaded 29404 images in 18.3531939983s\n",
+ "5 workers terminated.\n",
"MainProcess: Connected to madlib db.\n",
"Executing: CREATE TABLE test_gte5 (id SERIAL, x REAL[], y TEXT)\n",
"CREATE TABLE\n",
"Created table test_gte5 in madlib db\n",
"Spawning 5 workers...\n",
- "Initializing PoolWorker-46 [pid 36521]\n",
- "PoolWorker-46: Created temporary directory PoolWorker-46\n",
- "Initializing PoolWorker-47 [pid 36522]\n",
- "PoolWorker-47: Created temporary directory PoolWorker-47\n",
- "Initializing PoolWorker-48 [pid 36523]\n",
- "PoolWorker-50: Created temporary directory PoolWorker-50\n",
- "PoolWorker-48: Created temporary directory PoolWorker-48\n",
- "Initializing PoolWorker-49 [pid 36524]\n",
- "PoolWorker-49: Created temporary directory PoolWorker-49\n",
- "Initializing PoolWorker-50 [pid 36525]\n",
- "PoolWorker-46: Connected to madlib db.\n",
- "PoolWorker-47: Connected to madlib db.\n",
- "PoolWorker-48: Connected to madlib db.\n",
- "PoolWorker-49: Connected to madlib db.\n",
- "PoolWorker-50: Connected to madlib db.\n",
- "PoolWorker-46: Wrote 1000 images to
/tmp/madlib_DTfIW4fes1/test_gte50000.tmp\n",
- "PoolWorker-50: Wrote 861 images to
/tmp/madlib_dkZDQsdCfI/test_gte50000.tmp\n",
- "PoolWorker-48: Wrote 1000 images to
/tmp/madlib_eLa5ZPTJoU/test_gte50000.tmp\n",
- "PoolWorker-47: Wrote 1000 images to
/tmp/madlib_agBlKzPnjs/test_gte50000.tmp\n",
- "PoolWorker-49: Wrote 1000 images to
/tmp/madlib_89sgwkcIn8/test_gte50000.tmp\n",
- "PoolWorker-50: Loaded 861 images into test_gte5\n",
- "PoolWorker-48: Loaded 1000 images into test_gte5\n",
- "PoolWorker-47: Loaded 1000 images into test_gte5\n",
- "PoolWorker-49: Loaded 1000 images into test_gte5\n",
- "PoolWorker-46: Loaded 1000 images into test_gte5\n",
- "PoolWorker-50: Removed temporary directory PoolWorker-50\n",
- "PoolWorker-49: Removed temporary directory PoolWorker-49\n",
- "PoolWorker-46: Removed temporary directory PoolWorker-46\n",
- "PoolWorker-47: Removed temporary directory PoolWorker-47\n",
- "PoolWorker-48: Removed temporary directory PoolWorker-48\n",
- "Done! Loaded 4861 images in 3.05431604385s\n"
+ "Initializing PoolWorker-16 [pid 32468]\n",
+ "PoolWorker-16: Created temporary directory /tmp/madlib_yQApveqAcA\n",
+ "Initializing PoolWorker-17 [pid 32469]\n",
+ "PoolWorker-17: Created temporary directory /tmp/madlib_2LKVOaFY1A\n",
+ "Initializing PoolWorker-18 [pid 32470]\n",
+ "PoolWorker-18: Created temporary directory /tmp/madlib_qicxNf38wn\n",
+ "Initializing PoolWorker-19 [pid 32471]\n",
+ "PoolWorker-19: Created temporary directory /tmp/madlib_cD0CFY0uOR\n",
+ "Initializing PoolWorker-20 [pid 32472]\n",
+ "PoolWorker-20: Created temporary directory /tmp/madlib_yNG8SjRSWz\n",
+ "PoolWorker-16: Connected to madlib db.\n",
+ "PoolWorker-17: Connected to madlib db.\n",
+ "PoolWorker-18: Connected to madlib db.\n",
+ "PoolWorker-19: Connected to madlib db.\n",
+ "PoolWorker-20: Connected to madlib db.\n",
+ "PoolWorker-20: Wrote 861 images to
/tmp/madlib_yNG8SjRSWz/test_gte50000.tmp\n",
+ "PoolWorker-17: Wrote 1000 images to
/tmp/madlib_2LKVOaFY1A/test_gte50000.tmp\n",
+ "PoolWorker-18: Wrote 1000 images to
/tmp/madlib_qicxNf38wn/test_gte50000.tmp\n",
+ "PoolWorker-16: Wrote 1000 images to
/tmp/madlib_yQApveqAcA/test_gte50000.tmp\n",
+ "PoolWorker-19: Wrote 1000 images to
/tmp/madlib_cD0CFY0uOR/test_gte50000.tmp\n",
+ "PoolWorker-17: Loaded 1000 images into test_gte5\n",
+ "PoolWorker-18: Loaded 1000 images into test_gte5\n",
+ "PoolWorker-19: Loaded 1000 images into test_gte5\n",
+ "PoolWorker-16: Loaded 1000 images into test_gte5\n",
+ "PoolWorker-20: Loaded 861 images into test_gte5\n",
+ "PoolWorker-17: Removed temporary directory /tmp/madlib_2LKVOaFY1A\n",
+ "PoolWorker-16: Removed temporary directory /tmp/madlib_yQApveqAcA\n",
+ "PoolWorker-18: Removed temporary directory /tmp/madlib_qicxNf38wn\n",
+ "PoolWorker-20: Removed temporary directory /tmp/madlib_yNG8SjRSWz\n",
+ "PoolWorker-19: Removed temporary directory /tmp/madlib_cD0CFY0uOR\n",
+ "Done! Loaded 4861 images in 3.39312386513s\n",
+ "5 workers terminated.\n"
]
}
],
@@ -552,10 +556,10 @@
"%sql DROP TABLE IF EXISTS train_lt5, test_lt5, train_gte5, test_gte5\n",
"\n",
"# Save images to temporary directories and load into database\n",
- "iloader.load_np_array_to_table(x_train_lt5, y_train_lt5, 'train_lt5',
append=False, img_names=None)\n",
- "iloader.load_np_array_to_table(x_test_lt5, y_test_lt5, 'test_lt5',
append=False, img_names=None)\n",
- "iloader.load_np_array_to_table(x_train_gte5, y_train_gte5, 'train_gte5',
append=False, img_names=None)\n",
- "iloader.load_np_array_to_table(x_test_gte5, y_test_gte5, 'test_gte5',
append=False, img_names=None)"
+ "iloader.load_dataset_from_np(x_train_lt5, y_train_lt5, 'train_lt5',
append=False, no_temp_files=False)\n",
+ "iloader.load_dataset_from_np(x_test_lt5, y_test_lt5, 'test_lt5',
append=False, no_temp_files=False)\n",
+ "iloader.load_dataset_from_np(x_train_gte5, y_train_gte5, 'train_gte5',
append=False, no_temp_files=False)\n",
+ "iloader.load_dataset_from_np(x_test_gte5, y_test_gte5, 'test_gte5',
append=False, no_temp_files=False)"
]
},
{
@@ -608,14 +612,14 @@
" <td>x</td>\n",
" <td>text</td>\n",
" <td>[u'0', u'1', u'2', u'3', u'4']</td>\n",
- " <td>15298</td>\n",
+ " <td>1000</td>\n",
" <td>255.0</td>\n",
" <td>5</td>\n",
" </tr>\n",
"</table>"
],
"text/plain": [
- "[(u'train_lt5', u'train_lt5_packed', u'y', u'x', u'text', [u'0', u'1',
u'2', u'3', u'4'], 15298, 255.0, 5)]"
+ "[(u'train_lt5', u'train_lt5_packed', u'y', u'x', u'text', [u'0', u'1',
u'2', u'3', u'4'], 1000, 255.0, 5)]"
]
},
"execution_count": 11,
@@ -631,7 +635,7 @@
" 'train_lt5_packed', --
Output table\n",
" 'y', --
Dependent variable\n",
" 'x', --
Independent variable\n",
- " NULL, --
Buffer size\n",
+ " 1000, --
Buffer size\n",
" 255 --
Normalizing constant\n",
" );\n",
"\n",
@@ -719,7 +723,7 @@
},
{
"cell_type": "code",
- "execution_count": 8,
+ "execution_count": 13,
"metadata": {},
"outputs": [
{
@@ -753,17 +757,17 @@
" <td>x</td>\n",
" <td>text</td>\n",
" <td>[u'0', u'1', u'2', u'3', u'4']</td>\n",
- " <td>14702</td>\n",
+ " <td>1000</td>\n",
" <td>255.0</td>\n",
" <td>5</td>\n",
" </tr>\n",
"</table>"
],
"text/plain": [
- "[(u'train_gte5', u'train_gte5_packed', u'y', u'x', u'text', [u'0',
u'1', u'2', u'3', u'4'], 14702, 255.0, 5)]"
+ "[(u'train_gte5', u'train_gte5_packed', u'y', u'x', u'text', [u'0',
u'1', u'2', u'3', u'4'], 1000, 255.0, 5)]"
]
},
- "execution_count": 8,
+ "execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
@@ -776,7 +780,7 @@
" 'train_gte5_packed', --
Output table\n",
" 'y', --
Dependent variable\n",
" 'x', --
Independent variable\n",
- " NULL, --
Buffer size\n",
+ " 1000, --
Buffer size\n",
" 255 --
Normalizing constant\n",
" );\n",
"\n",
@@ -792,7 +796,7 @@
},
{
"cell_type": "code",
- "execution_count": 17,
+ "execution_count": 14,
"metadata": {},
"outputs": [
{
@@ -836,7 +840,7 @@
"[(u'test_gte5', u'test_gte5_packed', u'y', u'x', u'text', [u'0', u'1',
u'2', u'3', u'4'], 2431, 255.0, 5)]"
]
},
- "execution_count": 17,
+ "execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
@@ -867,7 +871,7 @@
},
{
"cell_type": "code",
- "execution_count": 19,
+ "execution_count": 15,
"metadata": {},
"outputs": [
{
@@ -945,7 +949,7 @@
},
{
"cell_type": "code",
- "execution_count": 20,
+ "execution_count": 16,
"metadata": {},
"outputs": [
{
@@ -974,7 +978,7 @@
"[(1, u'feature + classification layers trainable')]"
]
},
- "execution_count": 20,
+ "execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
@@ -1002,7 +1006,7 @@
},
{
"cell_type": "code",
- "execution_count": 21,
+ "execution_count": 17,
"metadata": {},
"outputs": [
{
@@ -1060,7 +1064,7 @@
},
{
"cell_type": "code",
- "execution_count": 22,
+ "execution_count": 18,
"metadata": {},
"outputs": [
{
@@ -1093,7 +1097,7 @@
" (2, u'only classification layers trainable')]"
]
},
- "execution_count": 22,
+ "execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
diff --git a/community-artifacts/Deep-learning/Madlib Image Loader Demo.ipynb
b/community-artifacts/Deep-learning/Madlib Image Loader Demo.ipynb
deleted file mode 100644
index aba3a1f..0000000
--- a/community-artifacts/Deep-learning/Madlib Image Loader Demo.ipynb
+++ /dev/null
@@ -1,488 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### Madlib Image Loader requires psycopg2. If you don't already have it
installed, run:\n",
- "\n",
- "`pip install psycopg2-binary`"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 1,
- "metadata": {},
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "Using TensorFlow backend.\n"
- ]
- }
- ],
- "source": [
- "import sys\n",
- "import os\n",
- "from keras.datasets import cifar10, cifar100, mnist, fashion_mnist, imdb,
reuters"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 2,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Add community-artifacts to PYTHON_PATH\n",
- " # TIP: You can skip this cell if madlib_image_loader.py is already
in your current directory\n",
- "\n",
- "home = %env HOME\n",
- " # TIP: Change home,'workspace' to wherever you have cloned
madlib-site repo\n",
- "madlib_site_dir =
os.path.join(home,'workspace','madlib-site','community-artifacts',
'Deep-learning')\n",
- "sys.path.append(madlib_site_dir)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 3,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Import image loader module\n",
- "from madlib_image_loader import ImageLoader, DbCredentials"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 4,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Specify database credentials, for connecting to db\n",
- "db_creds = DbCredentials(db_name='madlib',\n",
- " user='pivotal',\n",
- " host='localhost',\n",
- " port='15432',\n",
- " password='')"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 5,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Load dataset into np array\n",
- "train_data, _ = cifar10.load_data()\n",
- "data_x, data_y = train_data"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 6,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Initialize ImageLoader (increase num_workers to run faster)\n",
- "iloader = ImageLoader(num_workers=8, db_creds=db_creds)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 7,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "MainProcess: Connected to madlib db.\n",
- "Executing: CREATE TABLE cifar_10_test (id SERIAL, x REAL[], y TEXT)\n",
- "CREATE TABLE\n",
- "Created table cifar_10_test in madlib db\n",
- "Spawning 8 workers...\n",
- "Initializing PoolWorker-1 [pid 13756]\n",
- "PoolWorker-1: Created temporary directory /tmp/madlib_l1UVaZIxxr\n",
- "Initializing PoolWorker-2 [pid 13757]\n",
- "PoolWorker-2: Created temporary directory /tmp/madlib_AQTX6xmhMK\n",
- "Initializing PoolWorker-3 [pid 13758]\n",
- "PoolWorker-3: Created temporary directory /tmp/madlib_7QOFnBn8jB\n",
- "Initializing PoolWorker-4 [pid 13759]\n",
- "PoolWorker-4: Created temporary directory /tmp/madlib_1AgOTkwfPn\n",
- "Initializing PoolWorker-5 [pid 13761]\n",
- "PoolWorker-1: Connected to madlib db.\n",
- "Initializing PoolWorker-6 [pid 13762]\n",
- "PoolWorker-5: Created temporary directory /tmp/madlib_RzYo51sdaR\n",
- "Initializing PoolWorker-7 [pid 13764]\n",
- "PoolWorker-6: Created temporary directory /tmp/madlib_uip4DCxC9m\n",
- "PoolWorker-2: Connected to madlib db.\n",
- "PoolWorker-7: Created temporary directory /tmp/madlib_8Urhpm5TkR\n",
- "Initializing PoolWorker-8 [pid 13766]\n",
- "PoolWorker-3: Connected to madlib db.\n",
- "PoolWorker-8: Created temporary directory /tmp/madlib_SzPIAtCqyb\n",
- "PoolWorker-4: Connected to madlib db.\n",
- "PoolWorker-5: Connected to madlib db.\n",
- "PoolWorker-6: Connected to madlib db.\n",
- "PoolWorker-7: Connected to madlib db.\n",
- "PoolWorker-8: Connected to madlib db.\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_l1UVaZIxxr/cifar_10_test0000.tmp\n",
- "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_AQTX6xmhMK/cifar_10_test0000.tmp\n",
- "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_7QOFnBn8jB/cifar_10_test0000.tmp\n",
- "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_1AgOTkwfPn/cifar_10_test0000.tmp\n",
- "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_RzYo51sdaR/cifar_10_test0000.tmp\n",
- "PoolWorker-6: Wrote 1000 images to
/tmp/madlib_uip4DCxC9m/cifar_10_test0000.tmp\n",
- "PoolWorker-7: Wrote 1000 images to
/tmp/madlib_8Urhpm5TkR/cifar_10_test0000.tmp\n",
- "PoolWorker-8: Wrote 1000 images to
/tmp/madlib_SzPIAtCqyb/cifar_10_test0000.tmp\n",
- "PoolWorker-1: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-2: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-4: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-3: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-6: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-7: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-8: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-5: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_l1UVaZIxxr/cifar_10_test0001.tmp\n",
- "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_AQTX6xmhMK/cifar_10_test0001.tmp\n",
- "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_1AgOTkwfPn/cifar_10_test0001.tmp\n",
- "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_7QOFnBn8jB/cifar_10_test0001.tmp\n",
- "PoolWorker-1: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-6: Wrote 1000 images to
/tmp/madlib_uip4DCxC9m/cifar_10_test0001.tmp\n",
- "PoolWorker-7: Wrote 1000 images to
/tmp/madlib_8Urhpm5TkR/cifar_10_test0001.tmp\n",
- "PoolWorker-8: Wrote 1000 images to
/tmp/madlib_SzPIAtCqyb/cifar_10_test0001.tmp\n",
- "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_RzYo51sdaR/cifar_10_test0001.tmp\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_l1UVaZIxxr/cifar_10_test0002.tmp\n",
- "PoolWorker-2: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-4: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-3: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-6: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-7: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-5: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-8: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-1: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_AQTX6xmhMK/cifar_10_test0002.tmp\n",
- "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_1AgOTkwfPn/cifar_10_test0002.tmp\n",
- "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_7QOFnBn8jB/cifar_10_test0002.tmp\n",
- "PoolWorker-6: Wrote 1000 images to
/tmp/madlib_uip4DCxC9m/cifar_10_test0002.tmp\n",
- "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_RzYo51sdaR/cifar_10_test0002.tmp\n",
- "PoolWorker-7: Wrote 1000 images to
/tmp/madlib_8Urhpm5TkR/cifar_10_test0002.tmp\n",
- "PoolWorker-8: Wrote 1000 images to
/tmp/madlib_SzPIAtCqyb/cifar_10_test0002.tmp\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_l1UVaZIxxr/cifar_10_test0003.tmp\n",
- "PoolWorker-2: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-4: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-3: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-8: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-6: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-5: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-7: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-1: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_AQTX6xmhMK/cifar_10_test0003.tmp\n",
- "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_1AgOTkwfPn/cifar_10_test0003.tmp\n",
- "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_7QOFnBn8jB/cifar_10_test0003.tmp\n",
- "PoolWorker-8: Wrote 1000 images to
/tmp/madlib_SzPIAtCqyb/cifar_10_test0003.tmp\n",
- "PoolWorker-2: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-6: Wrote 1000 images to
/tmp/madlib_uip4DCxC9m/cifar_10_test0003.tmp\n",
- "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_RzYo51sdaR/cifar_10_test0003.tmp\n",
- "PoolWorker-7: Wrote 1000 images to
/tmp/madlib_8Urhpm5TkR/cifar_10_test0003.tmp\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_l1UVaZIxxr/cifar_10_test0004.tmp\n",
- "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_AQTX6xmhMK/cifar_10_test0004.tmp\n",
- "PoolWorker-4: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-3: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-6: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-7: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-8: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-5: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-1: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-2: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_1AgOTkwfPn/cifar_10_test0004.tmp\n",
- "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_7QOFnBn8jB/cifar_10_test0004.tmp\n",
- "PoolWorker-6: Wrote 1000 images to
/tmp/madlib_uip4DCxC9m/cifar_10_test0004.tmp\n",
- "PoolWorker-7: Wrote 1000 images to
/tmp/madlib_8Urhpm5TkR/cifar_10_test0004.tmp\n",
- "PoolWorker-8: Wrote 1000 images to
/tmp/madlib_SzPIAtCqyb/cifar_10_test0004.tmp\n",
- "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_RzYo51sdaR/cifar_10_test0004.tmp\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_l1UVaZIxxr/cifar_10_test0005.tmp\n",
- "PoolWorker-2: Wrote 1000 images to
/tmp/madlib_AQTX6xmhMK/cifar_10_test0005.tmp\n",
- "PoolWorker-4: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-4: Wrote 1000 images to
/tmp/madlib_1AgOTkwfPn/cifar_10_test0005.tmp\n",
- "PoolWorker-3: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-6: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-7: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-5: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-8: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-1: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-2: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-4: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-3: Wrote 1000 images to
/tmp/madlib_7QOFnBn8jB/cifar_10_test0005.tmp\n",
- "PoolWorker-6: Wrote 1000 images to
/tmp/madlib_uip4DCxC9m/cifar_10_test0005.tmp\n",
- "PoolWorker-7: Wrote 1000 images to
/tmp/madlib_8Urhpm5TkR/cifar_10_test0005.tmp\n",
- "PoolWorker-8: Wrote 1000 images to
/tmp/madlib_SzPIAtCqyb/cifar_10_test0005.tmp\n",
- "PoolWorker-5: Wrote 1000 images to
/tmp/madlib_RzYo51sdaR/cifar_10_test0005.tmp\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_l1UVaZIxxr/cifar_10_test0006.tmp\n",
- "PoolWorker-3: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-6: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-5: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-7: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-8: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-1: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-1: Wrote 1000 images to
/tmp/madlib_l1UVaZIxxr/cifar_10_test0007.tmp\n",
- "PoolWorker-1: Loaded 1000 images into cifar_10_test\n",
- "PoolWorker-4: Removed temporary directory /tmp/madlib_1AgOTkwfPn\n",
- "PoolWorker-3: Removed temporary directory /tmp/madlib_7QOFnBn8jB\n",
- "PoolWorker-7: Removed temporary directory /tmp/madlib_8Urhpm5TkR\n",
- "PoolWorker-5: Removed temporary directory /tmp/madlib_RzYo51sdaR\n",
- "PoolWorker-2: Removed temporary directory /tmp/madlib_AQTX6xmhMK\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "PoolWorker-1: Removed temporary directory /tmp/madlib_l1UVaZIxxr\n",
- "PoolWorker-6: Removed temporary directory /tmp/madlib_uip4DCxC9m\n",
- "PoolWorker-8: Removed temporary directory /tmp/madlib_SzPIAtCqyb\n",
- "Done! Loaded 50000 images in 52.8276519775s\n",
- "8 workers terminated.\n"
- ]
- }
- ],
- "source": [
- "# Save images to temporary directories and load into database\n",
- "iloader.load_dataset_from_np(data_x, data_y, 'cifar_10_test',
append=False)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 8,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Same thing, but without writing out any temporary files; everything
handled in-memory.\n",
- "# Should run about twice as fast, but not working yet.\n",
- "\n",
- "#iloader.ROWS_PER_FILE = 1000 # Try adjusting this downward, if running
low on memory\n",
- "#iloader.load_dataset_from_disk(data_x, data_y, 'cifar_10_test2',
append=True, no_temp_files=True)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 7,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Try loading a different dataset\n",
- "train_data, _ = fashion_mnist.load_data()\n",
- "data_x, data_y = train_data"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 10,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "MainProcess: Connected to madlib db.\n",
- "Appending to table fashion_mnist_test in madlib db\n",
- "Spawning 8 workers...\n",
- "Initializing PoolWorker-9 [pid 14871]\n",
- "PoolWorker-9: Created temporary directory /tmp/madlib_1j4HTsicJ8\n",
- "Initializing PoolWorker-10 [pid 14872]\n",
- "PoolWorker-10: Created temporary directory /tmp/madlib_XUO9OeFCRp\n",
- "Initializing PoolWorker-11 [pid 14873]\n",
- "PoolWorker-11: Created temporary directory /tmp/madlib_XXoZi8qgE1\n",
- "Initializing PoolWorker-12 [pid 14874]\n",
- "PoolWorker-12: Created temporary directory /tmp/madlib_hc0Qt4WpIv\n",
- "Initializing PoolWorker-13 [pid 14875]\n",
- "Initializing PoolWorker-14 [pid 14877]\n",
- "PoolWorker-13: Created temporary directory /tmp/madlib_2JwMfqwlOC\n",
- "PoolWorker-9: Connected to madlib db.\n",
- "PoolWorker-14: Created temporary directory /tmp/madlib_0kkBdCmGO4\n",
- "Initializing PoolWorker-15 [pid 14879]\n",
- "PoolWorker-10: Connected to madlib db.\n",
- "Initializing PoolWorker-16 [pid 14880]\n",
- "PoolWorker-15: Created temporary directory /tmp/madlib_NGqaV8pYyu\n",
- "PoolWorker-16: Created temporary directory /tmp/madlib_mkO0vnSoJ8\n",
- "PoolWorker-11: Connected to madlib db.\n",
- "PoolWorker-12: Connected to madlib db.\n",
- "PoolWorker-14: Connected to madlib db.\n",
- "PoolWorker-13: Connected to madlib db.\n",
- "PoolWorker-15: Connected to madlib db.\n",
- "PoolWorker-16: Connected to madlib db.\n",
- "PoolWorker-9: Wrote 1000 images to
/tmp/madlib_1j4HTsicJ8/fashion_mnist_test0000.tmp\n",
- "PoolWorker-10: Wrote 1000 images to
/tmp/madlib_XUO9OeFCRp/fashion_mnist_test0000.tmp\n",
- "PoolWorker-11: Wrote 1000 images to
/tmp/madlib_XXoZi8qgE1/fashion_mnist_test0000.tmp\n",
- "PoolWorker-12: Wrote 1000 images to
/tmp/madlib_hc0Qt4WpIv/fashion_mnist_test0000.tmp\n",
- "PoolWorker-14: Wrote 1000 images to
/tmp/madlib_0kkBdCmGO4/fashion_mnist_test0000.tmp\n",
- "PoolWorker-13: Wrote 1000 images to
/tmp/madlib_2JwMfqwlOC/fashion_mnist_test0000.tmp\n",
- "PoolWorker-15: Wrote 1000 images to
/tmp/madlib_NGqaV8pYyu/fashion_mnist_test0000.tmp\n",
- "PoolWorker-16: Wrote 1000 images to
/tmp/madlib_mkO0vnSoJ8/fashion_mnist_test0000.tmp\n",
- "PoolWorker-9: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-11: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-12: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-15: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-9: Wrote 1000 images to
/tmp/madlib_1j4HTsicJ8/fashion_mnist_test0001.tmp\n",
- "PoolWorker-11: Wrote 1000 images to
/tmp/madlib_XXoZi8qgE1/fashion_mnist_test0001.tmp\n",
- "PoolWorker-10: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-12: Wrote 1000 images to
/tmp/madlib_hc0Qt4WpIv/fashion_mnist_test0001.tmp\n",
- "PoolWorker-13: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-15: Wrote 1000 images to
/tmp/madlib_NGqaV8pYyu/fashion_mnist_test0001.tmp\n",
- "PoolWorker-14: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-16: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-10: Wrote 1000 images to
/tmp/madlib_XUO9OeFCRp/fashion_mnist_test0001.tmp\n",
- "PoolWorker-13: Wrote 1000 images to
/tmp/madlib_2JwMfqwlOC/fashion_mnist_test0001.tmp\n",
- "PoolWorker-14: Wrote 1000 images to
/tmp/madlib_0kkBdCmGO4/fashion_mnist_test0001.tmp\n",
- "PoolWorker-16: Wrote 1000 images to
/tmp/madlib_mkO0vnSoJ8/fashion_mnist_test0001.tmp\n",
- "PoolWorker-9: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-11: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-12: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-15: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-9: Wrote 1000 images to
/tmp/madlib_1j4HTsicJ8/fashion_mnist_test0002.tmp\n",
- "PoolWorker-11: Wrote 1000 images to
/tmp/madlib_XXoZi8qgE1/fashion_mnist_test0002.tmp\n",
- "PoolWorker-12: Wrote 1000 images to
/tmp/madlib_hc0Qt4WpIv/fashion_mnist_test0002.tmp\n",
- "PoolWorker-14: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-15: Wrote 1000 images to
/tmp/madlib_NGqaV8pYyu/fashion_mnist_test0002.tmp\n",
- "PoolWorker-13: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-10: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-16: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-14: Wrote 1000 images to
/tmp/madlib_0kkBdCmGO4/fashion_mnist_test0002.tmp\n",
- "PoolWorker-13: Wrote 1000 images to
/tmp/madlib_2JwMfqwlOC/fashion_mnist_test0002.tmp\n",
- "PoolWorker-10: Wrote 1000 images to
/tmp/madlib_XUO9OeFCRp/fashion_mnist_test0002.tmp\n",
- "PoolWorker-16: Wrote 1000 images to
/tmp/madlib_mkO0vnSoJ8/fashion_mnist_test0002.tmp\n",
- "PoolWorker-9: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-11: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-12: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-13: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-15: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-10: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-11: Wrote 1000 images to
/tmp/madlib_XXoZi8qgE1/fashion_mnist_test0003.tmp\n",
- "PoolWorker-12: Wrote 1000 images to
/tmp/madlib_hc0Qt4WpIv/fashion_mnist_test0003.tmp\n",
- "PoolWorker-9: Wrote 1000 images to
/tmp/madlib_1j4HTsicJ8/fashion_mnist_test0003.tmp\n",
- "PoolWorker-13: Wrote 1000 images to
/tmp/madlib_2JwMfqwlOC/fashion_mnist_test0003.tmp\n",
- "PoolWorker-15: Wrote 1000 images to
/tmp/madlib_NGqaV8pYyu/fashion_mnist_test0003.tmp\n",
- "PoolWorker-14: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-16: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-10: Wrote 1000 images to
/tmp/madlib_XUO9OeFCRp/fashion_mnist_test0003.tmp\n",
- "PoolWorker-16: Wrote 1000 images to
/tmp/madlib_mkO0vnSoJ8/fashion_mnist_test0003.tmp\n",
- "PoolWorker-14: Wrote 1000 images to
/tmp/madlib_0kkBdCmGO4/fashion_mnist_test0003.tmp\n",
- "PoolWorker-9: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-11: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-15: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-12: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-10: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-9: Wrote 1000 images to
/tmp/madlib_1j4HTsicJ8/fashion_mnist_test0004.tmp\n",
- "PoolWorker-14: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-11: Wrote 1000 images to
/tmp/madlib_XXoZi8qgE1/fashion_mnist_test0004.tmp\n",
- "PoolWorker-12: Wrote 1000 images to
/tmp/madlib_hc0Qt4WpIv/fashion_mnist_test0004.tmp\n",
- "PoolWorker-15: Wrote 1000 images to
/tmp/madlib_NGqaV8pYyu/fashion_mnist_test0004.tmp\n",
- "PoolWorker-13: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-16: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-10: Wrote 1000 images to
/tmp/madlib_XUO9OeFCRp/fashion_mnist_test0004.tmp\n",
- "PoolWorker-14: Wrote 1000 images to
/tmp/madlib_0kkBdCmGO4/fashion_mnist_test0004.tmp\n",
- "PoolWorker-13: Wrote 1000 images to
/tmp/madlib_2JwMfqwlOC/fashion_mnist_test0004.tmp\n",
- "PoolWorker-16: Wrote 1000 images to
/tmp/madlib_mkO0vnSoJ8/fashion_mnist_test0004.tmp\n",
- "PoolWorker-9: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-11: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-14: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-12: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-13: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-9: Wrote 1000 images to
/tmp/madlib_1j4HTsicJ8/fashion_mnist_test0005.tmp\n",
- "PoolWorker-10: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-11: Wrote 1000 images to
/tmp/madlib_XXoZi8qgE1/fashion_mnist_test0005.tmp\n",
- "PoolWorker-15: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-14: Wrote 1000 images to
/tmp/madlib_0kkBdCmGO4/fashion_mnist_test0005.tmp\n",
- "PoolWorker-12: Wrote 1000 images to
/tmp/madlib_hc0Qt4WpIv/fashion_mnist_test0005.tmp\n",
- "PoolWorker-13: Wrote 1000 images to
/tmp/madlib_2JwMfqwlOC/fashion_mnist_test0005.tmp\n",
- "PoolWorker-10: Wrote 1000 images to
/tmp/madlib_XUO9OeFCRp/fashion_mnist_test0005.tmp\n",
- "PoolWorker-15: Wrote 1000 images to
/tmp/madlib_NGqaV8pYyu/fashion_mnist_test0005.tmp\n",
- "PoolWorker-16: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-16: Wrote 1000 images to
/tmp/madlib_mkO0vnSoJ8/fashion_mnist_test0005.tmp\n",
- "PoolWorker-10: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-9: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-15: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-16: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-11: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-14: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-13: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-12: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-10: Wrote 1000 images to
/tmp/madlib_XUO9OeFCRp/fashion_mnist_test0006.tmp\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "PoolWorker-15: Wrote 1000 images to
/tmp/madlib_NGqaV8pYyu/fashion_mnist_test0006.tmp\n",
- "PoolWorker-9: Wrote 1000 images to
/tmp/madlib_1j4HTsicJ8/fashion_mnist_test0006.tmp\n",
- "PoolWorker-16: Wrote 1000 images to
/tmp/madlib_mkO0vnSoJ8/fashion_mnist_test0006.tmp\n",
- "PoolWorker-11: Wrote 1000 images to
/tmp/madlib_XXoZi8qgE1/fashion_mnist_test0006.tmp\n",
- "PoolWorker-14: Wrote 1000 images to
/tmp/madlib_0kkBdCmGO4/fashion_mnist_test0006.tmp\n",
- "PoolWorker-14: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-10: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-11: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-15: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-16: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-9: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-10: Wrote 1000 images to
/tmp/madlib_XUO9OeFCRp/fashion_mnist_test0007.tmp\n",
- "PoolWorker-15: Wrote 1000 images to
/tmp/madlib_NGqaV8pYyu/fashion_mnist_test0007.tmp\n",
- "PoolWorker-14: Wrote 1000 images to
/tmp/madlib_0kkBdCmGO4/fashion_mnist_test0007.tmp\n",
- "PoolWorker-16: Wrote 1000 images to
/tmp/madlib_mkO0vnSoJ8/fashion_mnist_test0007.tmp\n",
- "PoolWorker-11: Wrote 1000 images to
/tmp/madlib_XXoZi8qgE1/fashion_mnist_test0007.tmp\n",
- "PoolWorker-9: Wrote 1000 images to
/tmp/madlib_1j4HTsicJ8/fashion_mnist_test0007.tmp\n",
- "PoolWorker-14: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-16: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-10: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-15: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-11: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-9: Loaded 1000 images into fashion_mnist_test\n",
- "PoolWorker-12: Removed temporary directory /tmp/madlib_hc0Qt4WpIv\n",
- "PoolWorker-13: Removed temporary directory /tmp/madlib_2JwMfqwlOC\n",
- "PoolWorker-14: Removed temporary directory /tmp/madlib_0kkBdCmGO4\n",
- "PoolWorker-15: Removed temporary directory /tmp/madlib_NGqaV8pYyu\n",
- "PoolWorker-9: Removed temporary directory /tmp/madlib_1j4HTsicJ8\n",
- "PoolWorker-16: Removed temporary directory /tmp/madlib_mkO0vnSoJ8\n",
- "PoolWorker-10: Removed temporary directory /tmp/madlib_XUO9OeFCRp\n",
- "PoolWorker-11: Removed temporary directory /tmp/madlib_XXoZi8qgE1\n",
- "Done! Loaded 60000 images in 18.9483509064s\n",
- "8 workers terminated.\n"
- ]
- }
- ],
- "source": [
- "iloader.load_dataset_from_np(data_x, data_y, 'fashion_mnist_test',
append=True)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Load a dataset directly from disk\n",
- "iloader.load_dataset_from_disk('/tmp/image_test/cifar10',
'fromdisk_test')"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 2",
- "language": "python",
- "name": "python2"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 2
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython2",
- "version": "2.7.10"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
