This is an automated email from the ASF dual-hosted git repository.
fmcquillan pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/madlib-site.git
The following commit(s) were added to refs/heads/asf-site by this push:
new 4ef7217 update Load-model-architecture-v1.ipynb with faster way to
load model weights
4ef7217 is described below
commit 4ef7217f0adf914a7f0e63115a199e60a334677f
Author: Frank McQuillan <[email protected]>
AuthorDate: Tue Aug 20 16:14:41 2019 -0700
update Load-model-architecture-v1.ipynb with faster way to load model
weights
---
.../Deep-learning/Load-model-architecture-v1.ipynb | 164 ++++++++++-----------
1 file changed, 78 insertions(+), 86 deletions(-)
diff --git a/community-artifacts/Deep-learning/Load-model-architecture-v1.ipynb
b/community-artifacts/Deep-learning/Load-model-architecture-v1.ipynb
index e7cd2f1..4a8a5d5 100644
--- a/community-artifacts/Deep-learning/Load-model-architecture-v1.ipynb
+++ b/community-artifacts/Deep-learning/Load-model-architecture-v1.ipynb
@@ -90,12 +90,12 @@
" <th>version</th>\n",
" </tr>\n",
" <tr>\n",
- " <td>MADlib version: 1.16-dev, git revision:
rel/v1.15.1-119-gea1e0ac, cmake configuration time: Sat Jun 8 00:55:28 UTC
2019, build type: release, build system: Linux-3.10.0-957.12.1.el7.x86_64, C
compiler: gcc 4.8.5, C++ compiler: g++ 4.8.5</td>\n",
+ " <td>MADlib version: 1.17-dev, git revision:
rel/v1.16-10-g205bdba, cmake configuration time: Thu Aug 15 17:53:15 UTC 2019,
build type: release, build system: Linux-3.10.0-957.21.3.el7.x86_64, C
compiler: gcc 4.8.5, C++ compiler: g++ 4.8.5</td>\n",
" </tr>\n",
"</table>"
],
"text/plain": [
- "[(u'MADlib version: 1.16-dev, git revision: rel/v1.15.1-119-gea1e0ac,
cmake configuration time: Sat Jun 8 00:55:28 UTC 2019, build type: release,
build system: Linux-3.10.0-957.12.1.el7.x86_64, C compiler: gcc 4.8.5, C++
compiler: g++ 4.8.5',)]"
+ "[(u'MADlib version: 1.17-dev, git revision: rel/v1.16-10-g205bdba,
cmake configuration time: Thu Aug 15 17:53:15 UTC 2019, build type: release,
build system: Linux-3.10.0-957.21.3.el7.x86_64, C compiler: gcc 4.8.5, C++
compiler: g++ 4.8.5',)]"
]
},
"execution_count": 3,
@@ -120,9 +120,24 @@
},
{
"cell_type": "code",
- "execution_count": 28,
+ "execution_count": 4,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Using TensorFlow backend.\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Couldn't import dot_parser, loading of dot files will not be
possible.\n"
+ ]
+ }
+ ],
"source": [
"import keras\n",
"from keras.models import Sequential\n",
@@ -138,7 +153,7 @@
},
{
"cell_type": "code",
- "execution_count": 30,
+ "execution_count": 5,
"metadata": {},
"outputs": [
{
@@ -148,11 +163,11 @@
"_________________________________________________________________\n",
"Layer (type) Output Shape Param # \n",
"=================================================================\n",
- "dense_13 (Dense) (None, 10) 50 \n",
+ "dense_1 (Dense) (None, 10) 50 \n",
"_________________________________________________________________\n",
- "dense_14 (Dense) (None, 10) 110 \n",
+ "dense_2 (Dense) (None, 10) 110 \n",
"_________________________________________________________________\n",
- "dense_15 (Dense) (None, 3) 33 \n",
+ "dense_3 (Dense) (None, 3) 33 \n",
"=================================================================\n",
"Total params: 193\n",
"Trainable params: 193\n",
@@ -172,16 +187,16 @@
},
{
"cell_type": "code",
- "execution_count": 31,
+ "execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
- "'{\"class_name\": \"Sequential\", \"keras_version\": \"2.1.6\",
\"config\": [{\"class_name\": \"Dense\", \"config\": {\"kernel_initializer\":
{\"class_name\": \"VarianceScaling\", \"config\": {\"distribution\":
\"uniform\", \"scale\": 1.0, \"seed\": null, \"mode\": \"fan_avg\"}}, \"name\":
\"dense_13\", \"kernel_constraint\": null, \"bias_regularizer\": null,
\"bias_constraint\": null, \"dtype\": \"float32\", \"activation\": \"relu\",
\"trainable\": true, \"kernel_regularizer\": [...]
+ "'{\"class_name\": \"Sequential\", \"keras_version\": \"2.1.6\",
\"config\": [{\"class_name\": \"Dense\", \"config\": {\"kernel_initializer\":
{\"class_name\": \"VarianceScaling\", \"config\": {\"distribution\":
\"uniform\", \"scale\": 1.0, \"seed\": null, \"mode\": \"fan_avg\"}}, \"name\":
\"dense_1\", \"kernel_constraint\": null, \"bias_regularizer\": null,
\"bias_constraint\": null, \"dtype\": \"float32\", \"activation\": \"relu\",
\"trainable\": true, \"kernel_regularizer\": n [...]
]
},
- "execution_count": 31,
+ "execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
@@ -210,7 +225,7 @@
},
{
"cell_type": "code",
- "execution_count": 46,
+ "execution_count": 7,
"metadata": {},
"outputs": [
{
@@ -240,15 +255,15 @@
" <td>None</td>\n",
" <td>Sophie</td>\n",
" <td>A simple model</td>\n",
- " <td>__madlib_temp_22335672_1560210445_1050509__</td>\n",
+ " <td>__madlib_temp_16223038_1566342412_31364249__</td>\n",
" </tr>\n",
"</table>"
],
"text/plain": [
- "[(1, {u'class_name': u'Sequential', u'keras_version': u'2.1.6',
u'config': [{u'class_name': u'Dense', u'config': {u'kernel_initializer':
{u'class_name': u'VarianceScaling', u'config': {u'distribution': u'uniform',
u'scale': 1.0, u'seed': None, u'mode': u'fan_avg'}}, u'name': u'dense_1',
u'kernel_constraint': None, u'bias_regularizer': None, u'bias_constraint':
None, u'dtype': u'float32', u'activation': u'relu', u'trainable': True,
u'kernel_regularizer': None, u'bias_initializer': [...]
+ "[(1, {u'class_name': u'Sequential', u'keras_version': u'2.1.6',
u'config': [{u'class_name': u'Dense', u'config': {u'kernel_initializer':
{u'class_name': u'VarianceScaling', u'config': {u'distribution': u'uniform',
u'scale': 1.0, u'seed': None, u'mode': u'fan_avg'}}, u'name': u'dense_1',
u'kernel_constraint': None, u'bias_regularizer': None, u'bias_constraint':
None, u'dtype': u'float32', u'activation': u'relu', u'trainable': True,
u'kernel_regularizer': None, u'bias_initializer': [...]
]
},
- "execution_count": 46,
+ "execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
@@ -279,7 +294,7 @@
},
{
"cell_type": "code",
- "execution_count": 45,
+ "execution_count": 8,
"metadata": {},
"outputs": [
{
@@ -303,29 +318,29 @@
" <th>__internal_madlib_id__</th>\n",
" </tr>\n",
" <tr>\n",
- " <td>1</td>\n",
+ " <td>2</td>\n",
" <td>{u'class_name': u'Sequential', u'keras_version': u'2.1.6',
u'config': [{u'class_name': u'Dense', u'config': {u'kernel_initializer':
{u'class_name': u'VarianceScaling', u'config': {u'distribution': u'uniform',
u'scale': 1.0, u'seed': None, u'mode': u'fan_avg'}}, u'name': u'dense_1',
u'kernel_constraint': None, u'bias_regularizer': None, u'bias_constraint':
None, u'dtype': u'float32', u'activation': u'relu', u'trainable': True,
u'kernel_regularizer': None, u'bias_initia [...]
" <td>None</td>\n",
- " <td>Sophie</td>\n",
- " <td>A simple model</td>\n",
- " <td>__madlib_temp_81479542_1560210311_41108954__</td>\n",
+ " <td>Maria</td>\n",
+ " <td>Also a simple model</td>\n",
+ " <td>__madlib_temp_43018550_1566342414_433620__</td>\n",
" </tr>\n",
" <tr>\n",
- " <td>2</td>\n",
+ " <td>1</td>\n",
" <td>{u'class_name': u'Sequential', u'keras_version': u'2.1.6',
u'config': [{u'class_name': u'Dense', u'config': {u'kernel_initializer':
{u'class_name': u'VarianceScaling', u'config': {u'distribution': u'uniform',
u'scale': 1.0, u'seed': None, u'mode': u'fan_avg'}}, u'name': u'dense_1',
u'kernel_constraint': None, u'bias_regularizer': None, u'bias_constraint':
None, u'dtype': u'float32', u'activation': u'relu', u'trainable': True,
u'kernel_regularizer': None, u'bias_initia [...]
" <td>None</td>\n",
- " <td>Maria</td>\n",
- " <td>Also a simple model</td>\n",
- " <td>__madlib_temp_7879812_1560210313_52185533__</td>\n",
+ " <td>Sophie</td>\n",
+ " <td>A simple model</td>\n",
+ " <td>__madlib_temp_16223038_1566342412_31364249__</td>\n",
" </tr>\n",
"</table>"
],
"text/plain": [
- "[(1, {u'class_name': u'Sequential', u'keras_version': u'2.1.6',
u'config': [{u'class_name': u'Dense', u'config': {u'kernel_initializer':
{u'class_name': u'VarianceScaling', u'config': {u'distribution': u'uniform',
u'scale': 1.0, u'seed': None, u'mode': u'fan_avg'}}, u'name': u'dense_1',
u'kernel_constraint': None, u'bias_regularizer': None, u'bias_constraint':
None, u'dtype': u'float32', u'activation': u'relu', u'trainable': True,
u'kernel_regularizer': None, u'bias_initializer': [...]
- " (2, {u'class_name': u'Sequential', u'keras_version': u'2.1.6',
u'config': [{u'class_name': u'Dense', u'config': {u'kernel_initializer':
{u'class_name': u'VarianceScaling', u'config': {u'distribution': u'uniform',
u'scale': 1.0, u'seed': None, u'mode': u'fan_avg'}}, u'name': u'dense_1',
u'kernel_constraint': None, u'bias_regularizer': None, u'bias_constraint':
None, u'dtype': u'float32', u'activation': u'relu', u'trainable': True,
u'kernel_regularizer': None, u'bias_initializer': [...]
+ "[(2, {u'class_name': u'Sequential', u'keras_version': u'2.1.6',
u'config': [{u'class_name': u'Dense', u'config': {u'kernel_initializer':
{u'class_name': u'VarianceScaling', u'config': {u'distribution': u'uniform',
u'scale': 1.0, u'seed': None, u'mode': u'fan_avg'}}, u'name': u'dense_1',
u'kernel_constraint': None, u'bias_regularizer': None, u'bias_constraint':
None, u'dtype': u'float32', u'activation': u'relu', u'trainable': True,
u'kernel_regularizer': None, u'bias_initializer': [...]
+ " (1, {u'class_name': u'Sequential', u'keras_version': u'2.1.6',
u'config': [{u'class_name': u'Dense', u'config': {u'kernel_initializer':
{u'class_name': u'VarianceScaling', u'config': {u'distribution': u'uniform',
u'scale': 1.0, u'seed': None, u'mode': u'fan_avg'}}, u'name': u'dense_1',
u'kernel_constraint': None, u'bias_regularizer': None, u'bias_constraint':
None, u'dtype': u'float32', u'activation': u'relu', u'trainable': True,
u'kernel_regularizer': None, u'bias_initializer': [...]
]
},
- "execution_count": 45,
+ "execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
@@ -361,9 +376,27 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 9,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1 rows affected.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "[]"
+ ]
+ },
+ "execution_count": 9,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
"%%sql\n",
"UPDATE model_arch_library SET model_weights = model_data FROM iris_model
WHERE model_id = 2;"
@@ -380,7 +413,7 @@
},
{
"cell_type": "code",
- "execution_count": 47,
+ "execution_count": 10,
"metadata": {},
"outputs": [
{
@@ -400,15 +433,15 @@
" <th>count</th>\n",
" </tr>\n",
" <tr>\n",
- " <td>1</td>\n",
+ " <td>2</td>\n",
" </tr>\n",
"</table>"
],
"text/plain": [
- "[(1L,)]"
+ "[(2L,)]"
]
},
- "execution_count": 47,
+ "execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
@@ -430,8 +463,8 @@
"\n",
"# get weights, flatten and serialize\n",
"weights = model.get_weights()\n",
- "weights_flat = [ w.flatten() for w in weights ]\n",
- "weights1d = np.array([j for sub in weights_flat for j in sub])\n",
+ "weights_flat = [w.flatten() for w in weights]\n",
+ "weights1d = np.concatenate(weights_flat).ravel()\n",
"weights_bytea = weights1d.tostring()\n",
"\n",
"# load query\n",
@@ -455,12 +488,14 @@
"source": [
"<a id=\"load_model_wts_keras2\"></a>\n",
"## 3c. Load weights from Keras using psycopg2\n",
- "Psycopg is a PostgreSQL database adapter for the Python programming
language. As above we need to flatten then serialize the weights to store as a
PostgreSQL binary data type."
+ "Psycopg is a PostgreSQL database adapter for the Python programming
language. As above we need to flatten then serialize the weights to store as a
PostgreSQL binary data type.\n",
+ "\n",
+ "Note that the psycopg2.Binary function will increase the size of the
Python object for the weights, so if your model is large it might be better to
use a PL/Python function as in 3b. above."
]
},
{
"cell_type": "code",
- "execution_count": 48,
+ "execution_count": 11,
"metadata": {},
"outputs": [
{
@@ -478,15 +513,15 @@
" <th>count</th>\n",
" </tr>\n",
" <tr>\n",
- " <td>2</td>\n",
+ " <td>3</td>\n",
" </tr>\n",
"</table>"
],
"text/plain": [
- "[(2L,)]"
+ "[(3L,)]"
]
},
- "execution_count": 48,
+ "execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
@@ -508,9 +543,9 @@
"\n",
"# get weights, flatten and serialize\n",
"weights = model.get_weights()\n",
- "weights_flat = [ w.flatten() for w in weights ]\n",
- "weights1d = np.array([j for sub in weights_flat for j in sub])\n",
- "weights_bytea = psycopg2.Binary(weights1d.tostring())\n",
+ "weights_flat = [w.flatten() for w in weights]\n",
+ "weights1d = np.concatenate(weights_flat).ravel()\n",
+ "weights_bytea = p2.Binary(weights1d.tostring())\n",
"\n",
"query = \"SELECT madlib.load_keras_model('model_arch_library',
%s,%s)\"\n",
"cur.execute(query,[model.to_json(),weights_bytea])\n",
@@ -527,49 +562,6 @@
"<a id=\"delete_model\"></a>\n",
"# 4. Delete model"
]
- },
- {
- "cell_type": "code",
- "execution_count": 43,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "1 rows affected.\n",
- "1 rows affected.\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "<table>\n",
- " <tr>\n",
- " <th>count</th>\n",
- " </tr>\n",
- " <tr>\n",
- " <td>3</td>\n",
- " </tr>\n",
- "</table>"
- ],
- "text/plain": [
- "[(3L,)]"
- ]
- },
- "execution_count": 43,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "%%sql\n",
- "SELECT madlib.delete_keras_model('model_arch_library', -- Output
table\n",
- " 1 -- Model id\n",
- " );\n",
- "\n",
- "SELECT COUNT(*) FROM model_arch_library;"
- ]
}
],
"metadata": {
