indhub commented on a change in pull request #10568: [WIP] [MXNET-325] Model parallelism tutorial. URL: https://github.com/apache/incubator-mxnet/pull/10568#discussion_r181894869
########## File path: docs/tutorials/gluon/model_parallelism.md ########## @@ -0,0 +1,373 @@ + +# Model parallelism +- This is model parallelized version of http://gluon.mxnet.io/chapter05_recurrent-neural-networks/rnns-gluon.html. +- Similar to https://mxnet.incubator.apache.org/faq/model_parallel_lstm.html. + + +```python +import math +import os +import time +import numpy as np +import mxnet as mx +from mxnet import gluon, autograd +from mxnet.gluon import nn, rnn +import collections +``` + + +```python +class Dictionary(object): + def __init__(self): + self.word2idx = {} + self.idx2word = [] + + def add_word(self, word): + if word not in self.word2idx: + self.idx2word.append(word) + self.word2idx[word] = len(self.idx2word) - 1 + return self.word2idx[word] + + def __len__(self): + return len(self.idx2word) +``` + + +```python +class Corpus(object): + def __init__(self, path): + self.dictionary = Dictionary() + self.train = self.tokenize(path + 'train.txt') + self.valid = self.tokenize(path + 'valid.txt') + self.test = self.tokenize(path + 'test.txt') + + def tokenize(self, path): + """Tokenizes a text file.""" + assert os.path.exists(path) + # Add words to the dictionary + with open(path, 'r') as f: + tokens = 0 + for line in f: + words = line.split() + ['<eos>'] + tokens += len(words) + for word in words: + self.dictionary.add_word(word) + + # Tokenize file content + with open(path, 'r') as f: + ids = np.zeros((tokens,), dtype='int32') + token = 0 + for line in f: + words = line.split() + ['<eos>'] + for word in words: + ids[token] = self.dictionary.word2idx[word] + token += 1 + + return mx.nd.array(ids, dtype='int32') +``` + +`MultiGPULSTM` creates stacked LSTM with layers spread across multiple GPUs. +For example, `MultiGPULSTM(0, [1, 2, 2, 1], 400, 200, 0.5)` will create a stacked LSTM with one layer on GPU(0), two layers on GPU(1), two layers on GPU(2), one layer on GPU(3) with a hidden size of 400 embedding size of 200 and dropout probability of .5. + + +```python +class MultiGPULSTM(object): + + def __init__(self, start_device, num_layers_list, num_hidden, input_size, dropout): + """Create a MultiGPULSTM. num_layers_list dictates how many layers of LSTM + gets places in which device. For example, [1, 2, 2, 1] will create a stacked LSTM + with one layer on GPU(0), two layers on GPU(1), two layers on GPU(2), one layer on GPU(3)""" + self.lstm_dict = collections.OrderedDict() + device_index = start_device + self.trainers = [] + + for num_layers in num_layers_list: Review comment: Sure ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
