szha commented on a change in pull request #11626: [MXNET-651] MXNet Model
Backwards Compatibility Checker
URL: https://github.com/apache/incubator-mxnet/pull/11626#discussion_r201785017
##########
File path: tests/nightly/model_backwards_compatibility_check/common.py
##########
@@ -165,3 +177,163 @@ def forward(self, x):
x = F.tanh(self.fc1(x))
x = F.tanh(self.fc2(x))
return x
+
+class Dictionary(object):
+ def __init__(self):
+ self.word2idx = {}
+ self.idx2word = []
+
+ def add_word(self, word):
+ if word not in self.word2idx:
+ self.idx2word.append(word)
+ self.word2idx[word] = len(self.idx2word) - 1
+ return self.word2idx[word]
+
+ def __len__(self):
+ return len(self.idx2word)
+
+class Corpus(object):
+ def __init__(self, path):
+ self.dictionary = Dictionary()
+ self.download_data_from_s3()
+ self.train = self.tokenize(path + 'train.txt')
+ self.valid = self.tokenize(path + 'valid.txt')
+ self.test = self.tokenize(path + 'test.txt')
+
+ def download_data_from_s3(self, ):
+ print ('Downloading files from bucket : ptb-small-dataset' )
+ bucket = s3.Bucket('ptb-small-dataset')
Review comment:
See the dataset introduced in #11435
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
