piiswrong closed pull request #10008: gluon dataset download refactor
URL: https://github.com/apache/incubator-mxnet/pull/10008
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/python/mxnet/gluon/contrib/data/text.py
b/python/mxnet/gluon/contrib/data/text.py
index 1f529545a7b..8f69dbd7dad 100644
--- a/python/mxnet/gluon/contrib/data/text.py
+++ b/python/mxnet/gluon/contrib/data/text.py
@@ -28,16 +28,17 @@
from . import _constants as C
from ...data import dataset
-from ...utils import download, check_sha1
+from ...utils import download, check_sha1, _get_repo_file_url
from ....contrib import text
from .... import nd
class _LanguageModelDataset(dataset._DownloadedDataset): # pylint:
disable=abstract-method
- def __init__(self, repo_dir, root, vocabulary):
+ def __init__(self, root, namespace, vocabulary):
self._vocab = vocabulary
self._counter = None
- super(_LanguageModelDataset, self).__init__(repo_dir, root, None)
+ self._namespace = namespace
+ super(_LanguageModelDataset, self).__init__(root, None)
@property
def vocabulary(self):
@@ -76,7 +77,7 @@ def _get_data(self):
data_file_name, data_hash = self._data_file[self._segment]
path = os.path.join(self._root, data_file_name)
if not os.path.exists(path) or not check_sha1(path, data_hash):
- downloaded_file_path = download(self._get_url(archive_file_name),
+ downloaded_file_path =
download(_get_repo_file_url(self._namespace, archive_file_name),
path=self._root,
sha1_hash=archive_hash)
@@ -89,11 +90,17 @@ def _get_data(self):
open(dest, "wb") as target:
shutil.copyfileobj(source, target)
- data, label = self._read_batch(os.path.join(self._root,
data_file_name))
+ data, label = self._read_batch(path)
self._data = nd.array(data, dtype=data.dtype).reshape((-1,
self._seq_len))
self._label = nd.array(label, dtype=label.dtype).reshape((-1,
self._seq_len))
+ def __getitem__(self, idx):
+ return self._data[idx], self._label[idx]
+
+ def __len__(self):
+ return len(self._label)
+
class WikiText2(_WikiText):
"""WikiText-2 word-level dataset for language modeling, from Salesforce
research.
@@ -130,7 +137,7 @@ def __init__(self, root=os.path.join('~', '.mxnet',
'datasets', 'wikitext-2'),
'c7b8ce0aa086fb34dab808c5c49224211eb2b172')}
self._segment = segment
self._seq_len = seq_len
- super(WikiText2, self).__init__('wikitext-2', root, vocab)
+ super(WikiText2, self).__init__(root, 'wikitext-2', vocab)
class WikiText103(_WikiText):
@@ -167,4 +174,4 @@ def __init__(self, root=os.path.join('~', '.mxnet',
'datasets', 'wikitext-103'),
'8a5befc548865cec54ed4273cf87dbbad60d1e47')}
self._segment = segment
self._seq_len = seq_len
- super(WikiText103, self).__init__('wikitext-103', root, vocab)
+ super(WikiText103, self).__init__(root, 'wikitext-103', vocab)
diff --git a/python/mxnet/gluon/data/dataset.py
b/python/mxnet/gluon/data/dataset.py
index fe1e8132642..bf5fa0a6d1e 100644
--- a/python/mxnet/gluon/data/dataset.py
+++ b/python/mxnet/gluon/data/dataset.py
@@ -182,24 +182,18 @@ def __getitem__(self, idx):
def __len__(self):
return len(self._record.keys)
-apache_repo_url = 'https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/'
class _DownloadedDataset(Dataset):
"""Base class for MNIST, cifar10, etc."""
- def __init__(self, repo_dir, root, transform):
- self._root = os.path.expanduser(root)
- self._repo_dir = repo_dir
+ def __init__(self, root, transform):
+ super(_DownloadedDataset, self).__init__()
self._transform = transform
self._data = None
self._label = None
-
- repo_url = os.environ.get('MXNET_GLUON_REPO', apache_repo_url)
- if repo_url[-1] != '/':
- repo_url = repo_url+'/'
- self._base_url = repo_url
-
- if not os.path.isdir(self._root):
- os.makedirs(self._root)
+ root = os.path.expanduser(root)
+ self._root = root
+ if not os.path.isdir(root):
+ os.makedirs(root)
self._get_data()
def __getitem__(self, idx):
@@ -212,8 +206,3 @@ def __len__(self):
def _get_data(self):
raise NotImplementedError
-
- def _get_url(self, filename):
- return
'{base_url}gluon/dataset/{repo_dir}/{filename}'.format(base_url=self._base_url,
-
repo_dir=self._repo_dir,
-
filename=filename)
diff --git a/python/mxnet/gluon/data/vision/datasets.py
b/python/mxnet/gluon/data/vision/datasets.py
index 4568da67a95..d199cad7592 100644
--- a/python/mxnet/gluon/data/vision/datasets.py
+++ b/python/mxnet/gluon/data/vision/datasets.py
@@ -29,7 +29,7 @@
import numpy as np
from .. import dataset
-from ...utils import download, check_sha1
+from ...utils import download, check_sha1, _get_repo_file_url
from .... import nd, image, recordio
@@ -62,7 +62,8 @@ def __init__(self, root=os.path.join('~', '.mxnet',
'datasets', 'mnist'),
'c3a25af1f52dad7f726cce8cacb138654b760d48')
self._test_label = ('t10k-labels-idx1-ubyte.gz',
'763e7fa3757d93b0cdec073cef058b2004252c17')
- super(MNIST, self).__init__('mnist', root, transform)
+ self._namespace = 'mnist'
+ super(MNIST, self).__init__(root, transform)
def _get_data(self):
if self._train:
@@ -70,10 +71,10 @@ def _get_data(self):
else:
data, label = self._test_data, self._test_label
- data_file = download(self._get_url(data[0]),
+ data_file = download(_get_repo_file_url(self._namespace, data[0]),
path=self._root,
sha1_hash=data[1])
- label_file = download(self._get_url(label[0]),
+ label_file = download(_get_repo_file_url(self._namespace, label[0]),
path=self._root,
sha1_hash=label[1])
@@ -121,7 +122,8 @@ def __init__(self, root=os.path.join('~', '.mxnet',
'datasets', 'fashion-mnist')
'626ed6a7c06dd17c0eec72fa3be1740f146a2863')
self._test_label = ('t10k-labels-idx1-ubyte.gz',
'17f9ab60e7257a1620f4ad76bbbaf857c3920701')
- super(MNIST, self).__init__('fashion-mnist', root, transform) #
pylint: disable=bad-super-call
+ self._namespace = 'fashion-mnist'
+ super(MNIST, self).__init__(root, transform) # pylint:
disable=bad-super-call
class CIFAR10(dataset._DownloadedDataset):
@@ -152,7 +154,8 @@ def __init__(self, root=os.path.join('~', '.mxnet',
'datasets', 'cifar10'),
('data_batch_4.bin',
'aab85764eb3584312d3c7f65fd2fd016e36a258e'),
('data_batch_5.bin',
'26e2849e66a845b7f1e4614ae70f4889ae604628')]
self._test_data = [('test_batch.bin',
'67eb016db431130d61cd03c7ad570b013799c88c')]
- super(CIFAR10, self).__init__('cifar10', root, transform)
+ self._namespace = 'cifar10'
+ super(CIFAR10, self).__init__(root, transform)
def _read_batch(self, filename):
with open(filename, 'rb') as fin:
@@ -165,7 +168,7 @@ def _get_data(self):
if any(not os.path.exists(path) or not check_sha1(path, sha1)
for path, sha1 in ((os.path.join(self._root, name), sha1)
for name, sha1 in self._train_data +
self._test_data)):
- filename = download(self._get_url(self._archive_file[0]),
+ filename = download(_get_repo_file_url(self._namespace,
self._archive_file[0]),
path=self._root,
sha1_hash=self._archive_file[1])
@@ -212,7 +215,8 @@ def __init__(self, root=os.path.join('~', '.mxnet',
'datasets', 'cifar100'),
self._train_data = [('train.bin',
'e207cd2e05b73b1393c74c7f5e7bea451d63e08e')]
self._test_data = [('test.bin',
'8fb6623e830365ff53cf14adec797474f5478006')]
self._fine_label = fine_label
- super(CIFAR10, self).__init__('cifar100', root, transform) # pylint:
disable=bad-super-call
+ self._namespace = 'cifar100'
+ super(CIFAR10, self).__init__(root, transform) # pylint:
disable=bad-super-call
def _read_batch(self, filename):
with open(filename, 'rb') as fin:
diff --git a/python/mxnet/gluon/utils.py b/python/mxnet/gluon/utils.py
index 88effc99b9b..9ebfe4c06cc 100644
--- a/python/mxnet/gluon/utils.py
+++ b/python/mxnet/gluon/utils.py
@@ -192,10 +192,12 @@ def download(url, path=None, overwrite=False,
sha1_hash=None):
"""
if path is None:
fname = url.split('/')[-1]
- elif os.path.isdir(path):
- fname = os.path.join(path, url.split('/')[-1])
else:
- fname = path
+ path = os.path.expanduser(path)
+ if os.path.isdir(path):
+ fname = os.path.join(path, url.split('/')[-1])
+ else:
+ fname = path
if overwrite or not os.path.exists(fname) or (sha1_hash and not
check_sha1(fname, sha1_hash)):
dirname = os.path.dirname(os.path.abspath(os.path.expanduser(fname)))
@@ -218,3 +220,25 @@ def download(url, path=None, overwrite=False,
sha1_hash=None):
'the default repo.'.format(fname))
return fname
+
+def _get_repo_url():
+ """Return the base URL for Gluon dataset and model repository."""
+ default_repo =
'https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/'
+ repo_url = os.environ.get('MXNET_GLUON_REPO', default_repo)
+ if repo_url[-1] != '/':
+ repo_url = repo_url+'/'
+ return repo_url
+
+def _get_repo_file_url(namespace, filename):
+ """Return the URL for hoste file in Gluon repository.
+
+ Parameters
+ ----------
+ namespace : str
+ Namespace of the file.
+ filename : str
+ Name of the file
+ """
+ return
'{base_url}gluon/dataset/{namespace}/{filename}'.format(base_url=_get_repo_url(),
+
namespace=namespace,
+
filename=filename)
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
