szha closed pull request #11800: [MXNET-656] Update documentation for
count_tokens_from_str
URL: https://github.com/apache/incubator-mxnet/pull/11800
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/python/mxnet/contrib/text/utils.py
b/python/mxnet/contrib/text/utils.py
index cd8ce5b0a20..88ed75986dc 100644
--- a/python/mxnet/contrib/text/utils.py
+++ b/python/mxnet/contrib/text/utils.py
@@ -29,11 +29,14 @@ def count_tokens_from_str(source_str, token_delim=' ',
seq_delim='\n',
to_lower=False, counter_to_update=None):
"""Counts tokens in the specified string.
- For token_delim='<td>' and seq_delim='<sd>', a specified string of two
sequences of tokens may
- look like::
+ For token_delim=\'<td>\' and seq_delim=\'<sd>\', a specified string of two
sequences of
+ tokens may look like::
<td>token1<td>token2<td>token3<td><sd><td>token4<td>token5<td><sd>
+ <td> and <sd> are regular expressions. Make use of \\\\ to allow special
characters as
+ delimiters. The list of
+ special characters can be found at
https://docs.python.org/3/library/re.html.
Parameters
----------
@@ -63,6 +66,11 @@ def count_tokens_from_str(source_str, token_delim=' ',
seq_delim='\n',
>>> source_str = ' Life is great ! \\n life is good . \\n'
>>> count_tokens_from_str(token_line, ' ', '\\n', True)
Counter({'!': 1, '.': 1, 'good': 1, 'great': 1, 'is': 2, 'life': 2})
+
+
+ >>> source_str = '*Life*is*great*!*\\n*life*is*good*.*\\n'
+ >>> count_tokens_from_str(token_line, '\\*', '\\n', True)
+ Counter({'is': 2, 'life': 2, '!': 1, 'great': 1, 'good': 1, '.': 1})
"""
source_str = filter(None,
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
