commit 94cc91695b9f32d7f8c0c2e2789e4804b9d5dd1f
Author: Damian Johnson <[email protected]>
Date: Sun Nov 29 12:09:54 2015 -0800
Unit test with a minimal man page
I backed off from bundling a copy of tor's man page in our tests due to its
size (136 KB). However, a trimmed man page sidesteps that while still
providing
great test coverage.
Exercising our parser with a real (but minimal) copy of tor's man page,
providing unit test coverage of our parser that's quick to invoke.
---
stem/manual.py | 2 +-
test/unit/manual.py | 87 +++++++++++++++++
test/unit/tor_man_example | 227 +++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 315 insertions(+), 1 deletion(-)
diff --git a/stem/manual.py b/stem/manual.py
index 138380e..868f866 100644
--- a/stem/manual.py
+++ b/stem/manual.py
@@ -218,7 +218,7 @@ class Manual(object):
:var dict signals: mapping of signals tor accepts to their description
:var dict files: mapping of file paths to their description
- :var dict config_option: :class:`~stem.manual.ConfigOption` tuples for tor
configuration options
+ :var dict config_options: :class:`~stem.manual.ConfigOption` tuples for tor
configuration options
"""
def __init__(self, name, synopsis, description, commandline_options,
signals, files, config_options):
diff --git a/test/unit/manual.py b/test/unit/manual.py
index 1f5ea74..6bed338 100644
--- a/test/unit/manual.py
+++ b/test/unit/manual.py
@@ -3,6 +3,7 @@ Unit testing for the stem.manual module.
"""
import io
+import os
import unittest
import stem.prereq
@@ -28,6 +29,74 @@ except ImportError:
URL_OPEN = 'urllib.request.urlopen' if stem.prereq.is_python_3() else
'urllib2.urlopen'
+EXPECTED_DESCRIPTION = 'Tor is a connection-oriented anonymizing communication
service. Users choose a source-routed path through a set of nodes, and
negotiate a "virtual circuit" through the network, in which each node knows its
predecessor and successor, but no others. Traffic flowing down the circuit is
unwrapped by a symmetric key at each node, which reveals the downstream node.'
+
+EXPECTED_CLI_OPTIONS = {
+ '-f FILE': 'Specify a new configuration file to contain further Tor
configuration options OR pass - to make Tor read its configuration from
standard input. (Default: @CONFDIR@/torrc, or $HOME/.torrc if that file is not
found)',
+ '-h, -help': 'Display a short help message and exit.',
+ '--allow-missing-torrc': 'Do not require that configuration file specified
by -f exist if default torrc can be accessed.',
+}
+
+EXPECTED_SIGNALS = {
+ 'SIGHUP': 'The signal instructs Tor to reload its configuration (including
closing and reopening logs), and kill and restart its helper processes if
applicable.',
+ 'SIGTERM': 'Tor will catch this, clean up and sync to disk if necessary, and
exit.',
+ 'SIGINT': 'Tor clients behave as with SIGTERM; but Tor servers will do a
controlled slow shutdown, closing listeners and waiting 30 seconds before
exiting. (The delay can be configured with the ShutdownWaitLength config
option.)',
+}
+
+EXPECTED_FILES = {
+ '@LOCALSTATEDIR@/lib/tor/': 'The tor process stores keys and other data
here.',
+ 'DataDirectory/cached-status/': 'The most recently downloaded network status
document for each authority. Each file holds one such document; the filenames
are the hexadecimal identity key fingerprints of the directory authorities.
Mostly obsolete.',
+ 'DataDirectory/cached-certs': 'This file holds downloaded directory key
certificates that are used to verify authenticity of documents generated by Tor
directory authorities.',
+ 'DataDirectory/state': 'A set of persistent key-value mappings. These are
documented in the file. These include: o The current entry guards and their
status. o The current bandwidth accounting values (unused so far; see below).
o When the file was last written o What version of Tor generated the state
file o A short history of bandwidth usage, as produced in the server
descriptors.',
+ '@CONFDIR@/torrc': 'The configuration file, which contains "option value"
pairs.',
+ 'DataDirectory/bw_accounting': "Used to track bandwidth accounting values
(when the current period starts and ends; how much has been read and written so
far this period). This file is obsolete, and the data is now stored in the
'state' file as well. Only used when bandwidth accounting is enabled.",
+ '$HOME/.torrc': 'Fallback location for torrc, if @CONFDIR@/torrc is not
found.',
+}
+
+EXPECTED_CONFIG_OPTIONS = OrderedDict()
+
+EXPECTED_CONFIG_OPTIONS['AllowInvalidNodes'] = stem.manual.ConfigOption(
+ category = 'Client',
+ name = 'AllowInvalidNodes',
+ usage = 'entry|exit|middle|introduction|rendezvous|...',
+ summary = 'Permits use of relays flagged as invalid by authorities',
+ description = 'If some Tor servers are obviously not working right, the
directory authorities can manually mark them as invalid, meaning that it\'s not
recommended you use them for entry or exit positions in your circuits. You can
opt to use them in some circuit positions, though. The default is
"middle,rendezvous", and other choices are not advised.')
+
+EXPECTED_CONFIG_OPTIONS['ExcludeSingleHopRelays'] = stem.manual.ConfigOption(
+ category = 'Client',
+ name = 'ExcludeSingleHopRelays',
+ usage = '0|1',
+ summary = 'Permits use of relays that allow single hop connections',
+ description = 'This option controls whether circuits built by Tor will
include relays with the AllowSingleHopExits flag set to true. If
ExcludeSingleHopRelays is set to 0, these relays will be included. Note that
these relays might be at higher risk of being seized or observed, so they are
not normally included. Also note that relatively few clients turn off this
option, so using these relays might make your client stand out. (Default: 1)')
+
+EXPECTED_CONFIG_OPTIONS['Bridge'] = stem.manual.ConfigOption(
+ category = 'Client',
+ name = 'Bridge',
+ usage = '[transport] IP:ORPort [fingerprint]',
+ summary = 'Available bridges',
+ description = 'When set along with UseBridges, instructs Tor to use the
relay at "IP:ORPort" as a "bridge" relaying into the Tor network. If
"fingerprint" is provided (using the same format as for DirAuthority), we will
verify that the relay running at that location has the right fingerprint. We
also use fingerprint to look up the bridge descriptor at the bridge authority,
if it\'s provided and if UpdateBridgesFromAuthority is set too.\n\nIf
"transport" is provided, and matches to a ClientTransportPlugin line, we use
that pluggable transports proxy to transfer data to the bridge.')
+
+EXPECTED_CONFIG_OPTIONS['BandwidthRate'] = stem.manual.ConfigOption(
+ category = 'General',
+ name = 'BandwidthRate',
+ usage = 'N bytes|KBytes|MBytes|GBytes|KBits|MBits|GBits',
+ summary = 'Average bandwidth usage limit',
+ description = 'A token bucket limits the average incoming bandwidth usage on
this node to the specified number of bytes per second, and the average outgoing
bandwidth usage to that same value. If you want to run a relay in the public
network, this needs to be at the very least 75 KBytes for a relay (that is, 600
kbits) or 50 KBytes for a bridge (400 kbits) -- but of course, more is better;
we recommend at least 250 KBytes (2 mbits) if possible. (Default: 1
GByte)\n\nWith this option, and in other options that take arguments in bytes,
KBytes, and so on, other formats are also supported. Notably, "KBytes" can also
be written as "kilobytes" or "kb"; "MBytes" can be written as "megabytes" or
"MB"; "kbits" can be written as "kilobits"; and so forth. Tor also accepts
"byte" and "bit" in the singular. The prefixes "tera" and "T" are also
recognized. If no units are given, we default to bytes. To avoid confusion, we
recommend writing "bytes" or "bits" explicitly, since it\'s easy to forge
t that "B" means bytes, not bits.')
+
+EXPECTED_CONFIG_OPTIONS['BandwidthBurst'] = stem.manual.ConfigOption(
+ category = 'General',
+ name = 'BandwidthBurst',
+ usage = 'N bytes|KBytes|MBytes|GBytes|KBits|MBits|GBits',
+ summary = 'Maximum bandwidth usage limit',
+ description = 'Limit the maximum token bucket size (also known as the burst)
to the given number of bytes in each direction. (Default: 1 GByte)')
+
+EXPECTED_CONFIG_OPTIONS['MaxAdvertisedBandwidth'] = stem.manual.ConfigOption(
+ category = 'General',
+ name = 'MaxAdvertisedBandwidth',
+ usage = 'N bytes|KBytes|MBytes|GBytes|KBits|MBits|GBits',
+ summary = 'Limit for the bandwidth we advertise as being available for
relaying',
+ description = 'If set, we will not advertise more than this amount of
bandwidth for our BandwidthRate. Server operators who want to reduce the number
of clients who ask to build circuits through them (since this is proportional
to advertised bandwidth rate) can thus reduce the CPU demands on their server
without impacting network performance.')
+
class TestManual(unittest.TestCase):
def test_is_important(self):
@@ -37,6 +106,24 @@ class TestManual(unittest.TestCase):
self.assertFalse(stem.manual.is_important('ConstrainedSockSize'))
+ def test_parsing_example_man_page(self):
+ """
+ Read a trimmed copy of tor's man page. This gives a good exercise of our
+ parser with static content. As new oddball man oddities appear feel free to
+ expand our example (or add another).
+ """
+
+ man_path = os.path.join(os.path.dirname(__file__), 'tor_man_example')
+ manual = stem.manual.Manual.from_man(man_path)
+
+ self.assertEqual('tor - The second-generation onion router', manual.name)
+ self.assertEqual('tor [OPTION value]...', manual.synopsis)
+ self.assertEqual(EXPECTED_DESCRIPTION, manual.description)
+ self.assertEqual(EXPECTED_CLI_OPTIONS, manual.commandline_options)
+ self.assertEqual(EXPECTED_SIGNALS, manual.signals)
+ self.assertEqual(EXPECTED_FILES, manual.files)
+ self.assertEqual(EXPECTED_CONFIG_OPTIONS, manual.config_options)
+
def test_download_man_page_without_arguments(self):
try:
stem.manual.download_man_page()
diff --git a/test/unit/tor_man_example b/test/unit/tor_man_example
new file mode 100644
index 0000000..c6116f6
--- /dev/null
+++ b/test/unit/tor_man_example
@@ -0,0 +1,227 @@
+'\" t
+.\" Title: tor
+.\" Author: [see the "AUTHORS" section]
+.\" Generator: DocBook XSL Stylesheets v1.78.1 <http://docbook.sf.net/>
+.\" Date: 11/29/2015
+.\" Manual: Tor Manual
+.\" Source: Tor
+.\" Language: English
+.\"
+.TH "TOR" "1" "11/29/2015" "Tor" "Tor Manual"
+.\" -----------------------------------------------------------------
+.\" * Define some portability stuff
+.\" -----------------------------------------------------------------
+.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.\" http://bugs.debian.org/507673
+.\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html
+.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.ie \n(.g .ds Aq \(aq
+.el .ds Aq '
+.\" -----------------------------------------------------------------
+.\" * set default formatting
+.\" -----------------------------------------------------------------
+.\" disable hyphenation
+.nh
+.\" disable justification (adjust text to left margin only)
+.ad l
+.\" -----------------------------------------------------------------
+.\" * MAIN CONTENT STARTS HERE *
+.\" -----------------------------------------------------------------
+.SH "NAME"
+tor \- The second\-generation onion router
+.SH "SYNOPSIS"
+.sp
+\fBtor\fR [\fIOPTION\fR \fIvalue\fR]\&...
+.SH "DESCRIPTION"
+.sp
+Tor is a connection\-oriented anonymizing communication service\&. Users
choose a source\-routed path through a set of nodes, and negotiate a "virtual
circuit" through the network, in which each node knows its predecessor and
successor, but no others\&. Traffic flowing down the circuit is unwrapped by a
symmetric key at each node, which reveals the downstream node\&.
+.SH "COMMAND-LINE OPTIONS"
+.PP
+\fB\-h\fR, \fB\-help\fR
+.RS 4
+Display a short help message and exit\&.
+.RE
+.PP
+\fB\-f\fR \fIFILE\fR
+.RS 4
+Specify a new configuration file to contain further Tor configuration options
OR pass
+\fB\-\fR
+to make Tor read its configuration from standard input\&. (Default:
@CONFDIR@/torrc, or $HOME/\&.torrc if that file is not found)
+.RE
+.PP
+\fB\-\-allow\-missing\-torrc\fR
+.RS 4
+Do not require that configuration file specified by
+\fB\-f\fR
+exist if default torrc can be accessed\&.
+.RE
+.PP
+.sp
+Options on the command line override those in configuration files\&. See the
next section for more information\&.
+.SH "THE CONFIGURATION FILE FORMAT"
+.sp
+All configuration options in a configuration are written on a single line by
default\&. They take the form of an option name and a value, or an option name
and a quoted value (option value or option "value")\&. Anything after a #
character is treated as a comment\&. Options are case\-insensitive\&. C\-style
escaped characters are allowed inside quoted values\&. To split one
configuration entry into multiple lines, use a single backslash character (\e)
before the end of the line\&. Comments can be used in such multiline entries,
but they must start at the beginning of a line\&.
+.sp
+By default, an option on the command line overrides an option found in the
configuration file, and an option in a configuration file overrides one in the
defaults file\&.
+.SH "GENERAL OPTIONS"
+.PP
+\fBBandwidthRate\fR \fIN\fR
\fBbytes\fR|\fBKBytes\fR|\fBMBytes\fR|\fBGBytes\fR|\fBKBits\fR|\fBMBits\fR|\fBGBits\fR
+.RS 4
+A token bucket limits the average incoming bandwidth usage on this node to the
specified number of bytes per second, and the average outgoing bandwidth usage
to that same value\&. If you want to run a relay in the public network, this
needs to be
+\fIat the very least\fR
+75 KBytes for a relay (that is, 600 kbits) or 50 KBytes for a bridge (400
kbits) \(em but of course, more is better; we recommend at least 250 KBytes (2
mbits) if possible\&. (Default: 1 GByte)
+
+With this option, and in other options that take arguments in bytes, KBytes,
and so on, other formats are also supported\&. Notably, "KBytes" can also be
written as "kilobytes" or "kb"; "MBytes" can be written as "megabytes" or "MB";
"kbits" can be written as "kilobits"; and so forth\&. Tor also accepts "byte"
and "bit" in the singular\&. The prefixes "tera" and "T" are also recognized\&.
If no units are given, we default to bytes\&. To avoid confusion, we recommend
writing "bytes" or "bits" explicitly, since it\(cqs easy to forget that "B"
means bytes, not bits\&.
+.RE
+.PP
+\fBBandwidthBurst\fR \fIN\fR
\fBbytes\fR|\fBKBytes\fR|\fBMBytes\fR|\fBGBytes\fR|\fBKBits\fR|\fBMBits\fR|\fBGBits\fR
+.RS 4
+Limit the maximum token bucket size (also known as the burst) to the given
number of bytes in each direction\&. (Default: 1 GByte)
+.RE
+.PP
+\fBMaxAdvertisedBandwidth\fR \fIN\fR
\fBbytes\fR|\fBKBytes\fR|\fBMBytes\fR|\fBGBytes\fR|\fBKBits\fR|\fBMBits\fR|\fBGBits\fR
+.RS 4
+If set, we will not advertise more than this amount of bandwidth for our
BandwidthRate\&. Server operators who want to reduce the number of clients who
ask to build circuits through them (since this is proportional to advertised
bandwidth rate) can thus reduce the CPU demands on their server without
impacting network performance\&.
+.RE
+.SH "CLIENT OPTIONS"
+.sp
+The following options are useful only for clients (that is, if
\fBSocksPort\fR, \fBTransPort\fR, \fBDNSPort\fR, or \fBNATDPort\fR is
non\-zero):
+.PP
+\fBAllowInvalidNodes\fR
\fBentry\fR|\fBexit\fR|\fBmiddle\fR|\fBintroduction\fR|\fBrendezvous\fR|\fB\&...\fR
+.RS 4
+If some Tor servers are obviously not working right, the directory authorities
can manually mark them as invalid, meaning that it\(cqs not recommended you use
them for entry or exit positions in your circuits\&. You can opt to use them in
some circuit positions, though\&. The default is "middle,rendezvous", and other
choices are not advised\&.
+.RE
+.PP
+\fBExcludeSingleHopRelays\fR \fB0\fR|\fB1\fR
+.RS 4
+This option controls whether circuits built by Tor will include relays with
the AllowSingleHopExits flag set to true\&. If ExcludeSingleHopRelays is set to
0, these relays will be included\&. Note that these relays might be at higher
risk of being seized or observed, so they are not normally included\&. Also
note that relatively few clients turn off this option, so using these relays
might make your client stand out\&. (Default: 1)
+.RE
+.PP
+\fBBridge\fR [\fItransport\fR] \fIIP\fR:\fIORPort\fR [\fIfingerprint\fR]
+.RS 4
+When set along with UseBridges, instructs Tor to use the relay at "IP:ORPort"
as a "bridge" relaying into the Tor network\&. If "fingerprint" is provided
(using the same format as for DirAuthority), we will verify that the relay
running at that location has the right fingerprint\&. We also use fingerprint
to look up the bridge descriptor at the bridge authority, if it\(cqs provided
and if UpdateBridgesFromAuthority is set too\&.
+
+If "transport" is provided, and matches to a ClientTransportPlugin line, we
use that pluggable transports proxy to transfer data to the bridge\&.
+.RE
+.SH "SIGNALS"
+.sp
+Tor catches the following signals:
+.PP
+\fBSIGTERM\fR
+.RS 4
+Tor will catch this, clean up and sync to disk if necessary, and exit\&.
+.RE
+.PP
+\fBSIGINT\fR
+.RS 4
+Tor clients behave as with SIGTERM; but Tor servers will do a controlled slow
shutdown, closing listeners and waiting 30 seconds before exiting\&. (The delay
can be configured with the ShutdownWaitLength config option\&.)
+.RE
+.PP
+\fBSIGHUP\fR
+.RS 4
+The signal instructs Tor to reload its configuration (including closing and
reopening logs), and kill and restart its helper processes if applicable\&.
+.RE
+.SH "FILES"
+.PP
+\fB@CONFDIR@/torrc\fR
+.RS 4
+The configuration file, which contains "option value" pairs\&.
+.RE
+.PP
+\fB$HOME/\&.torrc\fR
+.RS 4
+Fallback location for torrc, if @CONFDIR@/torrc is not found\&.
+.RE
+.PP
+\fB@LOCALSTATEDIR@/lib/tor/\fR
+.RS 4
+The tor process stores keys and other data here\&.
+.RE
+.PP
+\fIDataDirectory\fR\fB/cached\-status/\fR
+.RS 4
+The most recently downloaded network status document for each authority\&.
Each file holds one such document; the filenames are the hexadecimal identity
key fingerprints of the directory authorities\&. Mostly obsolete\&.
+.RE
+.PP
+\fIDataDirectory\fR\fB/cached\-certs\fR
+.RS 4
+This file holds downloaded directory key certificates that are used to verify
authenticity of documents generated by Tor directory authorities\&.
+.RE
+.PP
+\fIDataDirectory\fR\fB/state\fR
+.RS 4
+A set of persistent key\-value mappings\&. These are documented in the file\&.
These include:
+.sp
+.RS 4
+.ie n \{\
+\h'-04'\(bu\h'+03'\c
+.\}
+.el \{\
+.sp -1
+.IP \(bu 2.3
+.\}
+The current entry guards and their status\&.
+.RE
+.sp
+.RS 4
+.ie n \{\
+\h'-04'\(bu\h'+03'\c
+.\}
+.el \{\
+.sp -1
+.IP \(bu 2.3
+.\}
+The current bandwidth accounting values (unused so far; see below)\&.
+.RE
+.sp
+.RS 4
+.ie n \{\
+\h'-04'\(bu\h'+03'\c
+.\}
+.el \{\
+.sp -1
+.IP \(bu 2.3
+.\}
+When the file was last written
+.RE
+.sp
+.RS 4
+.ie n \{\
+\h'-04'\(bu\h'+03'\c
+.\}
+.el \{\
+.sp -1
+.IP \(bu 2.3
+.\}
+What version of Tor generated the state file
+.RE
+.sp
+.RS 4
+.ie n \{\
+\h'-04'\(bu\h'+03'\c
+.\}
+.el \{\
+.sp -1
+.IP \(bu 2.3
+.\}
+A short history of bandwidth usage, as produced in the server descriptors\&.
+.RE
+.RE
+.PP
+\fIDataDirectory\fR\fB/bw_accounting\fR
+.RS 4
+Used to track bandwidth accounting values (when the current period starts and
ends; how much has been read and written so far this period)\&. This file is
obsolete, and the data is now stored in the \*(Aqstate\*(Aq file as well\&.
Only used when bandwidth accounting is enabled\&.
+.RE
+.SH "SEE ALSO"
+.sp
+\fBtorsocks\fR(1), \fBtorify\fR(1)
+.sp
+\fBhttps://www\&.torproject\&.org/\fR
+.sp
+\fBtorspec: \fR\fBhttps://spec\&.torproject\&.org\fR\fB \fR
+.SH "BUGS"
+.sp
+Plenty, probably\&. Tor is still in development\&. Please report them at
https://trac\&.torproject\&.org/\&;.
+.SH "AUTHORS"
+.sp
+Roger Dingledine [arma at mit\&.edu], Nick Mathewson [nickm at
alum\&.mit\&.edu]\&.
_______________________________________________
tor-commits mailing list
[email protected]
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
