[tor-commits] [stem/master] Expanded descriptor compression support

[atagar]

commit 079b48126fdc97d83df84d96e7423f99870f2584
Merge: 4a8f8d04 f7949b64
Author: Damian Johnson 
Date:   Sat Mar 31 21:39:00 2018 -0700

Expanded descriptor compression support

We supported plaintext and gzip when downloading descriptors, but recently 
tor
added lzma and zstd support as well...

  https://gitweb.torproject.org/torspec.git/commit/?id=1cb56af

Lzma support was added in Python 3.3...

https://docs.python.org/3/library/lzma.html

... and zstd is covered by the zstandard module...

https://pypi.python.org/pypi/zstandard

 docs/change_log.rst   |   1 +
 stem/descriptor/remote.py | 182 --
 stem/version.py   |   2 +
 test/unit/descriptor/__init__.py  |   9 ++
 test/unit/descriptor/data/compressed_gzip | Bin 0 -> 1543 bytes
 test/unit/descriptor/data/compressed_identity |  52 
 test/unit/descriptor/data/compressed_lzma | Bin 0 -> 1652 bytes
 test/unit/descriptor/data/compressed_zstd | Bin 0 -> 1550 bytes
 test/unit/descriptor/remote.py| 174 ++--
 9 files changed, 341 insertions(+), 79 deletions(-)

___
tor-commits mailing list
tor-commits@lists.torproject.org
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

[tor-commits] [stem/master] Expanded descriptor compression support

[atagar]

commit 72700087b94f2889b5b364738a1178c324862ba5
Author: Damian Johnson 
Date:   Wed Mar 28 11:57:10 2018 -0700

Expanded descriptor compression support

We supported plaintext and gzip when downloading descriptors, but recently 
tor
added lzma and zstd support as well...

  https://gitweb.torproject.org/torspec.git/commit/?id=1cb56af

In practice I'm having difficulty finding an example of this working in the
live tor network...

  https://trac.torproject.org/projects/tor/ticket/25667

That said, gonna go ahead and push this since even setting lzma/zstd aside 
it's
a nice step forward since it allows callers to specify compression headers.
---
 docs/change_log.rst |   1 +
 stem/descriptor/remote.py   | 159 ++--
 stem/version.py |   2 +
 test/integ/descriptor/remote.py |  20 +
 test/unit/descriptor/remote.py  |  50 ++---
 5 files changed, 182 insertions(+), 50 deletions(-)

diff --git a/docs/change_log.rst b/docs/change_log.rst
index bb42c982..a67aae2d 100644
--- a/docs/change_log.rst
+++ b/docs/change_log.rst
@@ -55,6 +55,7 @@ The following are only available within Stem's `git repository
  * **Descriptors**
 
   * `Fallback directory v2 support 
`_, 
which adds *nickname* and *extrainfo*
+  * Added zstd and lzma compression support (:spec:`1cb56af`)
   * Reduced maximum descriptors fetched by the remote module to match tor's 
new limit (:trac:`24743`)
   * Consensus **shared_randomness_*_reveal_count** attributes undocumented, 
and unavailable if retrieved before their corresponding 
shared_randomness_*_value attribute (:trac:`25046`)
   * Allow 'proto' line to have blank values (:spec:`a8455f4`)
diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py
index 9181dbcf..ab61d3a7 100644
--- a/stem/descriptor/remote.py
+++ b/stem/descriptor/remote.py
@@ -80,6 +80,21 @@ content. For example...
 
   Maximum number of microdescriptors that can requested at a time by their
   hashes.
+
+.. data:: Compression (enum)
+
+  Compression when downloading descriptors.
+
+  .. versionadded:: 1.7.0
+
+  === ===
+  Compression Description
+  === ===
+  **PLAINTEXT**   Uncompressed data.
+  **GZIP**`GZip compression `_.
+  **ZSTD**`Zstandard compression `_
+  **LZMA**`LZMA compression `_.
+  === ===
 """
 
 import io
@@ -91,6 +106,13 @@ import threading
 import time
 import zlib
 
+import stem.descriptor
+import stem.prereq
+import stem.util.enum
+
+from stem import Flag
+from stem.util import _hash_attr, connection, log, str_tools, tor_tools
+
 try:
   # added in python 2.7
   from collections import OrderedDict
@@ -103,11 +125,24 @@ try:
 except ImportError:
   import urllib2 as urllib
 
-import stem.descriptor
-import stem.prereq
+try:
+  # added in python 3.3
+  import lzma
+  LZMA_SUPPORTED = True
+except ImportError:
+  LZMA_SUPPORTED = False
 
-from stem import Flag
-from stem.util import _hash_attr, connection, log, str_tools, tor_tools
+ZSTD_SUPPORTED = False
+
+Compression = stem.util.enum.Enum(
+  ('PLAINTEXT', 'identity'),
+  ('GZIP', 'gzip'),  # can also be 'deflate'
+  ('ZSTD', 'x-zstd'),
+  ('LZMA', 'x-tor-lzma'),
+)
+
+ZSTD_UNAVAILABLE_MSG = 'ZSTD is not yet supported'
+LZMA_UNAVAILABLE_MSG = 'LZMA compression was requested but requires the lzma 
module, which was added in python 3.3'
 
 # Tor has a limited number of descriptors we can fetch explicitly by their
 # fingerprint or hashes due to a limit on the url length by squid proxies.
@@ -224,7 +259,7 @@ class Query(object):
 from stem.descriptor.remote import Query
 
 query = Query(
-  '/tor/server/all.z',
+  '/tor/server/all',
   block = True,
   timeout = 30,
 )
@@ -243,7 +278,7 @@ class Query(object):
 
 print('Current relays:')
 
-for desc in Query('/tor/server/all.z', 'server-descriptor 1.0'):
+for desc in Query('/tor/server/all', 'server-descriptor 1.0'):
   print(desc.fingerprint)
 
   In either case exceptions are available via our 'error' attribute.
@@ -256,28 +291,37 @@ class Query(object):
   === ===
   ResourceDescription
   === ===
-  /tor/server/all.z   all present server 
descriptors
-  /tor/server/fp/++.z  server descriptors with the 
given fingerprints
-  /tor/extra/all.zall present extrainfo 
descriptors
-  /tor/extra/fp/++.z   extrainfo descriptors with 
the given fingerprints
-  /tor/micro/d/-.z  microdescriptors with the 
given hashes