I am trying to use Tor version 0.22 with Privoxy but it doesn't seem to be
picking up proxy
middlewares.py
class RetryChangeProxyMiddleware(RetryMiddleware):
def _retry(self, request, reason, spider):
try:
tor_ctrl = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
tor_ctrl.connect(('127.0.0.1', 10050))
tor_ctrl.send('AUTHENTICATE "{}"\r\nSIGNAL
NEWNYM\r\n'.format(''))
response = tor_ctrl.recv(1024)
if response != '250 OK\r\n250 OK\r\n':
sys.stderr.write('Unexpected response from Tor
control port: {}\n'.format(response))
except Exception, e:
sys.stderr.write('Error connecting to Tor control port:
{}\n'.format(repr(e)))
log.msg('Proxy changed')
return RetryMiddleware._retry(self, request, reason, spider)
class ProxyMiddleware(object):
# overwrite process request
def process_request(self, request, spider):
# Set the location of the proxy
request.meta['proxy'] = "http://localhost:8118";
settings.py
DOWNLOADER_MIDDLEWARE = {
'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware': 90,
'travel.middlewares.RetryChangeProxyMiddleware':
600,
'scrapy.contrib.downloadermiddleware.httpproxy.HttpProxyMiddleware': 110,
'travel.middlewares.ProxyMiddleware': 100,
}
What I am doing wrong here ?
--
You received this message because you are subscribed to the Google Groups
"scrapy-users" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
To post to this group, send email to [email protected].
Visit this group at http://groups.google.com/group/scrapy-users.
For more options, visit https://groups.google.com/groups/opt_out.
