certainly, it should be attached to this message...
"J. op den Brouw" wrote:
> Can you please send it as an attachment, because copying it off
> the screen makes linewraps..
>
> Will Ballantyne wrote:
> >
> > sorry for the delay, I was away a few days. I am attaching the diff
> >
> > "J. op den Brouw" wrote:
> >
> > > I think we *all* want to share it with you. It should be on the patch
> > > site.
> > > Please do mail us you patch.
> > >
> > > Will Ballantyne wrote:
> > > >
> > > > I've got htdig working with ssl by using the ssl patch for 3.1.4 and
> > > > making the necessary changes. If anyone wants a new diffs file send me
> > > > a note...
> > >
> > > --Jesse
> >
> > ------------------------------------------------------------------------
> > diff -ur htdig-3.1.5/CONFIG htdig-3.1.5-patched/CONFIG
> > --- htdig-3.1.5/CONFIG Thu Feb 24 18:29:48 2000
> (snipped)
>
> --Jesse
diff -ur htdig-3.1.5/CONFIG htdig-3.1.5-patched/CONFIG
--- htdig-3.1.5/CONFIG Thu Feb 24 18:29:48 2000
+++ htdig-3.1.5-patched/CONFIG Sun Oct 22 12:39:40 2000
@@ -8,7 +8,7 @@
# These variables are set by configure
#
# This specifies the root of the directory tree to be used by ht://Dig
-prefix= /opt/www/htdig
+prefix= /home/www/htdig
# This specifies the root of the directory tree to be used for programs
# installed by ht://Dig
diff -ur htdig-3.1.5/Makefile.config.in htdig-3.1.5-patched/Makefile.config.in
--- htdig-3.1.5/Makefile.config.in Thu Feb 24 18:28:58 2000
+++ htdig-3.1.5-patched/Makefile.config.in Sun Oct 22 10:04:19 2000
@@ -24,13 +24,13 @@
SENDMAIL= @SENDMAIL@
DEFINES= -DDEFAULT_CONFIG_FILE=\"$(DEFAULT_CONFIG_FILE)\"
-LIBDIRS= -L../htlib -L../htcommon -L../db/dist -L/usr/lib
+LIBDIRS= -L../htlib -L../htcommon -L../db/dist -L/usr/lib -L/opt/ssl/lib
INCS= -I$(top_srcdir)/htlib -I$(top_srcdir)/htcommon \
- -I../db/dist -I../include
+ -I../db/dist -I../include -I/opt/ssl/include
HTLIBS= ../htcommon/libcommon.a \
../htlib/libht.a \
../db/dist/libdb.a
-LIBS= $(HTLIBS) @LIBS@
+LIBS= $(HTLIBS) @LIBS@ -lssl -lcrypto
DIST= @PACKAGE@-@VERSION@
DISTDIR= $(top_srcdir)/../$(DIST)
diff -ur htdig-3.1.5/htcommon/DocumentDB.cc htdig-3.1.5-patched/htcommon/DocumentDB.cc
--- htdig-3.1.5/htcommon/DocumentDB.cc Thu Feb 24 18:29:10 2000
+++ htdig-3.1.5-patched/htcommon/DocumentDB.cc Sun Oct 22 10:04:19 2000
@@ -217,7 +217,7 @@
while ((key = dbf->Get_Next()))
{
dbf->Get(key, data);
- if (strncmp(HtURLCodec::instance()->decode(key), "http:", 5) == 0)
+ if (strncmp(HtURLCodec::instance()->decode(key), "http:", 5) == 0 ||
+strncmp(HtURLCodec::instance()->decode(key), "https:", 6) == 0)
{
ref = new DocumentRef;
ref->Deserialize(data);
@@ -284,7 +284,7 @@
while ((coded_key = dbf->Get_Next()))
{
String key = HtURLCodec::instance()->decode(coded_key);
- if (mystrncasecmp(key, "http:", 5) == 0)
+ if (mystrncasecmp(key, "http:", 5) == 0 || mystrncasecmp(key, "https:", 6) ==
+0)
{
DocumentRef *ref = (*this)[key];
if (ref)
diff -ur htdig-3.1.5/htcommon/defaults.cc htdig-3.1.5-patched/htcommon/defaults.cc
--- htdig-3.1.5/htcommon/defaults.cc Thu Feb 24 18:29:10 2000
+++ htdig-3.1.5-patched/htcommon/defaults.cc Sun Oct 22 10:04:19 2000
@@ -38,7 +38,7 @@
{"bad_word_list", "${common_dir}/bad_words"},
{"build_select_lists", ""},
{"case_sensitive", "true"},
- {"common_url_parts", "http:// http://www. ftp:// ftp://ftp. /pub/
.html .htm .gif .jpg .jpeg /index.html /index.htm .com/ .com mailto:"},
+ {"common_url_parts", "https:// https://www. http:// http://www.
+ftp:// ftp://ftp. /pub/ .html .htm .gif .jpg .jpeg /index.html /index.htm .com/ .com
+mailto:"},
{"create_image_list", "false"},
{"create_url_list", "false"},
{"compression_level", "0"},
diff -ur htdig-3.1.5/htdig/Document.cc htdig-3.1.5-patched/htdig/Document.cc
--- htdig-3.1.5/htdig/Document.cc Thu Feb 24 18:29:10 2000
+++ htdig-3.1.5-patched/htdig/Document.cc Sun Oct 22 10:04:19 2000
@@ -220,6 +220,7 @@
tm.tm_year += 1900;
tm.tm_yday = 0; // clear these to prevent problems in strftime()
tm.tm_wday = 0;
+ tm.tm_isdst = -1;
if (debug > 2)
{
@@ -331,7 +332,7 @@
return Document_no_host;
}
}
-
+ c.assign_ssl(strcmp(url->service(), "https") == 0);
if (c.connect(1) == NOTOK)
{
if (debug)
diff -ur htdig-3.1.5/htdig/Images.cc htdig-3.1.5-patched/htdig/Images.cc
--- htdig-3.1.5/htdig/Images.cc Thu Feb 24 18:29:10 2000
+++ htdig-3.1.5-patched/htdig/Images.cc Sun Oct 22 10:04:19 2000
@@ -61,7 +61,7 @@
{
String u = url;
URL Url(url);
- if (strcmp(Url.service(), "http") != 0)
+ if (strcmp(Url.service(), "http") != 0 && strcmp(Url.service(), "https") != 0)
return 0;
u.lowercase();
@@ -81,6 +81,7 @@
return 0;
if (c.assign_server(Url.host()) == NOTOK)
return 0;
+ c.assign_ssl(strcmp(Url.service(), "https") == 0);
if (c.connect(1) == NOTOK)
{
diff -ur htdig-3.1.5/htdig/Retriever.cc htdig-3.1.5-patched/htdig/Retriever.cc
--- htdig-3.1.5/htdig/Retriever.cc Thu Feb 24 18:29:10 2000
+++ htdig-3.1.5-patched/htdig/Retriever.cc Sun Oct 22 11:18:42 2000
@@ -117,8 +117,7 @@
// from == 2 add url from db.log
// from == 3 urls in db.docs and there was a db.log
//
-void
-Retriever::Initial(char *list, int from)
+void Retriever::Initial(char *list, int from)
{
//
// Split the list of urls up into individual urls.
@@ -133,14 +132,15 @@
URL u(tokens[i]);
server = (Server *) servers[u.signature()];
url = u.get();
+cout << "URL: " << url;
if (debug > 2)
cout << "\t" << from << ":" << (int) log << ":" << url;
if (!server)
{
- String robotsURL = "http://";
- robotsURL << u.host() << "/robots.txt";
+ String robotsURL = u.service();
+ robotsURL << "://" << u.host() << "/robots.txt";
StringList *localRobotsFiles = GetLocal(robotsURL.get());
- server = new Server(u.host(), u.port(), localRobotsFiles);
+ server = new Server(u.host(), u.port(), strcmp(u.service(), "https") == 0,
+localRobotsFiles);
servers.Add(u.signature(), server);
delete localRobotsFiles;
}
@@ -668,10 +668,10 @@
// Currently, we only deal with HTTP URLs. Gopher and ftp will
// come later... ***FIX***
//
- if (strstr(u, "/../") || strncmp(u, "http://", 7) != 0)
+ if (strstr(u, "/../") || (strncmp(u, "http://", 7) != 0 && strncmp(u, "https://",
+8) != 0))
{
if (debug > 2)
- cout << endl <<" Rejected: Not an http or relative link!";
+ cout << endl <<" Rejected: Not an http, https or relative link!";
return FALSE;
}
@@ -1232,10 +1232,10 @@
//
// Hadn't seen this server, yet. Register it
//
- String robotsURL = "http://";
- robotsURL << url.host() << "/robots.txt";
+ String robotsURL = url.service();
+ robotsURL << "://" << url.host() << "/robots.txt";
StringList *localRobotsFile = GetLocal(robotsURL.get());
- server = new Server(url.host(), url.port(), localRobotsFile);
+ server = new Server(url.host(), url.port(), (int)
+(strcmp(url.service(), "https") == 0), localRobotsFile);
servers.Add(url.signature(), server);
delete localRobotsFile;
}
@@ -1365,10 +1365,10 @@
//
// Hadn't seen this server, yet. Register it
//
- String robotsURL = "http://";
- robotsURL << url.host() << "/robots.txt";
+ String robotsURL = url.service();
+ robotsURL << "://" << url.host() << "/robots.txt";
StringList *localRobotsFile = GetLocal(robotsURL.get());
- server = new Server(url.host(), url.port(), localRobotsFile);
+ server = new Server(url.host(), url.port(), strcmp(url.service(),
+"https") == 0, localRobotsFile);
servers.Add(url.signature(), server);
delete localRobotsFile;
}
diff -ur htdig-3.1.5/htdig/Server.cc htdig-3.1.5-patched/htdig/Server.cc
--- htdig-3.1.5/htdig/Server.cc Thu Feb 24 18:29:10 2000
+++ htdig-3.1.5-patched/htdig/Server.cc Sun Oct 22 10:51:57 2000
@@ -20,9 +20,9 @@
//*****************************************************************************
-// Server::Server(char *host, int port, StringList *local_robots_files)
+// Server::Server(char *host, int port, int ssl, StringList *local_robots_file)
//
-Server::Server(char *host, int port, StringList *local_robots_files)
+Server::Server(char *host, int port, int ssl, StringList *local_robots_files)
{
if (debug > 0)
cout << endl << "New server: " << host << ", " << port << endl;
@@ -40,7 +40,8 @@
//
// Attempt to get a robots.txt file from the specified server
//
- String url = "http://";
+ String url;
+ url = ssl ? "https://" : "http://";
url << host << ':' << port << "/robots.txt";
Document doc(url, 0);
diff -ur htdig-3.1.5/htdig/Server.h htdig-3.1.5-patched/htdig/Server.h
--- htdig-3.1.5/htdig/Server.h Thu Feb 24 18:29:10 2000
+++ htdig-3.1.5-patched/htdig/Server.h Sun Oct 22 10:46:02 2000
@@ -26,7 +26,7 @@
//
// Construction/Destruction
//
- Server(char *host, int port, StringList *local_robots_files = NULL);
+ Server(char *host, int port, int ssl, StringList *local_robots_file = NULL);
~Server();
//
diff -ur htdig-3.1.5/htlib/Connection.cc htdig-3.1.5-patched/htlib/Connection.cc
--- htdig-3.1.5/htlib/Connection.cc Thu Feb 24 18:29:11 2000
+++ htdig-3.1.5-patched/htlib/Connection.cc Sun Oct 22 10:04:19 2000
@@ -39,6 +39,10 @@
int rresvport(int *);
}
+SSL_CTX *Connection::ctx = NULL;
+SSL_METHOD *Connection::meth = NULL;
+
+
List all_connections;
Connection::Connection()
@@ -49,8 +53,26 @@
server_name = 0;
all_connections.Add(this);
timeout_value = 0;
+ ssl = NULL;
+ m_ssl_on = 0;
+ initSSL();
}
+void Connection::initSSL()
+{
+ if (ctx == NULL)
+ {
+ SSLeay_add_ssl_algorithms();
+ meth = SSLv2_client_method();
+ SSL_load_error_strings();
+ ctx = SSL_CTX_new(meth);
+ if (ctx == NULL)
+ {
+ printf("ctx NULL\n");
+ exit(1);
+ }
+ }
+}
//*************************************************************************
// Connection::Connection(int socket)
@@ -72,6 +94,9 @@
server_name = 0;
all_connections.Add(this);
timeout_value = 0;
+ ssl = NULL;
+ m_ssl_on = 0;
+ initSSL();
}
@@ -94,15 +119,15 @@
{
if (priv)
{
- int aport = IPPORT_RESERVED - 1;
+ int aport = IPPORT_RESERVED - 1;
- sock = rresvport(&aport);
+ sock = rresvport(&aport);
}
else
- sock = socket(AF_INET, SOCK_STREAM, 0);
+ sock = socket(AF_INET, SOCK_STREAM, 0);
if (sock == NOTOK)
- return NOTOK;
+ return NOTOK;
int on = 1;
setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (char *) &on, sizeof(on));
@@ -149,6 +174,8 @@
if (sock >= 0)
{
int ret = ::close(sock);
+ SSL_free(ssl);
+ ssl = NULL;
sock = -1;
return ret;
}
@@ -224,6 +251,16 @@
//*****************************************************************************
+// int Connection::assign_ssl(int ssl_on)
+//
+int Connection::assign_ssl(int ssl_on)
+{
+ m_ssl_on = ssl_on;
+ return OK;
+}
+
+
+//*****************************************************************************
// int Connection::connect(int allow_EINTR)
//
int Connection::connect(int allow_EINTR)
@@ -244,8 +281,24 @@
if (status == 0 || errno == EALREADY || errno == EISCONN)
{
- connected = 1;
- return OK;
+ if (m_ssl_on)
+ {
+ ssl = SSL_new(ctx);
+ if (ssl != NULL)
+ {
+ SSL_set_fd(ssl, sock);
+ if (SSL_connect(ssl) != -1)
+ {
+ connected = 1;
+ return OK;
+ }
+ }
+ }
+ else
+ {
+ connected = 1;
+ return OK;
+ }
}
#if 0
if (status == ECONNREFUSED)
@@ -373,26 +426,32 @@
need_io_stop = 0;
do
{
- errno = 0;
+ errno = 0;
+
+ if (timeout_value > 0)
+ {
+ fd_set fds;
+ FD_ZERO(&fds);
+ FD_SET(sock, &fds);
+
+ timeval tv;
+ tv.tv_sec = timeout_value;
+ tv.tv_usec = 0;
+
+ int selected = ::select(sock+1, &fds, 0, 0, &tv);
+ if (selected <= 0)
+ need_io_stop++;
+ }
- if (timeout_value > 0) {
- fd_set fds;
- FD_ZERO(&fds);
- FD_SET(sock, &fds);
-
- timeval tv;
- tv.tv_sec = timeout_value;
- tv.tv_usec = 0;
-
- int selected = ::select(sock+1, &fds, 0, 0, &tv);
- if (selected <= 0)
- need_io_stop++;
- }
-
- if (!need_io_stop)
- count = ::read(sock, buffer, maxlength);
- else
- count = -1; // Input timed out
+ if (!need_io_stop)
+ {
+ if (ssl != NULL)
+ count = SSL_read(ssl, buffer, maxlength);
+ else
+ count = ::read(sock, buffer, maxlength);
+ }
+ else
+ count = -1; // Input timed out
}
while (count < 0 && errno == EINTR && !need_io_stop);
need_io_stop = 0;
@@ -410,7 +469,10 @@
do
{
- count = ::write(sock, buffer, maxlength);
+ if (ssl != NULL)
+ count = SSL_write(ssl, buffer, maxlength);
+ else
+ count = ::write(sock, buffer, maxlength);
}
while (count < 0 && errno == EINTR && !need_io_stop);
need_io_stop = 0;
diff -ur htdig-3.1.5/htlib/Connection.h htdig-3.1.5-patched/htlib/Connection.h
--- htdig-3.1.5/htlib/Connection.h Thu Feb 24 18:29:11 2000
+++ htdig-3.1.5-patched/htlib/Connection.h Sun Oct 22 10:04:19 2000
@@ -36,6 +36,14 @@
#include <netinet/in.h>
#include <netdb.h>
+#include <openssl/rsa.h>
+#include <openssl/crypto.h>
+#include <openssl/x509.h>
+#include <openssl/pem.h>
+#include <openssl/ssl.h>
+#include <openssl/err.h>
+
+
class String;
class Connection : public io
@@ -45,6 +53,7 @@
Connection();
Connection(int socket);
~Connection();
+ void initSSL();
// (De)initialization
int open(int priv = 0);
@@ -64,6 +73,9 @@
int assign_server(unsigned int addr = INADDR_ANY);
char *get_server() {return server_name;}
+ // SLL stuff
+ int assign_ssl(int ssl_on);
+
// Connection establishment
int connect(int allow_EINTR = 0);
Connection *accept(int priv = 0);
@@ -90,6 +102,10 @@
private:
int sock;
+ int m_ssl_on;
+ SSL *ssl;
+ static SSL_CTX *ctx;
+ static SSL_METHOD *meth;
struct sockaddr_in server;
int connected;
char *peer;
diff -ur htdig-3.1.5/htlib/URL.cc htdig-3.1.5-patched/htlib/URL.cc
--- htdig-3.1.5/htlib/URL.cc Thu Feb 24 18:29:11 2000
+++ htdig-3.1.5-patched/htlib/URL.cc Sun Oct 22 12:29:29 2000
@@ -130,9 +130,15 @@
while (isalpha(*p))
p++;
int hasService = (*p == ':');
- if ((hasService && ((strncmp(ref, "http://", 7) == 0) ||
- (strncmp(ref, "http:", 5) != 0)))
- || strncmp(ref, "//", 2) == 0)
+
+ if ((hasService &&
+ ( (strncmp(ref, "http://", 7) == 0) ||
+ (strncmp(ref, "http:", 5) != 0) ||
+ (strncmp(ref, "https://", 8) == 0) ||
+ (strncmp(ref, "https:", 6) != 0)
+ )
+ )
+ || (strncmp(ref, "//", 2) == 0))
{
//
// No need to look at the parent url since this is a complete url...
@@ -217,7 +223,7 @@
_url << ":";
if (_host.length())
_url << "//" << _host;
- if (_port != 80 && strcmp(_service, "http") == 0)
+ if (_port != 80 && (strcmp(_service, "http") == 0 || strcmp(_service, "https") ==
+0))
_url << ':' << _port;
_url << _path;
}
@@ -486,7 +492,7 @@
if (_service.length() == 0 || _normal)
return;
- if (strcmp(_service, "http") != 0)
+ if (strcmp(_service, "http") != 0 && strcmp(_service, "https") != 0)
return;
removeIndex(_path);
@@ -543,7 +549,7 @@
_url << ":";
if (_host.length())
_url << "//" << _host;
- if (_port != 80 && strcmp(_service, "http") == 0)
+ if (_port != 80 && (strcmp(_service, "http") == 0 || strcmp(_service, "https") ==
+0))
_url << ':' << _port;
_url << _path;
_normal = 1;
diff -ur htdig-3.1.5/htlib/URL.h htdig-3.1.5-patched/htlib/URL.h
--- htdig-3.1.5/htlib/URL.h Thu Feb 24 18:29:11 2000
+++ htdig-3.1.5-patched/htlib/URL.h Sun Oct 22 10:04:19 2000
@@ -48,7 +48,7 @@
void host(char *h) {_host = h;}
int port() {return _port;}
void port(int p) {_port = p;}
- char *service() {return _service;}
+ char *service() {return _service.get();}
void service(char *s) {_service = s;}
char *path() {return _path;}
void path(char *p);
------------------------------------
To unsubscribe from the htdig mailing list, send a message to
[EMAIL PROTECTED]
You will receive a message to confirm this.
List archives: <http://www.htdig.org/mail/menu.html>
FAQ: <http://www.htdig.org/FAQ.html>
