Author: ruschein
Date: 2011-09-20 09:52:23 -0700 (Tue, 20 Sep 2011)
New Revision: 26879
Added:
csplugins/trunk/ucsd/ruschein/GSFS/include/ThreadSafeLogger.h
csplugins/trunk/ucsd/ruschein/GSFS/src/Resolver.cc
csplugins/trunk/ucsd/ruschein/GSFS/src/ThreadSafeLogger.cc
Modified:
csplugins/trunk/ucsd/ruschein/GSFS/Makefile
csplugins/trunk/ucsd/ruschein/GSFS/include/StringUtil.h
csplugins/trunk/ucsd/ruschein/GSFS/progs/FetchPage.cc
csplugins/trunk/ucsd/ruschein/GSFS/progs/Makefile
csplugins/trunk/ucsd/ruschein/GSFS/src/ProcessUtil.cc
csplugins/trunk/ucsd/ruschein/GSFS/src/WebUtil.cc
Log:
First version that can download a web page.
Modified: csplugins/trunk/ucsd/ruschein/GSFS/Makefile
===================================================================
--- csplugins/trunk/ucsd/ruschein/GSFS/Makefile 2011-09-20 16:29:34 UTC (rev
26878)
+++ csplugins/trunk/ucsd/ruschein/GSFS/Makefile 2011-09-20 16:52:23 UTC (rev
26879)
@@ -14,4 +14,5 @@
@$(CXX) $(CXXFLAGS) $< -c -o $@
clean:
- rm -f objs/*
+ rm -f objs/* lib/*.a
+
Modified: csplugins/trunk/ucsd/ruschein/GSFS/include/StringUtil.h
===================================================================
--- csplugins/trunk/ucsd/ruschein/GSFS/include/StringUtil.h 2011-09-20
16:29:34 UTC (rev 26878)
+++ csplugins/trunk/ucsd/ruschein/GSFS/include/StringUtil.h 2011-09-20
16:52:23 UTC (rev 26879)
@@ -81,8 +81,8 @@
#if defined(__linux__)
const std::string IVIA_STANDARD_LOCALE("en_US.ISO-8859-15");
-#elif defined(__APPLE__)
-const std::string IVIA_STANDARD_LOCALE("en_US.ISO-8859-15");
+#elif defined(__MACH__)
+const std::string IVIA_STANDARD_LOCALE("en_US.ISO8859-15");
#else
# error Your OS is not supported!
#endif
Added: csplugins/trunk/ucsd/ruschein/GSFS/include/ThreadSafeLogger.h
===================================================================
--- csplugins/trunk/ucsd/ruschein/GSFS/include/ThreadSafeLogger.h
(rev 0)
+++ csplugins/trunk/ucsd/ruschein/GSFS/include/ThreadSafeLogger.h
2011-09-20 16:52:23 UTC (rev 26879)
@@ -0,0 +1,69 @@
+/** \file ThreadSafeLogger.h
+ * \brief Declaration of class ThreadSafeLogger.
+ * \author Dr. Johannes Ruscheinski
+ */
+
+/*
+ * Copyright 2006-2007 Project iVia.
+ * Copyright 2006-2007 The Regents of The University of California.
+ *
+ * This file is part of the libiViaCore package.
+ *
+ * The libiViaCore package is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License as
published
+ * by the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * libiViaCore is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with libiViaCore; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#ifndef THREAD_SAFE_LOGGER_H
+#define THREAD_SAFE_LOGGER_H
+
+
+#include <Logger.h>
+#include <ThreadUtil.h>
+
+
+/** \class ThreadSafeLogger
+ * \brief A class for logging timestamped messages that is thread-safe.
+ */
+class ThreadSafeLogger: public Logger {
+ ThreadUtil::Mutex mutex_;
+public:
+ /** \brief Creates a new ThreadSafeLogger object that writes to a log
file.
+ * \param log_filename The log file name.
+ * \param open_mode Whether to clear the log file upon opening it
or not.
+ */
+ explicit ThreadSafeLogger(const std::string &log_filename, const
Logger::VerbosityLevel default_verbosity = Logger::VL_NORMAL,
+ const Logger::OpenMode open_mode =
Logger::DO_NOT_CLEAR)
+ : Logger(log_filename, default_verbosity, open_mode) { }
+
+ /** \brief Creates a new ThreadSafeLogger object that writes to a log
file.
+ * \param log_filename The log file name.
+ * \param open_mode Whether to clear the log file upon opening it
or not.
+ */
+ explicit ThreadSafeLogger(const char * const log_filename, const
Logger::VerbosityLevel default_verbosity = Logger::VL_NORMAL,
+ const Logger::OpenMode open_mode =
Logger::DO_NOT_CLEAR)
+ : Logger(log_filename, default_verbosity, open_mode) { }
+
+ /** \brief Creates a new ThreadSafeLogger object that writes to a File.
+ * \param log_file The File to write to.
+ * \param open_mode Whether to clear the log file upon opening it or
not.
+ */
+ explicit ThreadSafeLogger(File * const log_file, const
Logger::VerbosityLevel default_verbosity = Logger::VL_NORMAL,
+ const Logger::OpenMode open_mode =
Logger::DO_NOT_CLEAR)
+ : Logger(log_file, default_verbosity, open_mode) { }
+protected:
+ virtual void writeLog(const std::string &message, const unsigned
log_mask);
+};
+
+
+#endif // ifndef THREAD_SAFE_LOGGER_H
Modified: csplugins/trunk/ucsd/ruschein/GSFS/progs/FetchPage.cc
===================================================================
--- csplugins/trunk/ucsd/ruschein/GSFS/progs/FetchPage.cc 2011-09-20
16:29:34 UTC (rev 26878)
+++ csplugins/trunk/ucsd/ruschein/GSFS/progs/FetchPage.cc 2011-09-20
16:52:23 UTC (rev 26879)
@@ -1,5 +1,6 @@
#include <iostream>
#include <MsgUtil.h>
+#include <Downloader.h>
void PrintUsage() {
@@ -11,5 +12,13 @@
if (argc != 2)
PrintUsage();
- std::cout << "Got URL: " << argv[1] << '\n';
+ try {
+ Downloader downloader(argv[1]);
+ if (downloader.anErrorOccurred())
+ std::cerr << downloader.getLastErrorMessage() << '\n';
+ else
+ std::cout << downloader.getMessageBody() << '\n';
+ } catch (const std::exception &x) {
+ std::cerr << "** Caught exception: " << x.what() << '\n';
+ }
}
Modified: csplugins/trunk/ucsd/ruschein/GSFS/progs/Makefile
===================================================================
--- csplugins/trunk/ucsd/ruschein/GSFS/progs/Makefile 2011-09-20 16:29:34 UTC
(rev 26878)
+++ csplugins/trunk/ucsd/ruschein/GSFS/progs/Makefile 2011-09-20 16:52:23 UTC
(rev 26879)
@@ -3,7 +3,10 @@
FetchPage: FetchPage.o
- $(CXX) -o $@ $^ -lcurl -lpcre -lssl -L/opt/local/lib -lmagic -lz -L
../lib -liViaCoreSubset
+ $(CXX) -o $@ $^ -lcurl -lpcre -lssl -lcrypto -L/opt/local/lib -lmagic
-lz -L ../lib -liViaCoreSubset
FetchPage.o: FetchPage.cc
$(CXX) $(CXXFLAGS) -c $<
+
+clean:
+ rm -f *.o *~
Modified: csplugins/trunk/ucsd/ruschein/GSFS/src/ProcessUtil.cc
===================================================================
--- csplugins/trunk/ucsd/ruschein/GSFS/src/ProcessUtil.cc 2011-09-20
16:29:34 UTC (rev 26878)
+++ csplugins/trunk/ucsd/ruschein/GSFS/src/ProcessUtil.cc 2011-09-20
16:52:23 UTC (rev 26879)
@@ -547,6 +547,7 @@
typedef char *CharPtr;
+ /*
#ifdef __MACH__
extern char **environ;
#endif
@@ -565,7 +566,7 @@
char **argv = (char **)::alloca((args.size() + 1) * sizeof(char
*));
argv[0] = ::strdup(::basename(::strdup(args[0].c_str())));
unsigned arg_no = 1;
- for (/* Empty! */; arg_no < args.size(); ++arg_no)
+ for (; arg_no < args.size(); ++arg_no)
argv[arg_no] = ::strdup(args[arg_no].c_str());
argv[arg_no] = NULL;
@@ -581,7 +582,7 @@
// If we make it here, we're the parent process:
*child_pid = pid;
return master_fd;
-}
+}*/
namespace {
Added: csplugins/trunk/ucsd/ruschein/GSFS/src/Resolver.cc
===================================================================
--- csplugins/trunk/ucsd/ruschein/GSFS/src/Resolver.cc
(rev 0)
+++ csplugins/trunk/ucsd/ruschein/GSFS/src/Resolver.cc 2011-09-20 16:52:23 UTC
(rev 26879)
@@ -0,0 +1,1074 @@
+/** \file Resolver.cc
+ * \brief Implementation of class Resolver. Based on RFC1035.
+ * \author Dr. Johannes Ruscheinski
+ * \author Jiangtao Hu
+ */
+
+/*
+ * Copyright 2005-2008 Project iVia.
+ * Copyright 2005-2008 The Regents of The University of California.
+ *
+ * This file is part of the libiViaCore package.
+ *
+ * The libiViaCore package is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License as
published
+ * by the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * libiViaCore is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with libiViaCore; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#include <Resolver.h>
+#include <fstream>
+#include <map>
+#include <stdexcept>
+#include <cerrno>
+#include <ctime>
+#include <inttypes.h>
+#include <arpa/nameser_compat.h>
+#include <netdb.h>
+#include <netinet/tcp.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <sys/time.h>
+#include <Compiler.h>
+#include <File.h>
+#include <FileDescriptor.h>
+#include <FileUtil.h>
+#include <IniFile.h>
+#include <MiscUtil.h>
+#include <NetUtil.h>
+#include <SocketUtil.h>
+#include <StringUtil.h>
+#include <ThreadSafeLogger.h>
+#include <ThreadUtil.h>
+#include <TimerUtil.h>
+
+
+uint16_t Resolver::next_request_id_;
+
+
+namespace {
+
+
+ThreadUtil::Mutex resolver_mutex;
+
+
+std::string GetEtcDir() {
+#ifdef __MACH__
+ return MiscUtil::GetEnv("HOME") + std::string("/etc");
+#else
+ return ETC_DIR;
+#endif
+}
+
+
+} // unnamed namespace
+
+
+bool Resolver::Cache::lookup(const std::string &hostname, std::set<in_addr_t>
* const ip_addresses)
+{
+ GNU_HASH_MAP<std::string, CacheEntry>::iterator
entry(resolved_hostnames_cache_.find(hostname));
+ if (entry != resolved_hostnames_cache_.end()) {
+ const time_t now(std::time(NULL));
+ if (entry->second.expire_time_ > now) {
+ *ip_addresses = entry->second.ip_addresses_;
+ return true;
+ }
+
+ // Entry has expired => remove it from the cache:
+ resolved_hostnames_cache_.erase(entry);
+ }
+
+ return false;
+}
+
+
+void Resolver::Cache::insert(const std::string &hostname, const
std::set<in_addr_t> &ip_addresses, const uint32_t ttl)
+{
+ // Flush the cache if it contains more than 100,000 entries:
+ if (resolved_hostnames_cache_.size() > 100000)
+ resolved_hostnames_cache_.clear();
+ else {
+ // Check to see whether we already have information about this
"hostname":
+ GNU_HASH_MAP<std::string, CacheEntry>::iterator
cache_entry(resolved_hostnames_cache_.find(hostname));
+ if (unlikely(cache_entry != resolved_hostnames_cache_.end())) {
+ for (std::set<in_addr_t>::const_iterator
ip_address(ip_addresses.begin());
+ ip_address != ip_addresses.end(); ++ip_address)
+
cache_entry->second.ip_addresses_.insert(*ip_address);
+ return;
+ }
+ }
+
+ // Create a new cache entry:
+ const time_t now(std::time(NULL));
+ const Cache::CacheEntry new_cache_entry(now + ttl, ip_addresses);
+ resolved_hostnames_cache_.insert(std::make_pair<std::string,
CacheEntry>(hostname, new_cache_entry));
+}
+
+
+Resolver::Resolver(const std::list<std::string> &dns_servers, Logger * const
logger, const unsigned verbosity)
+ : verbosity_(logger != NULL ? verbosity : 0), logger_(logger),
cleanup_logger_(false), udp_fd_(-1),
+ reply_packet_(NULL), reply_packet_size_(0)
+{
+ // This variable keeps track of whether we have read in a
+ // list of DNS servers from one of the three sources yet:
+ bool dns_servers_processed = false;
+
+ // Get the resolver IP addresses from the "dns_server" parameter:
+ if (not dns_servers.empty()) {
+ for (std::list<std::string>::const_iterator
dns_server(dns_servers.begin());
+ dns_server != dns_servers.end(); ++dns_server)
+ {
+ in_addr_t ip_address;
+ if (unlikely(not
NetUtil::StringToNetworkAddress(*dns_server, &ip_address)))
+ throw Exception("in Resolver::Resolver: \"" +
*dns_server
+ + "\" is not a valid IP address
(1)!");
+ dns_server_ip_addresses_and_busy_counts_.insert(
+ std::make_pair<in_addr_t, unsigned>(ip_address,
0));
+ }
+
+ dns_servers_processed = true;
+ }
+
+ // If a Resolver.conf file exists, read it:
+ if (FileUtil::Exists(GetEtcDir() + "/Resolver.conf")) {
+ IniFile ini_file(GetEtcDir() + "/Resolver.conf");
+
+ if (logger_ == NULL) {
+ // Read logging instructions:
+ verbosity_ = ini_file.getUnsigned("Logging",
"verbosity", 0);
+ if (verbosity_ > 5)
+ verbosity_ = 5;
+ if (verbosity_ > 0) {
+ const std::string
log_filename(ini_file.getString("Logging", "log_filename"));
+ logger_ = new ThreadSafeLogger(log_filename);
+ cleanup_logger_ = true;
+ }
+ }
+
+ // Read DNS servers:
+ if (not dns_servers_processed and
ini_file.sectionIsDefined("DNS Servers")) {
+ // Add each entry in the [DNS Servers] section of
Resolver.conf:
+ const std::list<std::string>
names(ini_file.getSectionEntryNames("DNS Servers"));
+ for (std::list<std::string>::const_iterator
name(names.begin()); name != names.end(); ++name) {
+ const std::string
ip_address_str(ini_file.getString("DNS Servers", *name));
+ in_addr_t ip_address;
+ if (unlikely(not
NetUtil::StringToNetworkAddress(ip_address_str, &ip_address)))
+ throw Exception("Resolver.conf: \"" +
ip_address_str + "\""
+ " is not a
valid IP address (2)!");
+ dns_server_ip_addresses_and_busy_counts_.insert(
+ std::make_pair<in_addr_t,
unsigned>(ip_address, 0));
+ }
+
+ dns_servers_processed = true;
+ }
+ }
+
+ // As a last resort, attempt to get the resolver IP addresses from
/etc/resolv.conf:
+ if (not dns_servers_processed) {
+ std::vector<in_addr_t> server_ip_addresses;
+ Resolver::GetServersFromResolvDotConf(&server_ip_addresses);
+ for (std::vector<in_addr_t>::const_iterator
server_address(server_ip_addresses.begin());
+ server_address != server_ip_addresses.end();
++server_address)
+ dns_server_ip_addresses_and_busy_counts_.insert(
+ std::make_pair<in_addr_t,
unsigned>(*server_address, 0));
+
+ dns_servers_processed = true;
+ }
+
+ // Ensure we have at least one DNS Server:
+ if (dns_server_ip_addresses_and_busy_counts_.empty())
+ throw Exception("in Resolver::Resolver: no DNS Servers found");
+
+ initUdpSocket();
+}
+
+
+Resolver::Resolver(const std::string &dns_server, Logger * const logger, const
unsigned verbosity)
+ : verbosity_(logger != NULL ? verbosity : 0), logger_(logger),
cleanup_logger_(false)
+{
+ in_addr_t ip_address;
+ if (unlikely(not NetUtil::StringToNetworkAddress(dns_server,
&ip_address)))
+ throw Exception("in Resolver::Resolver: \"" + dns_server + "\"
is not a valid IP address (3)!");
+
+
dns_server_ip_addresses_and_busy_counts_.insert(std::make_pair<in_addr_t,
unsigned>(ip_address, 0));
+
+ initUdpSocket();
+}
+
+
+Resolver::Resolver(const in_addr_t dns_server, Logger * const logger, const
unsigned verbosity)
+ : verbosity_(logger != NULL ? verbosity : 0), logger_(logger),
cleanup_logger_(false)
+{
+
dns_server_ip_addresses_and_busy_counts_.insert(std::make_pair<in_addr_t,
unsigned>(dns_server, 0));
+
+ initUdpSocket();
+}
+
+
+Resolver::~Resolver()
+{
+ if (udp_fd_ != -1)
+ ::close(udp_fd_);
+ delete [] reply_packet_;
+
+ if (cleanup_logger_)
+ delete logger_;
+}
+
+
+void Resolver::initUdpSocket()
+{
+ udp_fd_ = ::socket(PF_INET, SOCK_DGRAM, 0);
+ if (unlikely(udp_fd_ == -1))
+ throw Exception("in Resolver::Resolver: socket(2) failed (" +
MsgUtil::ErrnoToString() + ")!");
+
+ // Turn off blocking because we are going to use select(2) which on
Linux doesn't reliably work with
+ // blocking file descriptors:
+ FileUtil::SetNonblocking(udp_fd_);
+
+ // Allocate a buffer to hold UDP DNS server replies and make sure that
it is 4-byte aligned:
+ const size_t MAX_UDP_REPLY_PACKET_SIZE(512);
+ reply_packet_ = reinterpret_cast<byte *>(new
uint32_t[(MAX_UDP_REPLY_PACKET_SIZE + sizeof(uint32_t) - 1)
+ /
sizeof(uint32_t)]);
+ reply_packet_size_ = MAX_UDP_REPLY_PACKET_SIZE;
+}
+
+
+namespace {
+
+
+// IsValidHostname -- sloppy implementation that only tests whether the
overall hostname is too long.
+//
+inline bool IsValidHostname(const std::string &hostname)
+{
+ return hostname.length() <= 255;
+}
+
+
+} // unnamed namespace
+
+
+void Resolver::submitRequest(const std::string &mixed_case_hostname)
+{
+ // If "name" is a dotted quad, we can just store it for future
reference by poll():
+ in_addr_t network_address;
+ if (NetUtil::StringToNetworkAddress(mixed_case_hostname,
&network_address)) {
+ std::set<in_addr_t> network_addresses;
+ network_addresses.insert(network_address);
+ resolved_addresses_.push_back(Result(RESOLVED,
mixed_case_hostname, network_addresses));
+ return;
+ }
+
+ const std::string hostname(StringUtil::ToLower(mixed_case_hostname));
+ if (unlikely(not IsValidHostname(hostname)))
+ throw Exception("in Resolver::submitRequest: \"" + hostname +
"\" is not a valid hostname!");
+
+ // Get the next request ID in a threadsafe manner:
+ uint16_t request_id;
+ {
+ ThreadUtil::MutexLocker mutex_locker(&resolver_mutex);
+ request_id = Resolver::next_request_id_;
+ ++Resolver::next_request_id_;
+ }
+
+ // Create the request packet:
+ unsigned char packet[512];
+ const ptrdiff_t packet_size(Resolver::GenerateRequestPacket(hostname,
request_id, packet));
+
+ // Find the least loaded server:
+ GNU_HASH_MAP<in_addr_t, unsigned>::const_iterator server_and_busy_count(
+ dns_server_ip_addresses_and_busy_counts_.begin());
+ in_addr_t least_loaded_server(server_and_busy_count->first);
+ unsigned lowest_busy_count_so_far(server_and_busy_count->second);
+ for (/* Empty! */; server_and_busy_count !=
dns_server_ip_addresses_and_busy_counts_.end();
+ ++server_and_busy_count)
+ {
+ if (server_and_busy_count->second < lowest_busy_count_so_far) {
+ lowest_busy_count_so_far =
server_and_busy_count->second;
+ least_loaded_server = server_and_busy_count->first;
+ }
+ }
+
+ // Now submit the request...
+ sendUdpRequest(least_loaded_server, packet,
static_cast<unsigned>(packet_size));
+
+ // ...and update the busy count of the server we used.
+ GNU_HASH_MAP<in_addr_t, unsigned>::iterator least_loaded_server_iter(
+
dns_server_ip_addresses_and_busy_counts_.find(least_loaded_server));
+ ++least_loaded_server_iter->second;
+}
+
+
+void Resolver::poll(std::list<Result> * const results)
+{
+ *results = resolved_addresses_;
+ resolved_addresses_.clear();
+
+ ssize_t read_retcode;
+ unsigned char packet[1000];
+ const TimeLimit time_limit(0);
+ while ((read_retcode = SocketUtil::TimedRead(udp_fd_, time_limit,
packet, sizeof(packet))) > 0) {
+ std::set<std::string> domainnames;
+ std::set<in_addr_t> ip_addresses;
+ uint32_t ttl;
+ uint16_t reply_id;
+ bool truncated;
+ if (DecodeReply(packet, read_retcode, &domainnames,
&ip_addresses, &ttl, &reply_id, &truncated, logger_, verbosity_)) {
+ // For now, we ignore truncated packets. We should
really send a TCP request instead:
+ if (truncated)
+ continue;
+
+ for (std::set<std::string>::const_iterator
domainname(domainnames.begin()); domainname!= domainnames.end(); ++domainname)
+ cache_.insert(*domainname, ip_addresses, ttl);
+ }
+ }
+}
+
+
+namespace { // helper functions for logging
+
+
+void LogIpAddresses(const std::string &domainname, const std::string &event,
const std::set<in_addr_t> &ip_addresses, const TimeLimit &time_limit,
+ Logger * const logger)
+{
+ if (logger == NULL)
+ return;
+
+ std::string result(domainname + " (" +
StringUtil::ToString(time_limit.getRemainingTime()) + "ms): " + event + ":");
+ for (std::set<in_addr_t>::const_iterator ip(ip_addresses.begin()); ip
!= ip_addresses.end(); ++ip)
+ result += " " + NetUtil::NetworkAddressToString(*ip);
+
+ logger->log(result);
+}
+
+
+void LogIpAddress(const std::string &domainname, const std::string &event,
+ const in_addr_t &ip_address, const TimeLimit &time_limit,
Logger * const logger)
+{
+
+ logger->log(domainname + " (" +
StringUtil::ToString(time_limit.getRemainingTime()) + "ms): "
+ + event + ": " +
NetUtil::NetworkAddressToString(ip_address));
+}
+
+
+} // unnamed namespace
+
+
+bool Resolver::resolve(const std::string &mixed_case_domainname, const
TimeLimit &time_limit, std::set<in_addr_t> * const ip_addresses)
+{
+ ip_addresses->clear();
+ const std::string
domainname(StringUtil::ToLower(mixed_case_domainname));
+
+ if (verbosity_ >= 3)
+ logger_->log("%s (%ums): Requested...", domainname.c_str(),
time_limit.getRemainingTime());
+
+ // Check to see whether "domainname" is already an IP address:
+ in_addr domainname_as_address;
+ if (::inet_aton(domainname.c_str(), &domainname_as_address)) {
+ ip_addresses->insert(domainname_as_address.s_addr);
+ if (verbosity_ >= 4)
+ LogIpAddress(domainname, "Already an address",
domainname_as_address.s_addr, time_limit, logger_);
+ return true;
+ }
+
+ // First see if we already know the answer to our query:
+ if (cache_.lookup(domainname, ip_addresses)) {
+ if (verbosity_ >= 4)
+ LogIpAddresses(domainname, "In cache", *ip_addresses,
time_limit, logger_);
+ return true;
+ }
+
+ // Decide which resolver we should use and get the next request ID in a
threadsafe manner:
+ uint16_t request_id;
+ static size_t next_resolver_index(::getpid());
+ in_addr_t selected_dns_server;
+ {
+ ThreadUtil::MutexLocker mutex_locker(&resolver_mutex);
+
+ // 1. Decide which resolver we should use:
+ const size_t
resolver_count(dns_server_ip_addresses_and_busy_counts_.size());
+ next_resolver_index = (next_resolver_index + 1) %
resolver_count;
+ GNU_HASH_MAP<in_addr_t, unsigned>::const_iterator
dns_server_ip_address_and_busy_count(
+ dns_server_ip_addresses_and_busy_counts_.begin());
+ for (unsigned i = 0; i < next_resolver_index; ++i)
+ ++dns_server_ip_address_and_busy_count;
+ selected_dns_server =
dns_server_ip_address_and_busy_count->first;
+ if (verbosity_ >= 5)
+ LogIpAddress(domainname, "DNS Server",
selected_dns_server, time_limit, logger_);
+
+ // 2. Get the next request ID in a threadsafe manner:
+ request_id = Resolver::next_request_id_;
+ ++Resolver::next_request_id_;
+ }
+
+ // Create the request packet:
+ unsigned char packet[512] __attribute__((aligned(sizeof(uint32_t))));
+ const ptrdiff_t packet_size(Resolver::GenerateRequestPacket(domainname,
request_id, packet));
+
+ // Now submit the request...
+ if (verbosity_ >= 5)
+ logger_->log("in resolve: about to send UPD request (ID %u).",
request_id);
+ sendUdpRequest(selected_dns_server, packet,
static_cast<unsigned>(packet_size));
+
+ // Now wait for the server's response:
+ for (;;) {
+ if (verbosity_ >= 5)
+ logger_->log("in resolve: remaining time %u ms",
time_limit.getRemainingTime());
+ ssize_t actual_packet_size(SocketUtil::TimedRead(udp_fd_,
time_limit, reply_packet_, reply_packet_size_));
+process_reply_packet:
+ if (actual_packet_size <= 0) {
+ if (errno == ETIMEDOUT) {
+ if (verbosity_ >= 4)
+ logger_->log("%s failed:
actual_packet_size <= 0, (timed out)", domainname.c_str());
+ return false;
+ }
+ else {
+ if (verbosity_ >= 4)
+ logger_->log("in resolve: resolving of
\"" + domainname + "\" failed: actual_packet_size <= 0, looping ("
+ + MsgUtil::ErrnoToString()
+ ")!");
+ continue;
+ }
+ }
+
+ std::set<std::string> resolved_domainnames;
+ uint32_t ttl;
+ uint16_t reply_id;
+ bool truncated;
+ if (DecodeReply(reply_packet_, actual_packet_size,
&resolved_domainnames, ip_addresses, &ttl, &reply_id, &truncated, logger_,
verbosity_))
+ {
+ if (ip_addresses->empty()) {
+ if (reply_id == request_id)
+ return false;
+ else
+ continue;
+ }
+
+ for (std::set<std::string>::const_iterator
resolved_domainname(resolved_domainnames.begin());
+ resolved_domainname != resolved_domainnames.end();
++resolved_domainname)
+ {
+ if (not truncated)
+ cache_.insert(*resolved_domainname,
*ip_addresses, ttl);
+ }
+
+ // Make sure we got the reply we were waiting for:
+ if (reply_id != request_id) {
+ if (verbosity_ >= 5)
+ logger_->log("in resolve: discarding
reply with unexpected ID: %u.", reply_id);
+ ip_addresses->clear();
+ continue;
+ }
+
+ if (truncated) { // => We attempt to get the
information using TCP instead of UDP.
+ if (verbosity_ >= 4)
+ logger_->log("in resolve: UDP packet
was truncated. Attempting TCP request.");
+ FileDescriptor
tcp_fd(sendTcpRequest(selected_dns_server, time_limit, packet,
static_cast<unsigned>(packet_size)));
+ if (tcp_fd == -1) {
+ if (verbosity_ >= 4)
+ logger_->log("in resolve:
failed to send request " + MsgUtil::ErrnoToString() + "!");
+ return false;
+ }
+
+ if (verbosity_ >= 5)
+ logger_->log("in resolve: remaining
time %u ms.", time_limit.getRemainingTime());
+ if (unlikely(not FileUtil::SetBlocking(tcp_fd)))
+ throw Exception("in Resolver::resolve:
FileUtil::SetBlocking() failed (" + MsgUtil::ErrnoToString() + ")!");
+ uint16_t reply_data_size;
+ SocketUtil::TimedRead(tcp_fd, time_limit,
&reply_data_size, sizeof(reply_data_size));
+ reply_data_size = ntohs(reply_data_size);
+ if (reply_data_size > reply_packet_size_) {
+ delete [] reply_packet_;
+ reply_packet_ = NULL;
+ reply_packet_ = reinterpret_cast<byte
*>(new uint32_t[(reply_data_size + sizeof(uint32_t) - 1) / sizeof(uint32_t)]);
+ reply_packet_size_ = reply_data_size;
+ }
+
+ actual_packet_size = reply_data_size;
+ ssize_t byte_got(0);
+ byte *buf_pointer = reply_packet_;
+
+ while (reply_data_size > 0 and (byte_got =
SocketUtil::TimedRead(tcp_fd, time_limit, buf_pointer, reply_data_size)) > 0) {
+ reply_data_size =
static_cast<uint16_t>(reply_data_size - byte_got);
+ buf_pointer += byte_got;
+ }
+
+ if (byte_got == -1) {
+ if (verbosity_ >= 4)
+ logger_->log("in resolve:
SocketUtil::TimedRead() failed: " + MsgUtil::ErrnoToString() + "!");
+ return false;
+ }
+ if (verbosity_ >= 5)
+ logger_->log("in resolve:
actual_packet_size = %zd.", actual_packet_size);
+
+ goto process_reply_packet;
+ }
+
+ if (verbosity_ >= 4)
+ LogIpAddresses(domainname, "Reply",
*ip_addresses, time_limit, logger_);
+ return true;
+ }
+ else {
+ if (verbosity_ >= 4)
+ logger_->log("%s Failed: could not decode
reply", domainname.c_str());
+ return false;
+ }
+ }
+}
+
+
+// Resolver::generateRequestPacket -- generates a DNS query packet. See
RFC1035 for details.
+//
+ptrdiff_t Resolver::GenerateRequestPacket(const std::string &hostname, const
uint16_t request_id, unsigned char * const packet)
+{
+ // Initialise the query header:
+ HEADER * const dns_header = reinterpret_cast<HEADER *>(packet);
+ std::memset(dns_header, '\0', sizeof(HEADER));
+ dns_header->id = htons(request_id);
+ dns_header->rd = 1; // Request recursion.
+ dns_header->qdcount = htons(1);
+
+ // Split the hostname into "labels" and append them to the packet
header:
+ std::list<std::string> labels;
+ StringUtil::Split(hostname, '.', &labels, /* suppress_empty_components
= */ false);
+ unsigned char *cp = packet + sizeof(HEADER);
+ for (std::list<std::string>::const_iterator label(labels.begin());
label != labels.end(); ++label) {
+ *cp++ = static_cast<unsigned char>(label->length());
+ for (const char *label_cp(label->c_str()); *label_cp != '\0';
++label_cp)
+ *cp++ = *label_cp;
+ }
+ *cp++ = 0;
+
+ uint16_t *sp = reinterpret_cast<uint16_t *>(cp);
+ *sp++ = htons(1); // Set the qtype field to type "A", indicating a host
address.
+ *sp++ = htons(1); // Set the qclass field to type "IN", indicating the
Internet.
+
+ return reinterpret_cast<unsigned char *>(sp) - packet;
+}
+
+
+void Resolver::sendUdpRequest(const in_addr_t resolver_ip_address, const
unsigned char * const packet, const unsigned packet_size) const
+{
+ if (unlikely(not SocketUtil::SendUdpRequest(udp_fd_,
resolver_ip_address, 53 /* DNS service port */, packet, packet_size)))
+ throw Exception("in Resolver::sendUdpRequest: sending a UDP
request failed (" + MsgUtil::ErrnoToString() + ")!");
+}
+
+
+int Resolver::sendTcpRequest(const in_addr_t resolver_ip_address, const
TimeLimit &time_limit, const unsigned char * const packet, const unsigned
packet_size) const
+{
+ SocketUtil::NagleOptionType nagle_option_type;
+#ifdef TCP_CORK
+ nagle_option_type = SocketUtil::USE_NAGLE;
+#else
+ nagle_option_type = SocketUtil::DISABLE_NAGLE;
+#endif
+ std::string error_message;
+ const int socket_fd(SocketUtil::TcpConnect(resolver_ip_address, 53,
time_limit, &error_message, nagle_option_type));
+ if (socket_fd == -1) {
+ if (verbosity_ >= 4)
+ logger_->log("in Resolver::sendTcpRequest:
SocketUtil::TcpConnect() failed (" + error_message + ")!");
+ return -1;
+ }
+
+#ifdef TCP_CORK
+ int optval(1);
+ if (unlikely(::setsockopt(socket_fd, SOL_TCP, TCP_CORK, &optval, sizeof
optval) == -1))
+ throw Exception("in Resolver::sendTcpRequest: setsockopt(2)
failed (1) (" + MsgUtil::ErrnoToString() + ")!");
+#endif
+
+ uint16_t buf_size((uint16_t)(htons(packet_size)));
+ if (SocketUtil::TimedWrite(socket_fd, time_limit, &buf_size,
sizeof(buf_size)) == -1) {
+ if (verbosity_ >= 4)
+ logger_->log("in Resolver::sendTcpRequest:
SocketUtil::TimedWrite() failed (" + MsgUtil::ErrnoToString() + ")!");
+ ::close(socket_fd);
+ return -1;
+ }
+
+ if (SocketUtil::TimedWrite(socket_fd, time_limit, packet, packet_size)
== -1) {
+ if (verbosity_ >= 4)
+ logger_->log("in Resolver::sendTcpRequest:
SocketUtil::TimedWrite() failed (" + MsgUtil::ErrnoToString() + ")!");
+ ::close(socket_fd);
+ return -1;
+ }
+
+#ifdef TCP_CORK
+ optval = 0;
+ if (unlikely(::setsockopt(socket_fd, SOL_TCP, TCP_CORK, &optval, sizeof
optval) == -1))
+ throw Exception("in Resolver::sendTcpRequest: setsockopt(2)
failed (2) (" + MsgUtil::ErrnoToString() + ")!");
+#endif
+
+ return socket_fd;
+}
+
+
+namespace {
+
+
+// ExtractDomainName -- extract a domain name from a DNS server response. See
RFC1035 for details.
+//
+// The trickiest part is that at any point in decoding we
can encounter what the RFC calls "message
+// compression". This is indicated that the two highest
bits of a 16 bit quantity are set. The
+// lower 14 bits then indicate an offset relative to the
start of the packet. Message compression
+// can occurr more than once withing one domainname.
+//
+std::string ExtractDomainName(const unsigned char * const packet_start, const
unsigned char *&cp)
+{
+ std::string domain_name;
+ bool first_label(true);
+ while (*cp != 0) {
+ if (not first_label)
+ domain_name += '.';
+ else
+ first_label = false;
+
+ // Message compression?
+ if ((*cp & 0xC0u) == 0xC0u) { // 2 high-order bits are set =>
message compression!
+ union {
+ unsigned char bytes_[2];
+ uint16_t u16_;
+ } overlay;
+ overlay.bytes_[0] = static_cast<unsigned char>(*cp &
~0xC0u);
+ ++cp;
+ overlay.bytes_[1] = *cp;
+ ++cp;
+ const uint16_t offset(ntohs(overlay.u16_));
+ const unsigned char *cp1(packet_start + offset);
+ return domain_name + ExtractDomainName(packet_start,
cp1);
+ }
+
+ const std::string::size_type label_length(*cp++);
+ domain_name += std::string(reinterpret_cast<const char *>(cp),
label_length);
+ cp += label_length;
+ }
+ ++cp; // Skip over trailing zero byte.
+
+ return domain_name;
+}
+
+
+} // unnamed namespace
+
+
+bool Resolver::DecodeReply(const unsigned char * const packet_start, const
size_t packet_size, std::set<std::string> * const domainnames,
+ std::set<in_addr_t> * const ip_addresses, uint32_t *
const ttl, uint16_t * const reply_id, bool * const truncated,
+ Logger * const logger, const unsigned _verbosity)
+{
+ // Set the actual verbosity:
+ const unsigned verbosity(logger != NULL ? _verbosity : 0);
+
+ if (unlikely(packet_size < sizeof(HEADER))) {
+ if (logger != NULL and verbosity > 2)
+ logger->log("in Resolver::DecodeReply: got a short
packet!");
+ return false;
+ }
+
+ domainnames->clear();
+ const HEADER *dns_header(reinterpret_cast<const HEADER
*>(packet_start));
+ *reply_id = ntohs(dns_header->id);
+ *truncated = dns_header->tc;
+ if (verbosity >= 5)
+ logger->log("in Resolver::DecodeReply: TC bit is %d",
(*truncated ? 1 : 0));
+ if (unlikely(*truncated))
+ return true; // Need to retry using TCP?
+
+ // We set the "recursion desired" bit and therefore also expect it in
our reply:
+ if (unlikely(dns_header->rd != 1))
+ return false;
+
+ switch (dns_header->rcode) {
+ case 0:
+ /* We succeeded! */
+ break;
+ case 1:
+ if (verbosity >= 2)
+ logger->log("in Resolver::DecodeReply: the server
indicated that we sent an invalid request!");
+ default:
+ /* Some kind of error condition that we ignore. */
+ return false;
+ }
+
+ const uint16_t qdcount = ntohs(dns_header->qdcount);
+ const uint16_t ancount = ntohs(dns_header->ancount);
+
+ // Skip over the question section:
+ const unsigned char *cp(packet_start + sizeof(HEADER));
+ for (unsigned question_record_index(0); question_record_index <
qdcount; ++question_record_index) {
+ const std::string
query_domainname(ExtractDomainName(packet_start, cp));
+ domainnames->insert(query_domainname);
+
+ //const uint16_t rr_type(ntohs(*reinterpret_cast<const uint16_t
*>(cp)));
+ cp += sizeof(uint16_t);
+
+ //const uint16_t rr_class(ntohs(*reinterpret_cast<const
uint16_t *>(cp)));
+ cp += sizeof(uint16_t);
+ }
+
+ for (unsigned resource_record_index(0); resource_record_index <
ancount; ++resource_record_index) {
+ const std::string rr_domainname(ExtractDomainName(packet_start,
cp));
+
+ const uint16_t rr_type(ntohs(*reinterpret_cast<const uint16_t
*>(cp)));
+ cp += sizeof(uint16_t);
+
+ const uint16_t rr_class(ntohs(*reinterpret_cast<const uint16_t
*>(cp)));
+ cp += sizeof(uint16_t);
+
+ *ttl = *reinterpret_cast<const uint32_t * const>(cp);
+ *ttl = ntohl(*ttl);
+ cp += sizeof(uint32_t);
+
+ const uint16_t rdlength(ntohs(*reinterpret_cast<const uint16_t
*>(cp)));
+ cp += sizeof(uint16_t);
+
+ if (rr_type == 1 and rr_class == 1) { // Type == A and class ==
IN.
+ if (rdlength != sizeof(in_addr_t)) {
+ if (verbosity >= 2)
+ logger->log("in Resolver::DecodeReply:
unexpected \"rdlength\" for A type RR record!");
+ return false;
+ }
+
+ domainnames->insert(rr_domainname);
+ in_addr_t ip_address;
+ std::memcpy(&ip_address, cp, sizeof(in_addr_t)); //
Intentionally returned in network byte order!
+ ip_addresses->insert(ip_address);
+ cp += rdlength;
+
+ if (verbosity >= 5)
+ logger->log("in Resolver::DecodeReply: %s: %s",
rr_domainname.c_str(),
+
NetUtil::NetworkAddressToString(ip_address).c_str());
+ }
+ else if (rr_type == 5 and rr_class == 1) { // Type == CNAME and
class == IN.
+ const std::string
cname_domainname(ExtractDomainName(packet_start, cp));
+ domainnames->insert(cname_domainname);
+ if (verbosity >= 5)
+ logger->log("in Resolver::DecodeReply: %s:
CNAME", rr_domainname.c_str());
+
+ }
+ else // Nothing we're interested in.
+ cp += rdlength; // Skip over the rdata field.
+ }
+
+ return true;
+}
+
+
+bool Resolver::ServerIsAlive(const std::string &server_ip_address, const
std::string &hostname_to_resolve,
+ const unsigned time_limit, Logger * const logger,
const unsigned verbosity)
+{
+ Resolver resolver(server_ip_address, logger, verbosity);
+
+ std::set<in_addr_t> ip_addresses;
+ if (not resolver.resolve(hostname_to_resolve, time_limit,
&ip_addresses))
+ return false;
+
+ return not ip_addresses.empty();
+}
+
+
+bool Resolver::ServerIsAlive(const in_addr_t server_ip_address, const
std::string &hostname_to_resolve,
+ const unsigned time_limit, Logger * const logger,
const unsigned verbosity)
+{
+ Resolver resolver(server_ip_address, logger, verbosity);
+
+ std::set<in_addr_t> ip_addresses;
+ if (not resolver.resolve(hostname_to_resolve, time_limit,
&ip_addresses))
+ return false;
+
+ return not ip_addresses.empty();
+}
+
+
+unsigned Resolver::GetServersFromResolvDotConf(std::vector<in_addr_t> * const
server_ip_addresses)
+{
+ server_ip_addresses->clear();
+
+ File resolv_conf("/etc/resolv.conf", "r");
+ if (resolv_conf.fail())
+ throw Exception("in Resolver::GetServersFromResolvDotConf:
can't open \"/etc/resolv.conf\" for reading!");
+
+ while (not resolv_conf.eof()) {
+ std::string line;
+ resolv_conf.getline(&line);
+ StringUtil::Trim(" \t", &line);
+ if (line.length() < 12)
+ continue;
+ if (line.substr(0, 10) == "nameserver") {
+ std::string resolver_ip_address;
+ resolver_ip_address = line.substr(10);
+ StringUtil::LeftTrim(&resolver_ip_address);
+
+ in_addr_t ip_address;
+ if (unlikely(not
NetUtil::StringToNetworkAddress(resolver_ip_address, &ip_address)))
+ throw Exception("in
Resolver::GetServersFromResolvDotConf: \"" + resolver_ip_address + "\" is not a
valid IP address (2)!");
+ server_ip_addresses->push_back(ip_address);
+ }
+ }
+
+ return static_cast<unsigned>(server_ip_addresses->size());
+}
+
+
+bool ThreadSafeDnsCache::lookup(const std::string &hostname,
std::set<in_addr_t> * const ip_addresses)
+{
+ // Synchronize access to the internal cache data structures:
+ ThreadUtil::MutexLocker mutex_locker(&cache_access_mutex_);
+
+ GNU_HASH_MAP<std::string, ThreadSafeDnsCacheEntry>::iterator
entry(resolved_hostnames_cache_.find(hostname));
+ if (entry != resolved_hostnames_cache_.end()) {
+ const time_t now(std::time(NULL));
+ if (entry->second.expire_time_ > now) {
+ *ip_addresses = entry->second.ip_addresses_;
+ return true;
+ }
+
+ // Entry has expired => remove it from the cache:
+ resolved_hostnames_cache_.erase(entry);
+ }
+
+ return false;
+}
+
+
+void ThreadSafeDnsCache::insert(const std::string &hostname, const
std::set<in_addr_t> &ip_addresses, const uint32_t ttl)
+{
+ // Synchronize access to the internal cache data structures:
+ ThreadUtil::MutexLocker mutex_locker(&cache_access_mutex_);
+
+ // Flush the cache if it contains more than 100,000 entries:
+ if (resolved_hostnames_cache_.size() > 100000)
+ resolved_hostnames_cache_.clear();
+ else {
+ // Check to see whether we already have information about this
"hostname":
+ GNU_HASH_MAP<std::string, ThreadSafeDnsCacheEntry>::iterator
cache_entry(resolved_hostnames_cache_.find(hostname));
+ if (unlikely(cache_entry != resolved_hostnames_cache_.end())) {
+ for (std::set<in_addr_t>::const_iterator
ip_address(ip_addresses.begin()); ip_address != ip_addresses.end();
++ip_address)
+
cache_entry->second.ip_addresses_.insert(*ip_address);
+ return;
+ }
+ }
+
+ // Create a new cache entry:
+ const time_t now(std::time(NULL));
+ const ThreadSafeDnsCache::ThreadSafeDnsCacheEntry new_cache_entry(now +
ttl, ip_addresses);
+ resolved_hostnames_cache_.insert(std::make_pair<std::string,
ThreadSafeDnsCacheEntry>(hostname, new_cache_entry));
+}
+
+
+SimpleResolver::SimpleResolver(const std::vector<std::string> &dns_servers)
+{
+ // Get the resolver IP addresses from the "dns_server" parameter:
+ if (not dns_servers.empty()) {
+ for (std::vector<std::string>::const_iterator
dns_server(dns_servers.begin());
+ dns_server != dns_servers.end(); ++dns_server)
+ {
+ in_addr_t ip_address;
+ if (unlikely(not
NetUtil::StringToNetworkAddress(*dns_server, &ip_address)))
+ throw Exception("in SimpleResolver::init: \"" +
*dns_server
+ + "\" is not a valid IP address
(1)!");
+ dns_server_ip_addresses_and_busy_counts_.push_back(
+ std::make_pair<in_addr_t, unsigned>(ip_address,
0));
+ }
+ }
+ else if (FileUtil::Exists(GetEtcDir() + "/Resolver.conf")) { // If a
Resolver.conf file exists, read it.
+ IniFile ini_file(GetEtcDir() + "/Resolver.conf");
+
+ // Read DNS servers:
+ if (ini_file.sectionIsDefined("DNS Servers")) {
+ // Add each entry in the [DNS Servers] section of
Resolver.conf:
+ const std::list<std::string>
names(ini_file.getSectionEntryNames("DNS Servers"));
+ for (std::list<std::string>::const_iterator
name(names.begin()); name != names.end(); ++name) {
+ const std::string
ip_address_str(ini_file.getString("DNS Servers", *name));
+ in_addr_t ip_address;
+ if (unlikely(not
NetUtil::StringToNetworkAddress(ip_address_str, &ip_address)))
+ throw Exception("Resolver.conf: \"" +
ip_address_str + "\""
+ " is not a valid IP
address (2)!");
+
dns_server_ip_addresses_and_busy_counts_.push_back(
+ std::make_pair<in_addr_t,
unsigned>(ip_address, 0));
+ }
+ }
+
+ if (unlikely(dns_server_ip_addresses_and_busy_counts_.empty()))
+ throw Exception("in SimpleResolver::init: failed to
find DNS server IP address in " + GetEtcDir() +
+ "/Resolver.conf!");
+ }
+ else { // As a last resort, attempt to get the resolver IP addresses
from /etc/resolv.conf.
+ std::ifstream resolv_conf("/etc/resolv.conf");
+ if (resolv_conf.fail())
+ throw Exception("in SimpleResolver::init: can't open
\"/etc/resolv.conf\" for reading!");
+
+ std::vector<std::string> resolvers;
+ while (resolv_conf) {
+ std::string line;
+ std::getline(resolv_conf, line);
+ StringUtil::Trim(" \t", &line);
+ if (line.length() < 12)
+ continue;
+ if (line.substr(0, 10) == "nameserver") {
+ std::string resolver_ip_address;
+ resolver_ip_address = line.substr(10);
+ StringUtil::LeftTrim(&resolver_ip_address);
+
+ in_addr_t ip_address;
+ if (unlikely(not
NetUtil::StringToNetworkAddress(resolver_ip_address, &ip_address)))
+ throw Exception("in
SimpleResolver::init: \"" + resolver_ip_address
+ + "\" is not a
valid IP address (2)!");
+
dns_server_ip_addresses_and_busy_counts_.push_back(
+ std::make_pair<in_addr_t,
unsigned>(ip_address, 0));
+ }
+ }
+
+ if (unlikely(dns_server_ip_addresses_and_busy_counts_.empty()))
+ throw Exception("in SimpleResolver::init: failed to
find DNS server IP address in /etc/resolv.conf!");
+ }
+
+ // Ensure we have at least one DNS Server:
+ if (dns_server_ip_addresses_and_busy_counts_.empty())
+ throw Exception("in SimpleResolver::init: no DNS Servers
found");
+}
+
+
+bool SimpleResolver::resolve(const std::string &hostname, const TimeLimit
&time_limit, std::set<in_addr_t> * const ip_addresses)
+{
+ ip_addresses->clear();
+
+ // Check to see whether "hostname" is already an IP address:
+ in_addr hostname_as_address;
+ if (::inet_aton(hostname.c_str(), &hostname_as_address)) {
+ ip_addresses->insert(hostname_as_address.s_addr);
+ return true;
+ }
+
+ // See if we already know the answer to our query:
+ if (dns_cache_.lookup(hostname, ip_addresses))
+ return true;
+
+ const FileDescriptor udp_fd(::socket(PF_INET, SOCK_DGRAM, 0));
+ if (unlikely(udp_fd == -1))
+ throw Exception("in SimpleResolver::resolve: socket(2) failed
(" + MsgUtil::ErrnoToString() + ")!");
+
+ // Turn off blocking because we are going to use select(2) which on
Linux doesn't reliably work with
+ // blocking file descriptors:
+ FileUtil::SetNonblocking(udp_fd);
+
+ // Decide which resolver we should use and get the next request ID in a
threadsafe manner:
+ const in_addr_t
resolver_address(getLeastBusyDnsServerAndIncUsageCount());
+
+ //
+ // Send the DNS request...
+ //
+
+ const uint16_t request_id(getNextRequestId());
+ unsigned char packet[512];
+ const ptrdiff_t packet_size(Resolver::GenerateRequestPacket(hostname,
request_id, packet));
+ if (unlikely(not SocketUtil::SendUdpRequest(udp_fd, resolver_address,
53 /* DNS service port */, packet, static_cast<unsigned>(packet_size))))
+ throw Exception("in SimpleResolver::resolve: sending a UDP
request failed (" + MsgUtil::ErrnoToString() + ")!");
+
+ //
+ // ...and now wait for a reply:
+ //
+
+ // Allocate a buffer on the stack to hold UDP DNS server replies and
make sure that it is 4-byte aligned:
+ const size_t MAX_UDP_REPLY_PACKET_SIZE(512);
+ uint32_t reply_packet_buffer[(MAX_UDP_REPLY_PACKET_SIZE +
sizeof(uint32_t) - 1) / sizeof(uint32_t)];
+ unsigned char * const reply_packet(reinterpret_cast<unsigned char
*>(reply_packet_buffer));
+ const size_t max_reply_packet_size(MAX_UDP_REPLY_PACKET_SIZE);
+
+ for (;;) {
+ const ssize_t
actual_reply_packet_size(SocketUtil::TimedRead(udp_fd, time_limit,
reply_packet, max_reply_packet_size));
+ if (actual_reply_packet_size <= 0) {
+ if (likely(errno == ETIMEDOUT))
+ return false;
+ throw Exception("in SimpleResolver::resolve:
SocketUtil::TimedRead() failed (" + MsgUtil::ErrnoToString() + ")!");
+ }
+ else if (processServerReply(reply_packet,
actual_reply_packet_size, request_id, ip_addresses))
+ break;
+ }
+
+ decDnsServerUsageCount(resolver_address);
+
+ return not ip_addresses->empty();
+}
+
+
+in_addr_t SimpleResolver::getLeastBusyDnsServerAndIncUsageCount()
+{
+ ThreadUtil::MutexLocker
mutex_locker(&dns_server_ip_addresses_and_busy_count_access_mutex_);
+
+ std::vector< std::pair<in_addr_t, unsigned> >::iterator
server_and_usage_count(
+ dns_server_ip_addresses_and_busy_counts_.begin());
+ std::vector< std::pair<in_addr_t, unsigned> >::iterator
least_busy_server(server_and_usage_count);
+ for (++server_and_usage_count; server_and_usage_count !=
dns_server_ip_addresses_and_busy_counts_.end();
+ ++server_and_usage_count)
+ {
+ if (server_and_usage_count->second < least_busy_server->second)
+ least_busy_server = server_and_usage_count;
+ }
+
+ // Update usage count...
+ ++least_busy_server->second;
+
+ // ...and return IP address.
+ return least_busy_server->first;
+}
+
+
+void SimpleResolver::decDnsServerUsageCount(const in_addr_t server_ip_address)
+{
+ ThreadUtil::MutexLocker
mutex_locker(&dns_server_ip_addresses_and_busy_count_access_mutex_);
+
+ // Look for the entry in "dns_server_ip_addresses_and_busy_counts_" and
decrement the busy count:
+ for (std::vector< std::pair<in_addr_t, unsigned> >::iterator
server_and_usage_count(
+ dns_server_ip_addresses_and_busy_counts_.begin());
+ server_and_usage_count !=
dns_server_ip_addresses_and_busy_counts_.end(); ++server_and_usage_count)
+ {
+ if (server_and_usage_count->first == server_ip_address) {
+ if (unlikely(server_and_usage_count->second == 0))
+ throw Exception("in
SimpleResolver::decDnsServerUsageCount: this should never happen!");
+ --server_and_usage_count->second;
+
+ return;
+ }
+ }
+
+ // If we get here we did not find a server entry matching
"server_ip_address" which should never happen!
+ throw Exception("in SimpleResolver::decDnsServerUsageCount: we should
never get here!");
+}
+
+
+uint16_t SimpleResolver::getNextRequestId()
+{
+ ThreadUtil::MutexLocker mutex_locker(&request_id_mutex_);
+
+ ++next_request_id_;
+ return next_request_id_;
+}
+
+
+bool SimpleResolver::processServerReply(const unsigned char * const
reply_packet, const size_t reply_packet_size, const uint16_t expected_reply_id,
+ std::set<in_addr_t> * const
ip_addresses)
+{
+ std::set<std::string> hostnames;
+ uint32_t ttl;
+ uint16_t reply_id;
+ bool truncated;
+ if (not Resolver::DecodeReply(reply_packet, reply_packet_size,
&hostnames, ip_addresses, &ttl,
+ &reply_id, &truncated)) // We received a
garbled reply packet and will ignore it!
+ return false;
+ else if (unlikely(reply_id != expected_reply_id)) // We most likely
received a reply to an earlier request
+ return false; // and will
ignore it!
+ else { // We're in luck.
+ if (not ip_addresses->empty()) {
+ // Update the DNS cache:
+ for (std::set<std::string>::const_iterator
hostname(hostnames.begin()); hostname != hostnames.end(); ++hostname)
+ dns_cache_.insert(*hostname, *ip_addresses,
ttl);
+ }
+
+ return true;
+ }
+}
Added: csplugins/trunk/ucsd/ruschein/GSFS/src/ThreadSafeLogger.cc
===================================================================
--- csplugins/trunk/ucsd/ruschein/GSFS/src/ThreadSafeLogger.cc
(rev 0)
+++ csplugins/trunk/ucsd/ruschein/GSFS/src/ThreadSafeLogger.cc 2011-09-20
16:52:23 UTC (rev 26879)
@@ -0,0 +1,53 @@
+/** \file ThreadSafeLogger.cc
+ * \brief Implementation of class ThreadSafeLogger.
+ * \author Dr. Johannes Ruscheinski
+ */
+
+/*
+ * Copyright 2006 Project iVia.
+ * Copyright 2006 The Regents of The University of California.
+ *
+ * This file is part of the libiViaCore package.
+ *
+ * The libiViaCore package is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License as
published
+ * by the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * libiViaCore is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with libiViaCore; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#include <ThreadSafeLogger.h>
+#ifndef CERRNO
+# include <cerrno>
+# define CERRNO
+#endif
+#ifndef CSTRING
+# include <cstring>
+# define CSTRING
+#endif
+#ifndef UNISTD_H
+# include <unistd.h>
+# define UNISTD_H
+#endif
+#ifndef MSG_UTIL_H
+# include <MsgUtil.h>
+#endif
+#ifndef TIME_UTIL_H
+# include <TimeUtil.h>
+#endif
+
+
+void ThreadSafeLogger::writeLog(const std::string &message, const unsigned
log_mask)
+{
+ ThreadUtil::MutexLocker mutex_locker(&mutex_);
+
+ Logger::writeLog(message, log_mask);
+}
Modified: csplugins/trunk/ucsd/ruschein/GSFS/src/WebUtil.cc
===================================================================
--- csplugins/trunk/ucsd/ruschein/GSFS/src/WebUtil.cc 2011-09-20 16:29:34 UTC
(rev 26878)
+++ csplugins/trunk/ucsd/ruschein/GSFS/src/WebUtil.cc 2011-09-20 16:52:23 UTC
(rev 26879)
@@ -1578,4 +1578,16 @@
}
+// ExtractURLs -- extracts all links from "document_source" and returns them
in "urls". "root_url" is used to turn relative URLs into absolute URLs if
+// requested.
+//
+void ExtractURLs(const std::string &/*document_source*/, const std::string
&/*default_base_url*/,
+ const ExtractedUrlForm /*extracted_url_form*/,
std::vector<UrlAndAnchorTexts> * const /*urls_and_anchor_texts*/,
+ const unsigned /*flags*/, const std::string
&/*user_agent_string*/, const unsigned /*page_cacher_max_fanout*/,
+ const unsigned /*individual_page_timeout*/, unsigned long *
const /*overall_timeout*/)
+{
+ throw std::runtime_error("WebUtil::ExtractURLs() has not been
implemented!");
+}
+
+
} // namespace WebUtil
--
You received this message because you are subscribed to the Google Groups
"cytoscape-cvs" group.
To post to this group, send email to [email protected].
To unsubscribe from this group, send email to
[email protected].
For more options, visit this group at
http://groups.google.com/group/cytoscape-cvs?hl=en.
