Support dynamic adaptation plans that cover multiple vectoring points.
The dynamic adaptation plan is specified using X-Next-Services ICAP
header or eCAP meta-info, as usual. A REQMOD adaptation service may
construct an adaptation plan that starts with REQMOD and ends with
RESPMOD. Multiple adaptations may be planned at each point.
The natural transaction handling order must be preserved: the plan
cannot go from RESPMOD back to REQMOD.
Adaptation::History object is used to keep future plan steps when
crossing vectoring points.
This change enables REQMOD services to schedule request-specific
response adaptation. For example, when the request is from a child,
appropriate content filtering needs to be applied to the response.
In some cases, such decisions can be done in squid.conf, but the
request state required to make the schedule may be too complex or even
not known to Squid core. In most cases, such decisions can be done in
the RESPMOD service (which always has access to request headers), but
that requires sending _all_ responses to that service unconditionally,
which often creates significant performance penalties. In both
scenarios, an adaptation schedule build by a REQMOD service may solve
the problem nicely.
AFAIK, the 3.1 code this patch is based on has been deployed for many
months without known problems.
Thank you,
Alex.
Support dynamic adaptation plans that cover multiple vectoring points.
The dynamic adaptation plan is specified using X-Next-Services ICAP header or
eCAP meta-info, as usual. A REQMOD adaptation service may construct an
adaptation plan that starts with REQMOD and ends with RESPMOD. Multiple
adaptations may be planned at each point.
The natural transaction handling order must be preserved: the plan cannot go
from RESPMOD back to REQMOD.
Adaptation::History object is used to keep future plan steps when crossing
vectoring points.
Ported from lp:3p1-rock branch (r9566+).
=== modified file 'src/adaptation/AccessCheck.cc'
--- src/adaptation/AccessCheck.cc 2010-11-28 15:29:51 +0000
+++ src/adaptation/AccessCheck.cc 2011-03-15 21:52:08 +0000
@@ -48,41 +48,65 @@
debugs(93, 5, HERE << "AccessCheck constructed for " <<
methodStr(filter.method) << " " << vectPointStr(filter.point));
}
Adaptation::AccessCheck::~AccessCheck()
{
#if ICAP_CLIENT
Adaptation::Icap::History::Pointer h = filter.request->icapHistory();
if (h != NULL)
h->stop("ACL");
#endif
if (callback_data)
cbdataReferenceDone(callback_data);
}
void
Adaptation::AccessCheck::start()
{
AsyncJob::start();
- check();
+
+ if (!usedDynamicRules())
+ check();
+}
+
+/// returns true if previous services configured dynamic chaining "rules"
+bool
+Adaptation::AccessCheck::usedDynamicRules()
+{
+ Adaptation::History::Pointer ah = filter.request->adaptHistory();
+ if (!ah)
+ return false; // dynamic rules not enabled or not triggered
+
+ DynamicGroupCfg services;
+ if (!ah->extractFutureServices(services)) { // clears history
+ debugs(85,9, HERE << "no service-proposed rules stored");
+ return false; // earlier service did not plan for the future
+ }
+
+ debugs(85,3, HERE << "using stored service-proposed rules: " << services);
+
+ ServiceGroupPointer g = new DynamicServiceChain(services, filter);
+ callBack(g);
+ Must(done());
+ return true;
}
/// Walk the access rules list to find rules with applicable service groups
void
Adaptation::AccessCheck::check()
{
debugs(93, 4, HERE << "start checking");
typedef AccessRules::iterator ARI;
for (ARI i = AllRules().begin(); i != AllRules().end(); ++i) {
AccessRule *r = *i;
if (isCandidate(*r)) {
debugs(93, 5, HERE << "check: rule '" << r->id << "' is a candidate");
candidates += r->id;
}
}
checkCandidates();
}
=== modified file 'src/adaptation/AccessCheck.h'
--- src/adaptation/AccessCheck.h 2009-07-16 18:40:00 +0000
+++ src/adaptation/AccessCheck.h 2011-03-15 21:52:08 +0000
@@ -38,29 +38,30 @@
typedef int Candidate;
typedef Vector<Candidate> Candidates;
Candidates candidates;
Candidate topCandidate() const { return *candidates.begin(); }
ServiceGroupPointer topGroup() const; // may return nil
void callBack(const ServiceGroupPointer &g);
bool isCandidate(AccessRule &r);
public:
void checkCandidates();
static void AccessCheckCallbackWrapper(int, void*);
void noteAnswer(int answer);
protected:
// AsyncJob API
virtual void start();
virtual bool doneAll() const { return false; } /// not done until mustStop
+ bool usedDynamicRules();
void check();
private:
CBDATA_CLASS2(AccessCheck);
};
} // namespace Adaptation
#endif /* SQUID_ADAPTATION__ACCESS_CHECK_H */
=== added file 'src/adaptation/DynamicGroupCfg.cc'
--- src/adaptation/DynamicGroupCfg.cc 1970-01-01 00:00:00 +0000
+++ src/adaptation/DynamicGroupCfg.cc 2011-03-15 21:56:15 +0000
@@ -0,0 +1,22 @@
+#include "config.h"
+
+#include "adaptation/DynamicGroupCfg.h"
+
+void
+Adaptation::DynamicGroupCfg::add(const String &item)
+{
+ if (services.empty()) { // first item
+ id = item;
+ } else {
+ id.append(',');
+ id.append(item);
+ }
+ services.push_back(item);
+}
+
+void
+Adaptation::DynamicGroupCfg::clear()
+{
+ id.clean();
+ services.clean();
+}
=== added file 'src/adaptation/DynamicGroupCfg.h'
--- src/adaptation/DynamicGroupCfg.h 1970-01-01 00:00:00 +0000
+++ src/adaptation/DynamicGroupCfg.h 2011-03-15 22:02:36 +0000
@@ -0,0 +1,34 @@
+#ifndef SQUID_ADAPTATION__DYNAMIC_GROUP_CFG_H
+#define SQUID_ADAPTATION__DYNAMIC_GROUP_CFG_H
+
+#include "Array.h"
+#include "SquidString.h"
+
+namespace Adaptation
+{
+
+/// DynamicServiceGroup configuration to remember future dynamic chains
+class DynamicGroupCfg
+{
+public:
+ typedef Vector<String> Store;
+ typedef String Id;
+
+ Id id; ///< group id
+ Store services; ///< services in the group
+
+ bool empty() const { return services.empty(); } ///< no services added
+ void add(const String &item); ///< updates group id and services
+ void clear(); ///< makes the config empty
+};
+
+inline
+std::ostream &operator <<(std::ostream &os, const DynamicGroupCfg &cfg)
+{
+ return os << cfg.id;
+}
+
+} // namespace Adaptation
+
+#endif /* SQUID_ADAPTATION__DYNAMIC_GROUP_CFG_H */
+
=== modified file 'src/adaptation/History.cc'
--- src/adaptation/History.cc 2011-02-18 19:39:05 +0000
+++ src/adaptation/History.cc 2011-03-15 21:52:08 +0000
@@ -131,20 +131,39 @@
}
bool Adaptation::History::extractNextServices(String &value)
{
if (theNextServices == TheNullServices)
return false;
value = theNextServices;
theNextServices = TheNullServices; // prevents resetting the plan twice
return true;
}
void Adaptation::History::recordMeta(const HttpHeader *lm)
{
lastMeta.clean();
lastMeta.update(lm, NULL);
allMeta.update(lm, NULL);
allMeta.compact();
}
+
+void
+Adaptation::History::setFutureServices(const DynamicGroupCfg &services)
+{
+ if (!theFutureServices.empty())
+ debugs(93,3, HERE << "old future services: " << theFutureServices);
+ debugs(93,3, HERE << "new future services: " << services);
+ theFutureServices = services; // may be empty
+}
+
+bool Adaptation::History::extractFutureServices(DynamicGroupCfg &value)
+{
+ if (theFutureServices.empty())
+ return false;
+
+ value = theFutureServices;
+ theFutureServices.clear();
+ return true;
+}
=== modified file 'src/adaptation/History.h'
--- src/adaptation/History.h 2011-02-18 19:39:05 +0000
+++ src/adaptation/History.h 2011-03-15 22:03:08 +0000
@@ -1,23 +1,24 @@
#ifndef SQUID_ADAPT_HISTORY_H
#define SQUID_ADAPT_HISTORY_H
+#include "adaptation/DynamicGroupCfg.h"
#include "Array.h"
#include "HttpHeader.h"
#include "RefCount.h"
#include "SquidString.h"
namespace Adaptation
{
/// collects information about adaptations related to a master transaction
class History: public RefCountable
{
public:
typedef RefCount<Adaptation::History> Pointer;
History();
/// record the start of a xact, return xact history ID
int recordXactStart(const String &serviceId, const timeval &when, bool retrying);
@@ -34,54 +35,61 @@
void updateXxRecord(const char *name, const String &value);
/// returns true and fills the record fields iff there is a db record
bool getXxRecord(String &name, String &value) const;
/// sets or resets next services for the Adaptation::Iterator to notice
void updateNextServices(const String &services);
/// returns true, fills the value, and resets iff next services were set
bool extractNextServices(String &value);
/// store the last meta header fields received from the adaptation service
void recordMeta(const HttpHeader *lm);
public:
/// Last received meta header (REQMOD or RESPMOD, whichever comes last).
HttpHeader lastMeta;
/// All REQMOD and RESPMOD meta headers merged. Last field wins conflicts.
HttpHeader allMeta;
+ /// sets future services for the Adaptation::AccessCheck to notice
+ void setFutureServices(const DynamicGroupCfg &services);
+
+ /// returns true, fills the value, and resets iff future services were set
+ bool extractFutureServices(DynamicGroupCfg &services);
+
private:
/// single Xaction stats (i.e., a historical record entry)
class Entry
{
public:
Entry(const String &serviceId, const timeval &when);
Entry(); // required by Vector<>
void stop(); ///< updates stats on transaction end
int rptm(); ///< returns response time [msec], calculates it if needed
String service; ///< adaptation service ID
timeval start; ///< when the xaction was started
private:
int theRptm; ///< calculated and cached response time value in msec
public:
bool retried; ///< whether the xaction was replaced by another
};
typedef Vector<Entry> Entries;
Entries theEntries; ///< historical record, in the order of xact starts
// theXx* will become a map<string,string>, but we only support one record
String theXxName; ///< name part of the cross-transactional database record
String theXxValue; ///< value part of the cross-xactional database record
String theNextServices; ///< services Adaptation::Iterator must use next
+ DynamicGroupCfg theFutureServices; ///< services AccessCheck must use
};
} // namespace Adaptation
#endif
=== modified file 'src/adaptation/Iterator.cc'
--- src/adaptation/Iterator.cc 2011-03-11 23:02:23 +0000
+++ src/adaptation/Iterator.cc 2011-03-15 21:52:08 +0000
@@ -192,42 +192,53 @@
Must(r);
Adaptation::History::Pointer ah = r->adaptHistory();
if (!ah) {
debugs(85,9, HERE << "no history to store a service-proposed plan");
return false; // the feature is not enabled or is not triggered
}
String services;
if (!ah->extractNextServices(services)) { // clears history
debugs(85,9, HERE << "no service-proposed plan received");
return false; // the service did not provide a new plan
}
if (!adopt) {
debugs(85,3, HERE << "rejecting service-proposed plan");
return false;
}
debugs(85,3, HERE << "retiring old plan: " << thePlan);
- theGroup = new DynamicServiceChain(services, theGroup); // refcounted
- thePlan = ServicePlan(theGroup, filter());
+
+ Adaptation::ServiceFilter filter = this->filter();
+ DynamicGroupCfg current, future;
+ DynamicServiceChain::Split(filter, services, current, future);
+
+ if (!future.empty()) {
+ ah->setFutureServices(future);
+ debugs(85,3, HERE << "noted future service-proposed plan: " << future);
+ }
+
+ // use the current config even if it is empty; we must replace the old plan
+ theGroup = new DynamicServiceChain(current, filter); // refcounted
+ thePlan = ServicePlan(theGroup, filter);
debugs(85,3, HERE << "adopted service-proposed plan: " << thePlan);
return true;
}
Adaptation::ServiceFilter Adaptation::Iterator::filter() const
{
// the method may differ from theGroup->method due to request satisfaction
Method method = methodNone;
// temporary variables, no locking needed
HttpRequest *req = NULL;
HttpReply *rep = NULL;
if (HttpRequest *r = dynamic_cast<HttpRequest*>(theMsg)) {
method = methodReqmod;
req = r;
rep = NULL;
} else if (HttpReply *theReply = dynamic_cast<HttpReply*>(theMsg)) {
method = methodRespmod;
req = theCause;
rep = theReply;
=== modified file 'src/adaptation/Makefile.am'
--- src/adaptation/Makefile.am 2011-03-11 23:02:23 +0000
+++ src/adaptation/Makefile.am 2011-03-15 21:52:08 +0000
@@ -7,40 +7,42 @@
if USE_ICAP_CLIENT
SUBDIRS += icap
endif
if USE_ECAP
SUBDIRS += ecap
endif
noinst_LTLIBRARIES = libadaptation.la
## start with the code shared among all adaptation schemes
libadaptation_la_SOURCES = \
AccessCheck.cc \
AccessCheck.h \
AccessRule.cc \
AccessRule.h \
Answer.cc \
Answer.h \
Config.cc \
Config.h \
+ DynamicGroupCfg.cc \
+ DynamicGroupCfg.h \
Elements.cc \
Elements.h \
forward.h \
Initiate.cc \
Initiate.h \
Initiator.cc \
Initiator.h \
Iterator.cc \
Iterator.h \
Message.cc \
Message.h \
Service.cc \
Service.h \
ServiceConfig.cc \
ServiceConfig.h \
ServiceGroups.cc \
ServiceGroups.h \
ServiceFilter.cc \
ServiceFilter.h \
History.cc \
=== modified file 'src/adaptation/ServiceGroups.cc'
--- src/adaptation/ServiceGroups.cc 2009-11-18 16:31:08 +0000
+++ src/adaptation/ServiceGroups.cc 2011-03-15 21:52:08 +0000
@@ -1,31 +1,30 @@
#include "squid.h"
#include "ConfigParser.h"
-#include "Array.h" // really Vector
#include "adaptation/Config.h"
#include "adaptation/AccessRule.h"
+#include "adaptation/DynamicGroupCfg.h"
#include "adaptation/Service.h"
#include "adaptation/ServiceFilter.h"
#include "adaptation/ServiceGroups.h"
-#define ServiceGroup ServiceGroup
Adaptation::ServiceGroup::ServiceGroup(const String &aKind, bool allSame):
kind(aKind), method(methodNone), point(pointNone),
allServicesSame(allSame)
{
}
Adaptation::ServiceGroup::~ServiceGroup()
{
}
void
Adaptation::ServiceGroup::parse()
{
ConfigParser::ParseString(&id);
wordlist *names = NULL;
ConfigParser::ParseWordList(&names);
for (wordlist *i = names; i; i = i->next)
services.push_back(i->key);
@@ -192,62 +191,85 @@
}
/* SingleService */
Adaptation::SingleService::SingleService(const String &aServiceId):
ServiceGroup("single-service group", false)
{
id = aServiceId;
services.push_back(aServiceId);
}
/* ServiceChain */
Adaptation::ServiceChain::ServiceChain(): ServiceGroup("adaptation chain", false)
{
}
-/* ServiceChain */
+/* DynamicServiceChain */
-Adaptation::DynamicServiceChain::DynamicServiceChain(const String &ids,
- const ServiceGroupPointer prev)
+Adaptation::DynamicServiceChain::DynamicServiceChain(
+ const DynamicGroupCfg &cfg, const ServiceFilter &filter)
{
kind = "dynamic adaptation chain"; // TODO: optimize by using String const
- id = ids; // use services ids as the dynamic group ID
+ id = cfg.id; // use services ids as the dynamic group ID
+ services = cfg.services;
// initialize cache to improve consistency checks in finalize()
- if (prev != NULL) {
- method = prev->method;
- point = prev->point;
- }
+ method = filter.method;
+ point = filter.point;
+
+ finalize(); // will report [dynamic] config errors
+}
- // populate services storage with supplied service ids
+void
+Adaptation::DynamicServiceChain::Split(const ServiceFilter &filter,
+ const String &ids, DynamicGroupCfg ¤t,
+ DynamicGroupCfg &future)
+{
+ // walk the list of services and split it into two parts:
+ // services that are applicable now and future services
+ bool doingCurrent = true;
const char *item = NULL;
int ilen = 0;
const char *pos = NULL;
- while (strListGetItem(&ids, ',', &item, &ilen, &pos))
- services.push_back(item);
+ while (strListGetItem(&ids, ',', &item, &ilen, &pos)) {
+ String id;
+ id.limitInit(item, ilen);
+ ServicePointer service = FindService(id);
+ if (doingCurrent) {
+ if (!service || // cannot tell or matches current location
+ (service->cfg().method == filter.method &&
+ service->cfg().point == filter.point)) {
+ current.add(id);
+ continue;
+ } else {
+ doingCurrent = false;
+ }
+ }
- finalize(); // will report [dynamic] config errors
+ if (!doingCurrent)
+ future.add(id);
+ }
}
/* ServicePlan */
Adaptation::ServicePlan::ServicePlan(): pos(0), atEof(true)
{
}
Adaptation::ServicePlan::ServicePlan(const ServiceGroupPointer &g,
const ServiceFilter &filter):
group(g), pos(0), atEof(!g || !g->has(pos))
{
// this will find the first service because starting pos is zero
if (!atEof && !group->findService(filter, pos))
atEof = true;
}
Adaptation::ServicePointer
Adaptation::ServicePlan::current() const
{
=== modified file 'src/adaptation/ServiceGroups.h'
--- src/adaptation/ServiceGroups.h 2009-08-23 09:30:49 +0000
+++ src/adaptation/ServiceGroups.h 2011-03-15 21:52:08 +0000
@@ -83,41 +83,45 @@
protected:
virtual bool replace(Pos &pos) const { return false; }
virtual bool advance(Pos &pos) const { return false; }
};
/// a group of services that must be used one after another
class ServiceChain: public ServiceGroup
{
public:
ServiceChain();
protected:
virtual bool replace(Pos &pos) const { return false; }
virtual bool advance(Pos &pos) const { return has(++pos); }
};
/// a temporary service chain built upon another service request
class DynamicServiceChain: public ServiceChain
{
public:
- DynamicServiceChain(const String &srvcs, const ServiceGroupPointer prev);
+ DynamicServiceChain(const DynamicGroupCfg &cfg, const ServiceFilter &f);
+
+ /// separates dynamic services matching current location from future ones
+ static void Split(const ServiceFilter &filter, const String &ids,
+ DynamicGroupCfg ¤t, DynamicGroupCfg &future);
};
/** iterates services stored in a group; iteration is not linear because we
need to both replace failed services and advance to the next chain link */
class ServicePlan
{
public:
typedef unsigned int Pos; // Vector<>::poistion_type
public:
ServicePlan();
explicit ServicePlan(const ServiceGroupPointer &g, const ServiceFilter &filter);
///< true iff there are no more services planned
bool exhausted() const { return atEof; }
/// returns nil if the plan is complete
ServicePointer current() const; ///< current service
ServicePointer replacement(const ServiceFilter &filter); ///< next to try after failure
=== modified file 'src/adaptation/forward.h'
--- src/adaptation/forward.h 2010-12-18 00:31:53 +0000
+++ src/adaptation/forward.h 2011-03-15 21:52:08 +0000
@@ -1,36 +1,37 @@
#ifndef SQUID_ADAPTATION__FORWARD_H
#define SQUID_ADAPTATION__FORWARD_H
// forward-declarations for commonly used adaptation classes
template <class C>
class RefCount;
// For various collections such as AllServices
// TODO: use std::hash_map<> instead
template <class Item>
class Vector;
namespace Adaptation
{
class Service;
class ServiceConfig;
+class DynamicGroupCfg;
class Class;
class Initiate;
class Initiator;
class AccessCheck;
class AccessRule;
class ServiceGroup;
class ServicePlan;
class ServiceFilter;
class Message;
class Answer;
typedef RefCount<Service> ServicePointer;
typedef RefCount<ServiceConfig> ServiceConfigPointer;
typedef RefCount<ServiceGroup> ServiceGroupPointer;
} // namespace Adaptation
#endif /* SQUID_ADAPTATION__FORWARD_H */
