I took the url_rewrite helper and used what ever seems needed to create
a url_store_rewrite helper.
I also added the needed variables for the next step and freeing them
where and if needed.
this is a more of a basic sketch.
Eliezer
=== modified file 'src/ClientRequestContext.h'
--- src/ClientRequestContext.h 2012-08-14 11:53:07 +0000
+++ src/ClientRequestContext.h 2012-09-08 14:32:13 +0000
@@ -33,6 +33,10 @@
void clientAccessCheckDone(const allow_t &answer);
void clientRedirectStart();
void clientRedirectDone(char *result);
+ //adding for storeurl
+ void clientStoreurlStart();
+ void clientStoreurlDone(char *result);
+ //end add for storeurl
void checkNoCache();
void checkNoCacheDone(const allow_t &answer);
#if USE_ADAPTATION
@@ -53,7 +57,11 @@
ClientHttpRequest *http;
ACLChecklist *acl_checklist; /* need ptr back so we can unreg if needed */
int redirect_state;
+ //added for storeurl
+ int storeurl_state;
+ bool storeurl_done;
+ //end add for storeurl
bool host_header_verify_done;
bool http_access_done;
bool adapted_http_access_done;
=== modified file 'src/client_side.cc'
--- src/client_side.cc 2012-09-06 13:12:26 +0000
+++ src/client_side.cc 2012-09-08 14:44:38 +0000
@@ -714,6 +714,7 @@
ClientHttpRequest::freeResources()
{
safe_free(uri);
+ safe_free(storeurl); //adding for store_url
safe_free(log_uri);
safe_free(redirect.location);
range_iter.boundary.clean();
=== modified file 'src/client_side_request.cc'
--- src/client_side_request.cc 2012-08-28 19:12:13 +0000
+++ src/client_side_request.cc 2012-09-09 09:59:52 +0000
@@ -131,6 +131,7 @@
static int clientHierarchical(ClientHttpRequest * http);
static void clientInterpretRequestHeaders(ClientHttpRequest * http);
static RH clientRedirectDoneWrapper;
+static RH clientStoreurlDoneWrapper; //adding for storeurl
static void checkNoCacheDoneWrapper(allow_t, void *);
extern "C" CSR clientGetMoreData;
extern "C" CSS clientReplyStatus;
@@ -151,10 +152,11 @@
debugs(85,3, HERE << this << " ClientRequestContext destructed");
}
-ClientRequestContext::ClientRequestContext(ClientHttpRequest *anHttp) : http(cbdataReference(anHttp)), acl_checklist (NULL), redirect_state (REDIRECT_NONE), error(NULL), readNextRequest(false)
+ClientRequestContext::ClientRequestContext(ClientHttpRequest *anHttp) : http(cbdataReference(anHttp)), acl_checklist (NULL), redirect_state (REDIRECT_NONE), storeurl_state (REDIRECT_NONE), error(NULL), readNextRequest(false)
{
http_access_done = false;
redirect_done = false;
+ storeurl_done = false; //added for storeurl and also at the method line
no_cache_done = false;
interpreted_req_hdrs = false;
#if USE_SSL
@@ -915,6 +917,32 @@
} else
redirectStart(http, clientRedirectDoneWrapper, this);
}
+//adding for storeurl
+static void
+clientStoreurlAccessCheckDone(allow_t answer, void *data)
+{
+ ClientRequestContext *context = (ClientRequestContext *)data;
+ ClientHttpRequest *http = context->http;
+ context->acl_checklist = NULL;
+
+ if (answer == ACCESS_ALLOWED)
+ storeurlStart(http, clientStoreurlDoneWrapper, context);
+ else
+ context->clientStoreurlDone(NULL);
+}
+
+void
+ClientRequestContext::clientStoreurlStart()
+{
+ debugs(33, 5, HERE << "' ClientRequestContext::clientStoreurlStart " << http->uri << "'");
+
+ if (Config.accessList.storeurl) {
+ acl_checklist = clientAclChecklistCreate(Config.accessList.storeurl, http);
+ acl_checklist->nonBlockingCheck(clientStoreurlAccessCheckDone, this);
+ } else
+ storeurlStart(http, clientStoreurlDoneWrapper, this);
+}
+//end add for storeurl add
static int
clientHierarchical(ClientHttpRequest * http)
@@ -1261,6 +1289,40 @@
http->doCallouts();
}
+//adding for storeurl
+
+void
+clientStoreurlDoneWrapper(void *data, char *result)
+{
+ ClientRequestContext *calloutContext = (ClientRequestContext *)data;
+
+ if (!calloutContext->httpStateIsValid())
+ return;
+
+ calloutContext->clientStoreurlDone(result);
+}
+
+void
+ClientRequestContext::clientStoreurlDone(char *result)
+{
+ debugs(85, 5, "clientStoreurlDone: '" << http->uri << "' result=" << (result ? result : "NULL"));
+ assert(storeurl_state == REDIRECT_PENDING);
+ storeurl_state = REDIRECT_DONE;
+
+ /*
+ * the next step is to implement here the actuall store_url_rewrite proccess of pushing
+ * the store_url into thr http request.
+ */
+ if (result) {
+ debugs(85, DBG_CRITICAL, "A result for storu_url accepted: " << result);
+ } else {
+ debugs(85, DBG_CRITICAL, "A result for storu_url was not accepted: " << result);
+ }
+
+
+ http->doCallouts();
+}
+//end add for storeurl
/** Test cache allow/deny configuration
* Sets flags.cachable=1 if caching is not denied.
@@ -1584,6 +1646,19 @@
}
}
+ if (!calloutContext->storeurl_done) {
+ calloutContext->storeurl_done = true;
+ assert(calloutContext->storeurl_state == REDIRECT_NONE);
+
+ if (Config.Program.storeurl) {
+ debugs(83, 3, HERE << "Doing calloutContext->clientSotreurlStart()");
+ calloutContext->storeurl_state = REDIRECT_PENDING;
+ calloutContext->clientStoreurlStart();
+ return;
+ }
+ }
+
+
if (!calloutContext->adapted_http_access_done) {
debugs(83, 3, HERE << "Doing calloutContext->clientAccessCheck2()");
calloutContext->adapted_http_access_done = true;
@@ -1597,6 +1672,8 @@
clientInterpretRequestHeaders(this);
}
+
+
if (!calloutContext->no_cache_done) {
calloutContext->no_cache_done = true;
=== modified file 'src/client_side_request.h'
--- src/client_side_request.h 2012-08-28 19:12:13 +0000
+++ src/client_side_request.h 2012-09-09 09:45:40 +0000
@@ -98,6 +98,7 @@
HttpRequest *request; /* Parsed URL ... */
char *uri;
char *log_uri;
+ char *storeurl; //added for storeurl, need to add a safe_free for that
struct {
int64_t offset;
@@ -210,6 +211,8 @@
extern void redirectStart(ClientHttpRequest *, RH *, void *);
extern void tunnelStart(ClientHttpRequest *, int64_t *, int *);
+extern void storeurlStart(ClientHttpRequest *, RH *, void *); //adding for storeurl
+
#if _USE_INLINE_
#include "Store.h"
#include "client_side_request.cci"
=== modified file 'src/redirect.cc'
--- src/redirect.cc 2012-08-28 19:12:13 +0000
+++ src/redirect.cc 2012-09-09 10:04:04 +0000
@@ -75,6 +75,12 @@
static int n_bypassed = 0;
CBDATA_TYPE(redirectStateData);
+//adding for store_url
+static helper *storeurl_rewriters = NULL;
+static OBJH storeurlStats;
+static int n_storeurl_bypassed = 0;
+//end
+
static void
redirectHandleReply(void *data, char *reply)
{
@@ -96,6 +102,29 @@
redirectStateFree(r);
}
+//adding for storeurl
+static void
+storeurlHandleReply(void *data, char *reply)
+{
+ redirectStateData *r = static_cast<redirectStateData *>(data);
+ char *t;
+ void *cbdata;
+ debugs(61, 5, "storeurlHandleRead: {" << (reply && *reply != '\0' ? reply : "<NULL>") << "}");
+
+ if (reply) {
+ if ((t = strchr(reply, ' ')))
+ *t = '\0';
+
+ if (*reply == '\0')
+ reply = NULL;
+ }
+
+ if (cbdataReferenceValidDone(r->data, &cbdata))
+ r->handler(cbdata, reply);
+
+ redirectStateFree(r);
+}
+//end add for storeurl
static void
redirectStateFree(redirectStateData * r)
@@ -118,6 +147,22 @@
storeAppendPrintf(sentry, "\nNumber of requests bypassed "
"because all redirectors were busy: %d\n", n_bypassed);
}
+//adding for store url
+static void
+storeurlStats(StoreEntry * sentry)
+{
+ if (storeurl_rewriters == NULL) {
+ storeAppendPrintf(sentry, "No Store Url Rewritters defined\n");
+ return;
+ }
+
+ helperStats(sentry, storeurl_rewriters, "Store Url Rewriters Statistics");
+
+ if (Config.onoff.storeurl_bypass)
+ storeAppendPrintf(sentry, "\nNumber of requests bypassed "
+ "because all redirectors were busy: %d\n", n_storeurl_bypassed);
+}
+//end add for storeurl
/**** PUBLIC FUNCTIONS ****/
@@ -237,46 +282,189 @@
redirectRegisterWithCacheManager(void)
{
Mgr::RegisterAction("redirector", "URL Redirector Stats", redirectStats, 0, 1);
+ //adding for store_url
+ Mgr::RegisterAction("storeurl", "Store URL Rewritter Stats", storeurlStats, 0, 1);
+ //end
}
void
redirectInit(void)
{
+ // changed the logic of the shutdown to fit both storeurl and redirector
static int init = 0;
- redirectRegisterWithCacheManager();
+ redirectRegisterWithCacheManager(); //registers both redirector and storeurl
- if (!Config.Program.redirect)
+ if (!Config.Program.redirect && !Config.Program.storeurl)
return;
- if (redirectors == NULL)
- redirectors = new helper("redirector");
-
- redirectors->cmdline = Config.Program.redirect;
-
- redirectors->childs.updateLimits(Config.redirectChildren);
-
- redirectors->ipc_type = IPC_STREAM;
-
- helperOpenServers(redirectors);
-
+ if (Config.Program.redirect){
+
+ if (redirectors == NULL)
+ redirectors = new helper("redirector");
+
+ redirectors->cmdline = Config.Program.redirect;
+
+ redirectors->childs.updateLimits(Config.redirectChildren);
+
+ redirectors->ipc_type = IPC_STREAM;
+
+ helperOpenServers(redirectors);
+ }
+
+ if (Config.Program.storeurl){
+ if (redirectors == NULL)
+ redirectors = new helper("storeurl");
+
+ storeurl_rewriters->cmdline = Config.Program.storeurl;
+
+ storeurl_rewriters->childs.updateLimits(Config.storeurlChildren);
+
+ storeurl_rewriters->ipc_type = IPC_STREAM;
+
+ helperOpenServers(storeurl_rewriters);
+ }
if (!init) {
- init = 1;
- CBDATA_INIT_TYPE(redirectStateData);
- }
+ init = 1;
+ CBDATA_INIT_TYPE(redirectStateData); //what it does exactly? initializing the struct for usage?
+ }
}
void
redirectShutdown(void)
{
- if (!redirectors)
+ // changed the logic of the shutdown to fit both storeurl and redirector
+ if (!redirectors && !storeurl_rewriters)
return;
- helperShutdown(redirectors);
+ if (redirectors)
+ helperShutdown(redirectors);
+
+ if (storeurl_rewriters)
+ helperShutdown(storeurl_rewriters);
if (!shutting_down)
return;
delete redirectors;
redirectors = NULL;
+ //adding for store_url
+ delete storeurl_rewriters;
+ storeurl_rewriters = NULL;
+ //end
+}
+
+
+//adding storeurl methods
+
+void
+storeurlStart(ClientHttpRequest * http, RH * handler, void *data)
+{
+ ConnStateData * conn = http->getConn();
+ redirectStateData *r = NULL;
+ const char *fqdn;
+ char buf[MAX_REDIRECTOR_REQUEST_STRLEN];
+ int sz;
+ http_status status;
+ char claddr[MAX_IPSTRLEN];
+ char myaddr[MAX_IPSTRLEN];
+ assert(http);
+ assert(handler);
+ debugs(61, 5, "storeurlStart: '" << http->uri << "'");
+
+ if (Config.onoff.storeurl_bypass && storeurl_rewriters->stats.queue_size) {
+ /* Skip redirector if there is one request queued */
+ ++n_storeurl_bypassed;
+ handler(data, NULL);
+ return;
+ }
+
+ r = cbdataAlloc(redirectStateData);
+ r->orig_url = xstrdup(http->uri);
+ if (conn != NULL)
+ r->client_addr = conn->log_addr;
+ else
+ r->client_addr.SetNoAddr();
+ r->client_ident = NULL;
+#if USE_AUTH
+ if (http->request->auth_user_request != NULL) {
+ r->client_ident = http->request->auth_user_request->username();
+ debugs(61, 5, HERE << "auth-user=" << (r->client_ident?r->client_ident:"NULL"));
+ }
+#endif
+
+ // HttpRequest initializes with null_string. So we must check both defined() and size()
+ if (!r->client_ident && http->request->extacl_user.defined() && http->request->extacl_user.size()) {
+ r->client_ident = http->request->extacl_user.termedBuf();
+ debugs(61, 5, HERE << "acl-user=" << (r->client_ident?r->client_ident:"NULL"));
+ }
+
+ if (!r->client_ident && conn != NULL && conn->clientConnection != NULL && conn->clientConnection->rfc931[0]) {
+ r->client_ident = conn->clientConnection->rfc931;
+ debugs(61, 5, HERE << "ident-user=" << (r->client_ident?r->client_ident:"NULL"));
+ }
+
+#if USE_SSL
+
+ if (!r->client_ident && conn != NULL && Comm::IsConnOpen(conn->clientConnection)) {
+ r->client_ident = sslGetUserEmail(fd_table[conn->clientConnection->fd].ssl);
+ debugs(61, 5, HERE << "ssl-user=" << (r->client_ident?r->client_ident:"NULL"));
+ }
+#endif
+
+ if (!r->client_ident)
+ r->client_ident = dash_str;
+
+ r->method_s = RequestMethodStr(http->request->method);
+
+ r->handler = handler;
+
+ r->data = cbdataReference(data);
+
+ if ((fqdn = fqdncache_gethostbyaddr(r->client_addr, 0)) == NULL)
+ fqdn = dash_str;
+
+ sz = snprintf(buf, MAX_REDIRECTOR_REQUEST_STRLEN, "%s %s/%s %s %s myip=%s myport=%d\n",
+ r->orig_url,
+ r->client_addr.NtoA(claddr,MAX_IPSTRLEN),
+ fqdn,
+ r->client_ident[0] ? rfc1738_escape(r->client_ident) : dash_str,
+ r->method_s,
+ http->request->my_addr.NtoA(myaddr,MAX_IPSTRLEN),
+ http->request->my_addr.GetPort());
+
+ if ((sz<=0) || (sz>=MAX_REDIRECTOR_REQUEST_STRLEN)) {
+ if (sz<=0) {
+ status = HTTP_INTERNAL_SERVER_ERROR;
+ debugs(61, DBG_CRITICAL, "ERROR: Gateway Failure. Can not build request to be passed to redirector. Request ABORTED.");
+ } else {
+ status = HTTP_REQUEST_URI_TOO_LARGE;
+ debugs(61, DBG_CRITICAL, "ERROR: Gateway Failure. Request passed to storeurl rewritter exceeds MAX_REDIRECTOR_REQUEST_STRLEN (" << MAX_REDIRECTOR_REQUEST_STRLEN << "). Request ABORTED.");
+ }
+
+ clientStreamNode *node = (clientStreamNode *)http->client_stream.tail->prev->data;
+ clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
+ assert (repContext);
+ Ip::Address tmpnoaddr;
+ tmpnoaddr.SetNoAddr();
+ repContext->setReplyToError(ERR_GATEWAY_FAILURE, status,
+ http->request->method, NULL,
+ http->getConn() != NULL && http->getConn()->clientConnection != NULL ?
+ http->getConn()->clientConnection->remote : tmpnoaddr,
+ http->request,
+ NULL,
+#if USE_AUTH
+ http->getConn() != NULL && http->getConn()->auth_user_request != NULL ?
+ http->getConn()->auth_user_request : http->request->auth_user_request);
+#else
+ NULL);
+#endif
+
+ node = (clientStreamNode *)http->client_stream.tail->data;
+ clientStreamRead(node, http, node->readBuffer);
+ return;
+ }
+
+ debugs(61,6, HERE << "sending '" << buf << "' to the helper");
+ helperSubmit(storeurl_rewriters, buf, storeurlHandleReply, r);
}
=== modified file 'src/structs.h'
--- src/structs.h 2012-08-28 13:00:30 +0000
+++ src/structs.h 2012-09-08 14:39:49 +0000
@@ -306,6 +306,9 @@
#endif
wordlist *redirect;
+ //adding for store_url
+ wordlist *storeurl;
+ //end
#if USE_UNLINKD
char *unlinkd;
@@ -323,6 +326,9 @@
#endif
HelperChildConfig redirectChildren;
+ //adding for store_url
+ HelperChildConfig storeurlChildren;
+ //end
time_t authenticateGCInterval;
time_t authenticateTTL;
time_t authenticateIpTTL;
@@ -421,6 +427,9 @@
int nonhierarchical_direct;
int strip_query_terms;
int redirector_bypass;
+ //added for store_url
+ int storeurl_bypass;
+ //end
int ignore_unknown_nameservers;
int client_pconns;
int server_pconns;
@@ -484,6 +493,9 @@
acl_access *brokenPosts;
#endif
acl_access *redirector;
+ //adding for storeurl
+ acl_access *storeurl;
+ //end
acl_access *reply;
acl_address *outgoing_address;
#if USE_HTCP
@@ -983,6 +995,7 @@
1; /* this should be killed, also in httpstateflags */
unsigned int refresh:1;
unsigned int redirected:1;
+ unsigned int storeurl_rewritted:1; //adding for storeurl
unsigned int need_validation:1;
unsigned int fail_on_validation_err:1; ///< whether we should fail if validation fails
unsigned int stale_if_hit:1; ///< reply is stale if it is a hit
