http://git-wip-us.apache.org/repos/asf/trafficserver/blob/e8b899b0/plugins/experimental/ats_pagespeed/ats_pagespeed.cc ---------------------------------------------------------------------- diff --git a/plugins/experimental/ats_pagespeed/ats_pagespeed.cc b/plugins/experimental/ats_pagespeed/ats_pagespeed.cc new file mode 100644 index 0000000..4f6cea5 --- /dev/null +++ b/plugins/experimental/ats_pagespeed/ats_pagespeed.cc @@ -0,0 +1,1093 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// TODO(oschaaf): remove what isn't used +#ifndef __STDC_LIMIT_MACROS +#define __STDC_LIMIT_MACROS +#endif +#include <stdlib.h> +#include <errno.h> +#include <stdio.h> +#include <limits.h> +#include <stdint.h> +#include <sys/inotify.h> +#include <unistd.h> + +#include <ts/ts.h> + +#include <vector> +#include <set> + + +#include "ats_pagespeed.h" + +#include "ats_config.h" +#include "ats_header_utils.h" +#include "ats_rewrite_options.h" +#include "ats_log_message_handler.h" + +#include "base/logging.h" +#include "net/instaweb/http/public/response_headers.h" +#include "net/instaweb/util/public/string_util.h" + +#include "ats_base_fetch.h" +#include "ats_resource_intercept.h" +#include "ats_beacon_intercept.h" +#include "ats_process_context.h" +#include "ats_rewrite_driver_factory.h" +#include "ats_rewrite_options.h" +#include "ats_server_context.h" + +#include "net/instaweb/rewriter/public/rewrite_stats.h" +#include "net/instaweb/system/public/in_place_resource_recorder.h" + +#include "net/instaweb/automatic/public/proxy_fetch.h" +#include "net/instaweb/http/public/content_type.h" +#include "net/instaweb/http/public/request_context.h" +#include "net/instaweb/rewriter/public/experiment_matcher.h" +#include "net/instaweb/rewriter/public/experiment_util.h" +#include "net/instaweb/rewriter/public/process_context.h" +#include "net/instaweb/rewriter/public/resource_fetch.h" +#include "net/instaweb/rewriter/public/rewrite_driver.h" +#include "net/instaweb/rewriter/public/rewrite_query.h" +#include "net/instaweb/rewriter/public/static_asset_manager.h" +#include "net/instaweb/public/global_constants.h" +#include "net/instaweb/public/version.h" +#include "net/instaweb/util/public/google_message_handler.h" +#include "net/instaweb/util/public/google_url.h" +#include "net/instaweb/util/public/gzip_inflater.h" +#include "net/instaweb/util/public/query_params.h" +#include "net/instaweb/util/public/statistics_logger.h" +#include "net/instaweb/util/public/stdio_file_system.h" +#include "net/instaweb/util/public/string.h" +#include "net/instaweb/util/public/string_writer.h" +#include "net/instaweb/util/public/time_util.h" +#include "net/instaweb/util/stack_buffer.h" +#include "net/instaweb/system/public/system_request_context.h" + + +#include <dirent.h> + +using namespace net_instaweb; + +static AtsProcessContext* ats_process_context; +static const char* DEBUG_TAG = "ats_pagespeed_transform"; +static int TXN_INDEX_ARG; +static int TXN_INDEX_OWNED_ARG; +static int TXN_INDEX_OWNED_ARG_SET; +static int TXN_INDEX_OWNED_ARG_UNSET; +TSMutex config_mutex = TSMutexCreate(); +AtsConfig* config = NULL; +TransformCtx* get_transaction_context(TSHttpTxn txnp) { + return (TransformCtx *) TSHttpTxnArgGet(txnp, TXN_INDEX_ARG); +} + +static TransformCtx * +ats_ctx_alloc() +{ + TransformCtx *ctx; + + ctx = (TransformCtx *) TSmalloc(sizeof(TransformCtx)); + ctx->downstream_vio = NULL; + ctx->downstream_buffer = NULL; + ctx->downstream_length = 0; + ctx->state = transform_state_initialized; + + ctx->base_fetch = NULL; + ctx->proxy_fetch = NULL; + + ctx->inflater = NULL; + ctx->url_string = NULL; + ctx->gurl = NULL; + ctx->write_pending = false; + ctx->fetch_done = false; + ctx->resource_request = false; + ctx->beacon_request = false; + ctx->transform_added = false; + ctx->mps_user_agent = false; + ctx->user_agent = NULL; + ctx->server_context = NULL; + ctx->html_rewrite = false; + ctx->request_method = NULL; + ctx->alive = 0xaaaa; + ctx->options = NULL; + ctx->to_host = NULL; + return ctx; +} + +void +ats_ctx_destroy(TransformCtx * ctx) +{ + TSReleaseAssert(ctx); + CHECK(ctx->alive == 0xaaaa) << "Already dead!"; + ctx->alive = 0xbbbb; + + if (ctx->base_fetch != NULL) { + ctx->base_fetch->Release(); + ctx->base_fetch = NULL; + } + + if (ctx->proxy_fetch != NULL) { + ctx->proxy_fetch->Done(false /* failure */); + ctx->proxy_fetch = NULL; + } + + if (ctx->inflater != NULL) { + delete ctx->inflater; + ctx->inflater = NULL; + } + + if (ctx->downstream_buffer) { + TSIOBufferDestroy(ctx->downstream_buffer); + } + + if (ctx->url_string != NULL) { + delete ctx->url_string; + ctx->url_string = NULL; + } + + if (ctx->gurl != NULL) { + delete ctx->gurl; + ctx->gurl = NULL; + } + if (ctx->user_agent != NULL) { + delete ctx->user_agent; + ctx->user_agent = NULL; + } + ctx->request_method = NULL; + if (ctx->options != NULL) { + delete ctx->options; + ctx->options = NULL; + } + if (ctx->to_host != NULL) { + delete ctx->to_host; + ctx->to_host = NULL; + } + TSfree(ctx); +} + +RewriteOptions* ps_determine_request_options( + ServerContext* server_context, + RequestHeaders* request_headers, + ResponseHeaders* response_headers, + GoogleUrl* url) { + // Stripping ModPagespeed query params before the property cache lookup to + // make cache key consistent for both lookup and storing in cache. + // + // Sets option from request headers and url. + RewriteQuery rewrite_query; + if (!server_context->GetQueryOptions(url, request_headers, + response_headers, &rewrite_query)) { + // Failed to parse query params or request headers. Treat this as if there + // were no query params given. + TSError("ps_route rerquest: parsing headers or query params failed."); + return NULL; + } + + // Will be NULL if there aren't any options set with query params or in + // headers. + return rewrite_query.ReleaseOptions(); +} + +bool ps_determine_options(ServerContext* server_context, + // Directory-specific options, usually null. They've already been rebased off + // of the global options as part of the configuration process. + RewriteOptions* directory_options, + RequestHeaders* request_headers, + ResponseHeaders* response_headers, + RewriteOptions** options, + GoogleUrl* url) { + // Global options for this server. Never null. + RewriteOptions* global_options = server_context->global_options(); + + // Request-specific options, nearly always null. If set they need to be + // rebased on the directory options or the global options. + RewriteOptions* request_options = ps_determine_request_options( + server_context, request_headers, response_headers, url); + + // Because the caller takes ownership of any options we return, the only + // situation in which we can avoid allocating a new RewriteOptions is if the + // global options are ok as are. + if (directory_options == NULL && request_options == NULL && + !global_options->running_experiment()) { + return true; + } + + // Start with directory options if we have them, otherwise request options. + if (directory_options != NULL) { + //*options = directory_options->Clone(); + // OS: HACK! TODO! + *options = global_options->Clone(); + (*options)->Merge(*directory_options); + } else { + *options = global_options->Clone(); + } + + // Modify our options in response to request options or experiment settings, + // if we need to. If there are request options then ignore the experiment + // because we don't want experiments to be contaminated with unexpected + // settings. + if (request_options != NULL) { + (*options)->Merge(*request_options); + delete request_options; + } + // TODO(oschaaf): experiments + /*else if ((*options)->running_experiment()) { + bool ok = ps_set_experiment_state_and_cookie( + r, request_headers, *options, url->Host()); + if (!ok) { + delete *options; + *options = NULL; + return false; + } + }*/ + + return true; +} + +void +handle_send_response_headers(TSHttpTxn txnp) { + TransformCtx* ctx = get_transaction_context(txnp); + // TODO(oschaaf): Fix the response headers!! + bool is_owned = TSHttpTxnArgGet(txnp, TXN_INDEX_OWNED_ARG) == &TXN_INDEX_OWNED_ARG_SET; + if (!is_owned) { + return; + } + CHECK(ctx->alive == 0xaaaa) << "Already dead !"; + if (ctx->html_rewrite) { + TSMBuffer bufp = NULL; + TSMLoc hdr_loc = NULL; + if (ctx->base_fetch == NULL) { + // TODO(oschaaf): figure out when this happens. + return; + } + + if (TSHttpTxnClientRespGet(txnp, &bufp, &hdr_loc) == TS_SUCCESS) { + ResponseHeaders* pagespeed_headers = + ctx->base_fetch->response_headers(); + for (int i = 0 ; i < pagespeed_headers->NumAttributes() ; i++) { + const GoogleString& name_gs = pagespeed_headers->Name(i); + const GoogleString& value_gs = pagespeed_headers->Value(i); + + // We should avoid touching these fields, as ATS will drop keepalive when we do. + if ( StringCaseEqual(name_gs, "Connection") || StringCaseEqual(name_gs, "Transfer-Encoding") ) { + continue; + } + + TSMLoc field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, name_gs.data(), name_gs.size()); + if (field_loc != NULL) { + TSMimeHdrFieldValuesClear(bufp, hdr_loc, field_loc); + TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, field_loc, -1, + value_gs.data(), value_gs.size()); + } else if (TSMimeHdrFieldCreate(bufp, hdr_loc, &field_loc) == TS_SUCCESS) { + if (TSMimeHdrFieldNameSet(bufp, hdr_loc, field_loc, name_gs.data(), name_gs.size()) == TS_SUCCESS) { + TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, field_loc, -1, + value_gs.data(), value_gs.size()); + TSMimeHdrFieldAppend(bufp, hdr_loc, field_loc); + } else { + CHECK(false) << "Field name set failure"; + } + TSHandleMLocRelease(bufp, hdr_loc, field_loc); + } else { + CHECK(false) << "Field create failure"; + } + } + + TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc); + } else { + DCHECK(false) << "Could not get response headers?!"; + } + } +} + +static void +copy_response_headers_to_psol(TSMBuffer bufp, TSMLoc hdr_loc, ResponseHeaders* psol_headers) { + int n_mime_headers = TSMimeHdrFieldsCount(bufp, hdr_loc); + TSMLoc field_loc; + const char *name, *value; + int name_len, value_len; + GoogleString header; + for (int i = 0; i < n_mime_headers; ++i) { + field_loc = TSMimeHdrFieldGet(bufp, hdr_loc, i); + if (!field_loc) { + TSDebug(DEBUG_TAG, "[%s] Error while obtaining header field #%d", __FUNCTION__, i); + continue; + } + name = TSMimeHdrFieldNameGet(bufp, hdr_loc, field_loc, &name_len); + StringPiece s_name(name, name_len); + int n_field_values = TSMimeHdrFieldValuesCount(bufp, hdr_loc, field_loc); + for (int j = 0; j < n_field_values; ++j) { + value = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, field_loc, j, &value_len); + if ( NULL == value || !value_len ) { + TSDebug(DEBUG_TAG, "[%s] Error while getting value #%d of header [%.*s]", + __FUNCTION__, j, name_len, name); + } else { + StringPiece s_value(value, value_len); + psol_headers->Add(s_name, s_value); + //TSDebug(DEBUG_TAG, "Add response header [%.*s:%.*s]",name_len, name, value_len, value); + } + } + TSHandleMLocRelease(bufp, hdr_loc, field_loc); + } +} + +void +copy_request_headers_to_psol(TSMBuffer bufp, TSMLoc hdr_loc, RequestHeaders* psol_headers) { + int n_mime_headers = TSMimeHdrFieldsCount(bufp, hdr_loc); + TSMLoc field_loc; + const char *name, *value; + int name_len, value_len; + GoogleString header; + for (int i = 0; i < n_mime_headers; ++i) { + field_loc = TSMimeHdrFieldGet(bufp, hdr_loc, i); + if (!field_loc) { + TSDebug(DEBUG_TAG, "[%s] Error while obtaining header field #%d", __FUNCTION__, i); + continue; + } + name = TSMimeHdrFieldNameGet(bufp, hdr_loc, field_loc, &name_len); + StringPiece s_name(name, name_len); + int n_field_values = TSMimeHdrFieldValuesCount(bufp, hdr_loc, field_loc); + for (int j = 0; j < n_field_values; ++j) { + value = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, field_loc, j, &value_len); + if ( NULL == value || !value_len ) { + TSDebug(DEBUG_TAG, "[%s] Error while getting value #%d of header [%.*s]", + __FUNCTION__, j, name_len, name); + } else { + StringPiece s_value(value, value_len); + psol_headers->Add(s_name, s_value); + //TSDebug(DEBUG_TAG, "Add request header [%.*s:%.*s]",name_len, name, value_len, value); + } + } + TSHandleMLocRelease(bufp, hdr_loc, field_loc); + } +} + +// TODO(oschaaf): this is not sustainable when we get more +// configuration options like this. +bool get_override_expiry(const StringPiece& host) { + TSMutexLock(config_mutex); + AtsHostConfig* hc = config->Find(host.data(), host.size()); + TSMutexUnlock(config_mutex); + return hc->override_expiry(); +} + +AtsRewriteOptions* get_host_options(const StringPiece& host) { + TSMutexLock(config_mutex); + AtsRewriteOptions* r = NULL; + AtsHostConfig* hc = config->Find(host.data(), host.size()); + if (hc->options() != NULL) { + // We return a clone here to avoid having to thing about + // configuration reloads and outstanding options + r = hc->options()->Clone(); + } + TSMutexUnlock(config_mutex); + return r; +} + +std::string get_remapped_host(TSHttpTxn txn) { + TSMBuffer server_req_buf; + TSMLoc server_req_loc; + std::string to_host; + if (TSHttpTxnServerReqGet(txn, &server_req_buf, &server_req_loc) == TS_SUCCESS + || TSHttpTxnCachedReqGet(txn, &server_req_buf, &server_req_loc) == TS_SUCCESS) { + to_host = get_header(server_req_buf, server_req_loc, "Host"); + TSHandleMLocRelease(server_req_buf, TS_NULL_MLOC, server_req_loc); + } else { + fprintf(stderr, "@@@@@@@ FAILED \n"); + } + return to_host; +} + +static void +ats_transform_init(TSCont contp, TransformCtx * ctx) +{ + //prepare the downstream for transforming + TSVConn downstream_conn; + TSMBuffer bufp; + TSMLoc hdr_loc; + TSMBuffer reqp; + TSMLoc req_hdr_loc; + ctx->state = transform_state_output; + + + // TODO: check cleanup flow + if (TSHttpTxnTransformRespGet(ctx->txn, &bufp, &hdr_loc) != TS_SUCCESS) { + TSError("Error TSHttpTxnTransformRespGet"); + return; + } + if (TSHttpTxnClientReqGet(ctx->txn, &reqp, &req_hdr_loc) != TS_SUCCESS) { + TSError("Error TSHttpTxnClientReqGet"); + return; + } + + AtsServerContext* server_context = ats_process_context->server_context(); + if (server_context->IsPagespeedResource(*ctx->gurl)) { + CHECK(false) << "PageSpeed resource should not get here!"; + } + + downstream_conn = TSTransformOutputVConnGet(contp); + ctx->downstream_buffer = TSIOBufferCreate(); + ctx->downstream_vio = TSVConnWrite(downstream_conn, contp, TSIOBufferReaderAlloc(ctx->downstream_buffer), INT64_MAX); + + // TODO(oschaaf): fix host/ip(?) + SystemRequestContext* system_request_context = + new SystemRequestContext(server_context->thread_system()->NewMutex(), + server_context->timer(), + "www.foo.com", + 80, + "127.0.0.1"); + + ctx->base_fetch = new AtsBaseFetch(server_context, RequestContextPtr(system_request_context), + ctx->downstream_vio, ctx->downstream_buffer, false); + + + RewriteOptions* options = NULL; + RequestHeaders* request_headers = new RequestHeaders(); + ctx->base_fetch->SetRequestHeadersTakingOwnership(request_headers); + copy_request_headers_to_psol(reqp, req_hdr_loc, request_headers); + + TSHttpStatus status = TSHttpHdrStatusGet(bufp, hdr_loc); + // TODO(oschaaf): http version + ctx->base_fetch->response_headers()->set_status_code(status); + copy_response_headers_to_psol(bufp, hdr_loc, ctx->base_fetch->response_headers()); + ctx->base_fetch->response_headers()->ComputeCaching(); + const char* host = ctx->gurl->HostAndPort().as_string().c_str(); + //request_headers->Lookup1(HttpAttributes::kHost); + if (host != NULL && strlen(host) > 0) { + ctx->options = get_host_options(host); + } + bool ok = ps_determine_options(server_context, + ctx->options, + request_headers, + ctx->base_fetch->response_headers(), + &options, + ctx->gurl); + + // Take ownership of custom_options. + scoped_ptr<RewriteOptions> custom_options(options); + + if (!ok) { + TSError("Failure while determining request options for psol"); + options = server_context->global_options(); + } else { + // ps_determine_options modified url, removing any ModPagespeedFoo=Bar query + // parameters. Keep url_string in sync with url. + ctx->gurl->Spec().CopyToString(ctx->url_string); + } + + RewriteDriver* driver; + if (custom_options.get() == NULL) { + driver = server_context->NewRewriteDriver(ctx->base_fetch->request_context()); + } else { + driver = server_context->NewCustomRewriteDriver(custom_options.release(), ctx->base_fetch->request_context()); + } + + driver->SetUserAgent(ctx->user_agent->c_str()); + driver->SetRequestHeaders(*request_headers); + + bool page_callback_added = false; + scoped_ptr<ProxyFetchPropertyCallbackCollector> + property_callback( + ProxyFetchFactory::InitiatePropertyCacheLookup( + false /* is resource fetch?*/, + *ctx->gurl, + server_context, + options, + ctx->base_fetch, + false /* requires_blink_cohort (no longer unused) */, + &page_callback_added)); + + ctx->proxy_fetch = + ats_process_context->proxy_fetch_factory()->CreateNewProxyFetch( + *(ctx->url_string), ctx->base_fetch, driver, + property_callback.release(), + NULL /* original_content_fetch */); + + TSHandleMLocRelease(reqp, TS_NULL_MLOC, req_hdr_loc); + TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc); +} + +static void +ats_transform_one(TransformCtx * ctx, TSIOBufferReader upstream_reader, int amount) +{ + TSIOBufferBlock downstream_blkp; + const char *upstream_buffer; + int64_t upstream_length; + + while (amount > 0) { + downstream_blkp = TSIOBufferReaderStart(upstream_reader); + if (!downstream_blkp) { + TSError("couldn't get from IOBufferBlock"); + return; + } + + upstream_buffer = TSIOBufferBlockReadStart(downstream_blkp, upstream_reader, &upstream_length); + if (!upstream_buffer) { + TSError("couldn't get from TSIOBufferBlockReadStart"); + return; + } + + if (upstream_length > amount) { + upstream_length = amount; + } + + TSDebug("ats-speed", "transform!"); + // TODO(oschaaf): use at least the message handler from the server conrtext here? + if (ctx->inflater == NULL) { + ctx->proxy_fetch->Write(StringPiece((char*)upstream_buffer, upstream_length), ats_process_context->message_handler()); + } else { + char buf[net_instaweb::kStackBufferSize]; + + ctx->inflater->SetInput((char*)upstream_buffer, upstream_length); + + while (ctx->inflater->HasUnconsumedInput()) { + int num_inflated_bytes = ctx->inflater->InflateBytes( + buf, net_instaweb::kStackBufferSize); + if (num_inflated_bytes < 0) { + TSError("Corrupted inflation"); + } else if (num_inflated_bytes > 0) { + ctx->proxy_fetch->Write(StringPiece(buf, num_inflated_bytes), + ats_process_context->message_handler()); + } + } + } + //ctx->proxy_fetch->Flush(NULL); + TSIOBufferReaderConsume(upstream_reader, upstream_length); + amount -= upstream_length; + } + // TODO(oschaaf): get the output from the base fetch, and send it downstream. + // This would require proper locking around the base fetch buffer + // We could also have a look at directly writing to the traffic server buffers +} + + +static void +ats_transform_finish(TransformCtx * ctx) +{ + if (ctx->state == transform_state_output) { + ctx->state = transform_state_finished; + ctx->proxy_fetch->Done(true); + ctx->proxy_fetch = NULL; + } +} + +static void +ats_transform_do(TSCont contp) +{ + TSVIO upstream_vio; + TransformCtx *ctx; + int64_t upstream_todo; + int64_t upstream_avail; + int64_t downstream_bytes_written; + + ctx = (TransformCtx*)TSContDataGet(contp); + + if (ctx->state == transform_state_initialized) { + ats_transform_init(contp, ctx); + } + + upstream_vio = TSVConnWriteVIOGet(contp); + downstream_bytes_written = ctx->downstream_length; + + if (!TSVIOBufferGet(upstream_vio)) { + ats_transform_finish(ctx); + return; + } + + upstream_todo = TSVIONTodoGet(upstream_vio); + + if (upstream_todo > 0) { + upstream_avail = TSIOBufferReaderAvail(TSVIOReaderGet(upstream_vio)); + + if (upstream_todo > upstream_avail) { + upstream_todo = upstream_avail; + } + + if (upstream_todo > 0) { + ats_transform_one(ctx, TSVIOReaderGet(upstream_vio), upstream_todo); + TSVIONDoneSet(upstream_vio, TSVIONDoneGet(upstream_vio) + upstream_todo); + } + } + + if (TSVIONTodoGet(upstream_vio) > 0) { + if (upstream_todo > 0) { + if (ctx->downstream_length > downstream_bytes_written) { + TSVIOReenable(ctx->downstream_vio); + } + TSContCall(TSVIOContGet(upstream_vio), TS_EVENT_VCONN_WRITE_READY, upstream_vio); + } + } else { + ats_transform_finish(ctx); + TSContCall(TSVIOContGet(upstream_vio), TS_EVENT_VCONN_WRITE_COMPLETE, upstream_vio); + } +} + + +static int +ats_pagespeed_transform(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */) +{ + if (TSVConnClosedGet(contp)) { + //ats_ctx_destroy((TransformCtx*)TSContDataGet(contp)); + TSContDestroy(contp); + return 0; + } else { + switch (event) { + case TS_EVENT_ERROR:{ + fprintf(stderr, "ats speed transform event: [%d] TS EVENT ERROR?!\n", event); + TSVIO upstream_vio = TSVConnWriteVIOGet(contp); + TSContCall(TSVIOContGet(upstream_vio), TS_EVENT_ERROR, upstream_vio); + } + break; + case TS_EVENT_VCONN_WRITE_COMPLETE: + TSVConnShutdown(TSTransformOutputVConnGet(contp), 0, 1); + break; + case TS_EVENT_VCONN_WRITE_READY: + ats_transform_do(contp); + break; + case TS_EVENT_IMMEDIATE: + ats_transform_do(contp); + break; + default: + DCHECK(false) << "unknown event: " << event; + ats_transform_do(contp); + break; + } + } + + return 0; +} + +static void +ats_pagespeed_transform_add(TSHttpTxn txnp) +{ + TransformCtx* ctx = get_transaction_context(txnp); + CHECK(ctx); + if (ctx->transform_added) { // Happens with a stale cache hit + return; + } else { + ctx->transform_added = true; + } + + TSHttpTxnUntransformedRespCache(txnp, 1); + TSHttpTxnTransformedRespCache(txnp, 0); + + TSVConn connp; + + connp = TSTransformCreate(ats_pagespeed_transform, txnp); + TSContDataSet(connp, ctx); + TSHttpTxnHookAdd(txnp, TS_HTTP_RESPONSE_TRANSFORM_HOOK, connp); +} + +// Returns true if a server intercept was set up +// Which means we should not attempt any further transformation +void +handle_read_request_header(TSHttpTxn txnp) { + TSMBuffer reqp = NULL; + TSMLoc hdr_loc = NULL; + char *url = NULL; + int url_length = -1; + + TransformCtx* ctx = ats_ctx_alloc(); + ctx->txn = txnp; + TSHttpTxnArgSet(txnp, TXN_INDEX_ARG, (void*) ctx); + TSHttpTxnArgSet(txnp, TXN_INDEX_OWNED_ARG, &TXN_INDEX_OWNED_ARG_SET); + + if (TSHttpTxnClientReqGet(txnp, &reqp, &hdr_loc) == TS_SUCCESS) { + url = TSHttpTxnEffectiveUrlStringGet(txnp, &url_length); + if (!url || url_length <= 0) { + DCHECK(false) << "Could not get url!"; + } else { + std::string s_url = std::string(url,url_length); + GoogleUrl gurl(s_url); + + ctx->url_string = new GoogleString(url, url_length); + ctx->gurl = new GoogleUrl(*(ctx->url_string)); + if (!ctx->gurl->IsWebValid()) { + TSDebug("ats-speed", "URL != WebValid(): %s", ctx->url_string->c_str()); + } else { + const char * method; + int method_len; + method = TSHttpHdrMethodGet(reqp, hdr_loc, &method_len); + bool head_or_get = method == TS_HTTP_METHOD_GET || method == TS_HTTP_METHOD_HEAD; + ctx->request_method = method; + GoogleString user_agent = get_header(reqp, hdr_loc, "User-Agent"); + ctx->user_agent = new GoogleString(user_agent); + ctx->server_context = ats_process_context->server_context(); + if (user_agent.find(kModPagespeedSubrequestUserAgent) != user_agent.npos) { + ctx->mps_user_agent = true; + } + if (ats_process_context->server_context()->IsPagespeedResource(gurl)) { + if (head_or_get && !ctx->mps_user_agent) { + ctx->resource_request = true; + TSHttpTxnArgSet(txnp, TXN_INDEX_OWNED_ARG, &TXN_INDEX_OWNED_ARG_UNSET); + } + } else if (ctx->gurl->PathSansQuery() == "/pagespeed_message" + || ctx->gurl->PathSansQuery() == "/pagespeed_statistics" + || ctx->gurl->PathSansQuery() == "/pagespeed_global_statistics" + || ctx->gurl->PathSansQuery() == "/pagespeed_console" + || ctx->gurl->PathSansLeaf() == "/ats_pagespeed_static/" + || ctx->gurl->PathSansQuery() == "/robots.txt" + ) { + ctx->resource_request = true; + TSHttpTxnArgSet(txnp, TXN_INDEX_OWNED_ARG, &TXN_INDEX_OWNED_ARG_UNSET); + } + else if (StringCaseEqual(gurl.PathSansQuery() ,"/ats_pagespeed_beacon")) { + ctx->beacon_request = true; + TSHttpTxnArgSet(txnp, TXN_INDEX_OWNED_ARG, &TXN_INDEX_OWNED_ARG_UNSET); + hook_beacon_intercept(txnp); + } + } + TSfree((void*)url); + } // gurl->IsWebValid() == true + TSHandleMLocRelease(reqp, TS_NULL_MLOC, hdr_loc); + } else { + DCHECK(false) << "Could not get client request header\n"; + } + + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); +} + +bool +cache_hit(TSHttpTxn txnp) { + int obj_status; + if (TSHttpTxnCacheLookupStatusGet(txnp, &obj_status) == TS_ERROR) { + // TODO(oschaaf): log warning + return false; + } + return obj_status == TS_CACHE_LOOKUP_HIT_FRESH; +} + +static int +transform_plugin(TSCont contp, TSEvent event, void *edata) +{ + TSHttpTxn txn = (TSHttpTxn) edata; + + CHECK(event == TS_EVENT_HTTP_READ_RESPONSE_HDR || event == TS_EVENT_HTTP_READ_CACHE_HDR + || event == TS_EVENT_HTTP_SEND_REQUEST_HDR || event == TS_EVENT_HTTP_READ_REQUEST_HDR + || event == TS_EVENT_HTTP_TXN_CLOSE || event == TS_EVENT_HTTP_SEND_RESPONSE_HDR) + << "Invalid transform event"; + + if (event != TS_EVENT_HTTP_READ_REQUEST_HDR) { + // Bail if an intercept is running + bool is_owned = TSHttpTxnArgGet(txn, TXN_INDEX_OWNED_ARG) == &TXN_INDEX_OWNED_ARG_SET; + if (!is_owned) { + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + return 0; + } + } + + if (event == TS_EVENT_HTTP_SEND_RESPONSE_HDR) { + handle_send_response_headers(txn); + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + return 0; + } if (event == TS_EVENT_HTTP_TXN_CLOSE) { + TransformCtx* ctx = get_transaction_context(txn); + //if (ctx != NULL && !ctx->resource_request && !ctx->beacon_request && !ctx->html_rewrite) { + // For intercepted requests like beacons and resource requests, we don't own the + // ctx here - the interceptor does. + + if (ctx != NULL) { + bool is_owned = TSHttpTxnArgGet(txn, TXN_INDEX_OWNED_ARG) == &TXN_INDEX_OWNED_ARG_SET; + if (is_owned) { + ats_ctx_destroy(ctx); + } + } + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + return 0; + } if (event == TS_EVENT_HTTP_READ_REQUEST_HDR) { + handle_read_request_header(txn); + return 0; + } else if (event == TS_EVENT_HTTP_SEND_REQUEST_HDR) { + TSMBuffer request_header_buf = NULL; + TSMLoc request_header_loc = NULL; + + if (TSHttpTxnServerReqGet(txn, &request_header_buf, &request_header_loc) == TS_SUCCESS) { + hide_accept_encoding(request_header_buf, request_header_loc, "@xxAccept-Encoding"); + // Turn off pagespeed optimization at the origin + set_header(request_header_buf, request_header_loc, "PageSpeed", "off"); + TSHandleMLocRelease(request_header_buf, TS_NULL_MLOC, request_header_loc); + } else { + CHECK(false) << "Could not find server request header"; + } + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + return 0; + } else if (event == TS_EVENT_HTTP_READ_RESPONSE_HDR) { + TSMBuffer request_header_buf = NULL; + TSMLoc request_header_loc = NULL; + + if (TSHttpTxnServerReqGet(txn, &request_header_buf, &request_header_loc) == TS_SUCCESS) { + restore_accept_encoding(request_header_buf, request_header_loc, "@xxAccept-Encoding"); + TSHandleMLocRelease(request_header_buf, TS_NULL_MLOC, request_header_loc); + } else { + CHECK(false) << "Could not find server request header"; + } + } + + CHECK(event == TS_EVENT_HTTP_READ_RESPONSE_HDR || event == TS_EVENT_HTTP_READ_CACHE_HDR); + + TransformCtx* ctx = get_transaction_context(txn); + if (ctx == NULL) { + // TODO(oschaaf): document how and when this happens. + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + return 0; + } + std::string* to_host = new std::string(); + to_host->append(get_remapped_host(ctx->txn)); + ctx->to_host = to_host; + TSMBuffer response_header_buf = NULL; + TSMLoc response_header_loc = NULL; + + // TODO(oschaaf): from configuration! + bool override_expiry = false; + + const char* host = ctx->gurl->HostAndPort().as_string().c_str(); + //request_headers->Lookup1(HttpAttributes::kHost); + if (host != NULL && strlen(host) > 0) { + override_expiry = get_override_expiry(host); + } + + + if (ctx->mps_user_agent && override_expiry) { + if (TSHttpTxnServerRespGet(txn, &response_header_buf, &response_header_loc) == TS_SUCCESS) { + // TODO => set cacheable. + unset_header(response_header_buf, response_header_loc, "Cache-Control"); + unset_header(response_header_buf, response_header_loc, "Expires"); + unset_header(response_header_buf, response_header_loc, "Age"); + set_header(response_header_buf, response_header_loc, "Cache-Control", "public, max-age=3600"); + TSHandleMLocRelease(response_header_buf, TS_NULL_MLOC, response_header_loc); + } + } + bool ok = ctx->gurl->IsWebValid() && + !(ctx->resource_request || ctx->beacon_request || ctx->mps_user_agent); + if (!ok) { + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + return 0; + } + + bool have_response_header = false; + + if (TSHttpTxnServerRespGet(txn, &response_header_buf, &response_header_loc) == TS_SUCCESS) { + have_response_header = true; + if (override_expiry) { + unset_header(response_header_buf, response_header_loc, "Cache-Control"); + unset_header(response_header_buf, response_header_loc, "Expires"); + unset_header(response_header_buf, response_header_loc, "Age"); + set_header(response_header_buf, response_header_loc, "Cache-Control", "public, max-age=3600"); + } + } + else if (TSHttpTxnCachedRespGet(txn, &response_header_buf, &response_header_loc) == TS_SUCCESS) { + have_response_header = true; + } + if (!have_response_header) { + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + return 0; + } + + if (ok) { + if (ctx->request_method != TS_HTTP_METHOD_GET && ctx->request_method != TS_HTTP_METHOD_HEAD + && ctx->request_method != TS_HTTP_METHOD_POST) { + ok = false; + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + return 0; + } + } + + TSHttpStatus status = TSHttpHdrStatusGet(response_header_buf, response_header_loc); + if (ok) { + if (!(status == TS_HTTP_STATUS_OK || status == TS_HTTP_STATUS_NOT_FOUND)) { + ok = false; + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + return 0; + } + } + if (ok) { + StringPiece s_content_type = get_header(response_header_buf, response_header_loc, "Content-Type"); + const net_instaweb::ContentType* content_type = + net_instaweb::MimeTypeToContentType(s_content_type); + + if ((content_type == NULL || !content_type->IsHtmlLike())) { + ok = false; + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + return 0; + } + } + + if (ok) { + StringPiece content_encoding = get_header(response_header_buf, response_header_loc, "Content-Encoding"); + net_instaweb::GzipInflater::InflateType inflate_type; + bool is_encoded = false; + + if (StringCaseEqual(content_encoding, "deflate")) { + is_encoded = true; + inflate_type = GzipInflater::kDeflate; + } else if (StringCaseEqual(content_encoding, "gzip")) { + is_encoded = true; + inflate_type = GzipInflater::kGzip; + } + + if (is_encoded) { + ctx->inflater = new GzipInflater(inflate_type); + ctx->inflater->Init(); + } + TSDebug(DEBUG_TAG, "Will optimize [%s]", ctx->url_string->c_str()); + ctx->html_rewrite = true; + set_header(response_header_buf,response_header_loc,"@gzip_nocache","0"); + ats_pagespeed_transform_add(txn); + } + + TSHandleMLocRelease(response_header_buf, TS_NULL_MLOC, response_header_loc); + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + + return 0; +} + +bool RegisterPlugin() { + TSPluginRegistrationInfo info; + + info.plugin_name = (char *)"ats_pagespeed"; + info.vendor_name = (char *)"Apache Software Foundation"; + info.support_email = (char *)"[email protected]"; + + if (TSPluginRegister(TS_SDK_VERSION_3_0, &info) != TS_SUCCESS) { + TSError("Failed to register ATSSpeed"); + return false; + } + + return true; +} + +void cleanup_process() { + delete ats_process_context; + AtsRewriteDriverFactory::Terminate(); + AtsRewriteOptions::Terminate(); +} + +static void +process_configuration() +{ + AtsConfig* new_config = new AtsConfig((AtsThreadSystem*)ats_process_context->server_context()->thread_system()); + DIR *dir; + struct dirent *ent; + + if ((dir = opendir ("/usr/local/etc/trafficserver/psol/")) != NULL) { + while ((ent = readdir (dir)) != NULL) { + size_t len = strlen(ent->d_name); + if (len <= 0) continue; + if (ent->d_name[0] == '.') continue; + if (ent->d_name[len-1] == '~') continue; + if (ent->d_name[0] == '#') continue; + GoogleString s("/usr/local/etc/trafficserver/psol/"); + s.append(ent->d_name); + fprintf (stderr, "parse [%s]\n", s.c_str()); + if (!new_config->Parse(s.c_str())) { + TSError("Error parsing %s", s.c_str()); + } + } + closedir (dir); + } + + AtsConfig* old_config; + TSMutexLock(config_mutex); + fprintf(stderr, "Update configuration\n"); + old_config = config; + config = new_config; + TSMutexUnlock(config_mutex); + if (old_config != NULL) { + delete old_config; + } +} + +static void * +config_notification_callback(void *data) +{ + int BUF_MAX = 1024 * (sizeof(struct inotify_event) + 16); + char buf[BUF_MAX]; + int fd,wd; + + fd = inotify_init(); + + if (fd < 0) { + perror( "inotify_init" ); + CHECK(false) << "Failed to initialize inotify"; + } + + wd = inotify_add_watch(fd, "/usr/local/etc/trafficserver/psol/", IN_MODIFY | IN_CREATE | IN_DELETE); + + while (1) { + int len = read(fd, buf, BUF_MAX); + int i = 0; + bool do_update = false; + while ( i < len ) { + struct inotify_event *event = ( struct inotify_event * ) &buf[ i ]; + if ( event->len ) { + if (!(event->mask & IN_ISDIR)) { + const char* name = event->name; + size_t name_len = strlen(event->name); + if (name_len > 0 && name[0] != '.' && name[0] != '#' && name[name_len-1] != '~' ) { + do_update = true; + } + } + } + i += ( sizeof (struct inotify_event) ) + event->len; + } + if (do_update) { + process_configuration(); + } + } + + inotify_rm_watch( fd, wd ); + close( fd ); + + return NULL; +} + + +void TSPluginInit(int argc, const char *argv[]) { + if (RegisterPlugin() == true) { + if (TSHttpArgIndexReserve("ats_pagespeed", "Stores the transaction context", &TXN_INDEX_ARG) != TS_SUCCESS) { + CHECK(false) << "failed to reserve an argument index"; + } + if (TSHttpArgIndexReserve("ats_pagespeed", "Stores the transaction context", &TXN_INDEX_OWNED_ARG) != TS_SUCCESS) { + CHECK(false) << "failed to reserve an argument index"; + } + + AtsRewriteOptions::Initialize(); + AtsRewriteDriverFactory::Initialize(); + net_instaweb::log_message_handler::Install(); + atexit(cleanup_process); + ats_process_context = new AtsProcessContext(); + process_configuration(); + TSCont transform_contp = TSContCreate(transform_plugin, NULL); + TSHttpHookAdd(TS_HTTP_READ_RESPONSE_HDR_HOOK, transform_contp); + TSHttpHookAdd(TS_HTTP_READ_CACHE_HDR_HOOK, transform_contp); + TSHttpHookAdd(TS_HTTP_SEND_REQUEST_HDR_HOOK, transform_contp); + TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, transform_contp); + TSHttpHookAdd(TS_HTTP_TXN_CLOSE_HOOK, transform_contp); + TSHttpHookAdd(TS_HTTP_SEND_RESPONSE_HDR_HOOK, transform_contp); + + setup_resource_intercept(); + CHECK(TSThreadCreate(config_notification_callback, NULL)) << ""; + ats_process_context->message_handler()->Message( + kInfo, "TSPluginInit OK"); + } +}
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/e8b899b0/plugins/experimental/ats_pagespeed/ats_pagespeed.h ---------------------------------------------------------------------- diff --git a/plugins/experimental/ats_pagespeed/ats_pagespeed.h b/plugins/experimental/ats_pagespeed/ats_pagespeed.h new file mode 100644 index 0000000..ccf897a --- /dev/null +++ b/plugins/experimental/ats_pagespeed/ats_pagespeed.h @@ -0,0 +1,102 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef ATS_PAGESPEED_H_ +#define ATS_PAGESPEED_H_ + +#include <string> + +#include <ts/ts.h> + +#include "net/instaweb/util/public/google_url.h" +#include "net/instaweb/util/public/string.h" +#include "net/instaweb/util/public/string_util.h" + +namespace net_instaweb { + +class AtsBaseFetch; +class AtsRewriteOptions; +class AtsServerContext; +class GzipInflater; +class ProxyFetch; +class RewriteOptions; +class RequestHeaders; +class ResponseHeaders; +class ServerContext; + +} // namespace net_instaweb + +enum transform_state { + transform_state_initialized, + transform_state_output, + transform_state_finished +}; + +typedef struct +{ + TSHttpTxn txn; + TSVIO downstream_vio; + TSIOBuffer downstream_buffer; + int64_t downstream_length; + enum transform_state state; + + net_instaweb::AtsBaseFetch* base_fetch; + net_instaweb::ProxyFetch* proxy_fetch; + net_instaweb::GzipInflater* inflater; + + bool write_pending; + bool fetch_done; + GoogleString* url_string; + bool beacon_request; + bool resource_request; + bool mps_user_agent; + bool transform_added; + net_instaweb::GoogleUrl* gurl; + net_instaweb::AtsServerContext* server_context; + GoogleString* user_agent; + bool html_rewrite; + const char* request_method; + int alive; + net_instaweb::AtsRewriteOptions* options; + // TODO: Use GoogleString* + std::string* to_host; +} TransformCtx; + +TransformCtx* get_transaction_context(TSHttpTxn txnp); +void ats_ctx_destroy(TransformCtx * ctx); +bool cache_hit(TSHttpTxn txnp); + +bool ps_determine_options(net_instaweb::ServerContext* server_context, + // Directory-specific options, usually null. They've already been rebased off + // of the global options as part of the configuration process. + net_instaweb::RewriteOptions* directory_options, + net_instaweb::RequestHeaders* request_headers, + net_instaweb::ResponseHeaders* response_headers, + net_instaweb::RewriteOptions** options, + net_instaweb::GoogleUrl* url); + +void copy_request_headers_to_psol(TSMBuffer bufp, TSMLoc hdr_loc, net_instaweb::RequestHeaders* psol_headers); +// You will own options returned by this: +net_instaweb::AtsRewriteOptions* get_host_options(const StringPiece& host); + +#endif /* ATS_PAGESPEED_H_ */ http://git-wip-us.apache.org/repos/asf/trafficserver/blob/e8b899b0/plugins/experimental/ats_pagespeed/ats_process_context.cc ---------------------------------------------------------------------- diff --git a/plugins/experimental/ats_pagespeed/ats_process_context.cc b/plugins/experimental/ats_pagespeed/ats_process_context.cc new file mode 100644 index 0000000..f3ca481 --- /dev/null +++ b/plugins/experimental/ats_pagespeed/ats_process_context.cc @@ -0,0 +1,86 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "ats_process_context.h" + +#include <vector> + +#include "ats_rewrite_driver_factory.h" +#include "ats_server_context.h" +#include "ats_message_handler.h" +#include "ats_thread_system.h" + +#include "net/instaweb/automatic/public/proxy_fetch.h" +#include "net/instaweb/util/public/pthread_shared_mem.h" + +namespace net_instaweb { + + AtsProcessContext::AtsProcessContext() : ProcessContext() { + AtsThreadSystem* ts = new AtsThreadSystem(); + message_handler_.reset(new AtsMessageHandler(ts->NewMutex())); + driver_factory_.reset( + new AtsRewriteDriverFactory( + *this, ts, ""/*hostname, not used*/, -1/*port, not used*/)); + server_context_ = driver_factory()->MakeAtsServerContext(); + + AtsRewriteOptions* root_options_ = (AtsRewriteOptions*)driver_factory_->default_options(); + AtsRewriteOptions* server_options = root_options_->Clone(); + AtsRewriteOptions* options = new AtsRewriteOptions(driver_factory_->thread_system()); + server_options->Merge(*options); + delete options; + + server_context_->global_options()->Merge(*server_options); + delete server_options; + + message_handler_->Message(kInfo,"global default options:\r\n[%s]",driver_factory_->default_options()->OptionsToString().c_str()); + message_handler_->Message(kInfo,"server ctx default options:\r\n[%s]",server_context_->global_options()->OptionsToString().c_str()); + std::vector<SystemServerContext*> server_contexts; + server_contexts.push_back(server_context_); + + //Statistics* statistics = + // driver_factory_->MakeGlobalSharedMemStatistics(*(SystemRewriteOptions*)server_context_->global_options()); + GoogleString error_message; + int error_index = -1; + Statistics* global_statistics = NULL; + driver_factory_.get()->PostConfig( + server_contexts, &error_message, &error_index, &global_statistics); + if (error_index != -1) { + server_contexts[error_index]->message_handler()->Message( + kError, "ngx_pagespeed is enabled. %s", error_message.c_str()); + //return NGX_ERROR; + CHECK(false); + } + + AtsRewriteDriverFactory::InitStats(global_statistics); + + driver_factory()->RootInit(); + driver_factory()->ChildInit(); + + proxy_fetch_factory_.reset(new ProxyFetchFactory(server_context_)); + message_handler_->Message(kInfo, "Process context constructed"); +} + +AtsProcessContext::~AtsProcessContext() { +} + +} // namespace net_instaweb http://git-wip-us.apache.org/repos/asf/trafficserver/blob/e8b899b0/plugins/experimental/ats_pagespeed/ats_process_context.h ---------------------------------------------------------------------- diff --git a/plugins/experimental/ats_pagespeed/ats_process_context.h b/plugins/experimental/ats_pagespeed/ats_process_context.h new file mode 100644 index 0000000..aa344b2 --- /dev/null +++ b/plugins/experimental/ats_pagespeed/ats_process_context.h @@ -0,0 +1,58 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef ATS_PROCESS_CONTEXT_H_ +#define ATS_PROCESS_CONTEXT_H_ + +#include "net/instaweb/util/public/google_message_handler.h" +#include "net/instaweb/util/public/message_handler.h" +#include "net/instaweb/util/public/scoped_ptr.h" +#include "net/instaweb/rewriter/public/process_context.h" + +namespace net_instaweb { + +class AtsRewriteDriverFactory; +class ProxyFetchFactory; +class AtsServerContext; + +class AtsProcessContext : ProcessContext { + public: + explicit AtsProcessContext(); + virtual ~AtsProcessContext(); + + // TODO(oschaaf): const correctness + MessageHandler* message_handler() { return message_handler_.get(); } + AtsRewriteDriverFactory* driver_factory() { return driver_factory_.get(); } + ProxyFetchFactory* proxy_fetch_factory() { return proxy_fetch_factory_.get(); } + AtsServerContext* server_context() { return server_context_; } + private: + scoped_ptr<MessageHandler> message_handler_; + scoped_ptr<AtsRewriteDriverFactory> driver_factory_; + scoped_ptr<ProxyFetchFactory> proxy_fetch_factory_; + AtsServerContext* server_context_; +}; + + +} // namespace net_instaweb + +#endif // ATS_PROCESS_CONTEXT_H_ http://git-wip-us.apache.org/repos/asf/trafficserver/blob/e8b899b0/plugins/experimental/ats_pagespeed/ats_resource_intercept.cc ---------------------------------------------------------------------- diff --git a/plugins/experimental/ats_pagespeed/ats_resource_intercept.cc b/plugins/experimental/ats_pagespeed/ats_resource_intercept.cc new file mode 100644 index 0000000..42ece8f --- /dev/null +++ b/plugins/experimental/ats_pagespeed/ats_resource_intercept.cc @@ -0,0 +1,363 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include <ts/ts.h> + +#include <stdio.h> + +#include "ats_resource_intercept.h" + + +#include "ats_base_fetch.h" +#include "ats_rewrite_driver_factory.h" +#include "ats_rewrite_options.h" +#include "ats_server_context.h" +#include "ats_pagespeed.h" + +#include "net/instaweb/http/public/request_context.h" +#include "net/instaweb/rewriter/public/resource_fetch.h" +#include "net/instaweb/rewriter/public/static_asset_manager.h" +#include "net/instaweb/system/public/system_request_context.h" + +#include "net/instaweb/util/public/string_writer.h" + + +using namespace net_instaweb; + +struct InterceptCtx +{ + TSVConn vconn; + TSIOBuffer req_buffer; + TSIOBufferReader req_reader; + TSIOBuffer resp_buffer; + TSIOBufferReader resp_reader; + GoogleString* response; + TransformCtx* request_ctx; + RequestHeaders* request_headers; + + InterceptCtx() + : vconn(NULL) + , req_buffer(NULL) + , req_reader(NULL) + , resp_buffer(NULL) + , resp_reader(NULL) + , response( new GoogleString() ) + , request_ctx(NULL) + , request_headers(NULL) + { + }; +}; + +static void +shutdown (TSCont cont, InterceptCtx * intercept_ctx) { + if (intercept_ctx->req_reader != NULL) { + TSIOBufferReaderFree(intercept_ctx->req_reader); + intercept_ctx->req_reader = NULL; + } + if (intercept_ctx->req_buffer != NULL) { + TSIOBufferDestroy(intercept_ctx->req_buffer); + intercept_ctx->req_buffer = NULL; + } + if (intercept_ctx->resp_reader != NULL) { + TSIOBufferReaderFree(intercept_ctx->resp_reader); + intercept_ctx->resp_reader = NULL; + } + if (intercept_ctx->resp_buffer != NULL) { + TSIOBufferDestroy(intercept_ctx->resp_buffer); + intercept_ctx->resp_buffer = NULL; + } + if (intercept_ctx->vconn != NULL) { + TSVConnShutdown(intercept_ctx->vconn, 0, 1); + TSVConnClose(intercept_ctx->vconn); + intercept_ctx->vconn = NULL; + } + if (intercept_ctx->response != NULL) { + delete intercept_ctx->response; + intercept_ctx->response = NULL; + } + // TODO(oschaaf): think the ordering of this one through. + if (intercept_ctx->request_ctx) { + ats_ctx_destroy(intercept_ctx->request_ctx); + intercept_ctx->request_ctx = NULL; + } + if (intercept_ctx->request_headers != NULL) { + delete intercept_ctx->request_headers; + intercept_ctx->request_headers = NULL; + } + delete intercept_ctx; + TSContDestroy(cont); +} + +static int +resource_intercept(TSCont cont, TSEvent event, void *edata) +{ + InterceptCtx *intercept_ctx = static_cast<InterceptCtx *>(TSContDataGet(cont)); + bool shutDown = false; + + // TODO(oschaaf): have a look at https://github.com/apache/trafficserver/blob/master/plugins/experimental/esi/serverIntercept.c + // and see if we have any edge cases we should fix. + switch (event) { + case TS_EVENT_NET_ACCEPT: { + intercept_ctx->vconn = static_cast<TSVConn>(edata); + intercept_ctx->req_buffer = TSIOBufferCreate(); + intercept_ctx->req_reader = TSIOBufferReaderAlloc(intercept_ctx->req_buffer); + intercept_ctx->resp_buffer = TSIOBufferCreate(); + intercept_ctx->resp_reader = TSIOBufferReaderAlloc(intercept_ctx->resp_buffer); + TSVConnRead(intercept_ctx->vconn, cont, intercept_ctx->req_buffer, 0x7fffffff); + } break; + case TS_EVENT_VCONN_READ_READY: { + CHECK(intercept_ctx->request_ctx->base_fetch == NULL) << "Base fetch must not be set!"; + CHECK(intercept_ctx->request_ctx->url_string != NULL) << "Url must be set!"; + + TSVConnShutdown(intercept_ctx->vconn, 1, 0); + + // response will already have a size for internal pages at this point. + // resources, however, will have to be fetched. + // TODO(oschaaf): this is extremely ugly. + if (intercept_ctx->response->size() == 0) { + // TODO(oschaaf): unused - must we close / clean this up? + TSVIO downstream_vio = TSVConnWrite( + intercept_ctx->vconn, cont, intercept_ctx->resp_reader, 0x7fffffff); + + AtsServerContext* server_context = intercept_ctx->request_ctx->server_context; + + // TODO:(oschaaf) host/port + SystemRequestContext* system_request_context = + new SystemRequestContext(server_context->thread_system()->NewMutex(), + server_context->timer(), + "www.foo.com",// TODO(oschaaf): compute these + 80, + "127.0.0.1"); + + intercept_ctx->request_ctx->base_fetch = new AtsBaseFetch( + server_context, RequestContextPtr(system_request_context), + downstream_vio, intercept_ctx->resp_buffer, true); + intercept_ctx->request_ctx->base_fetch->set_request_headers( + intercept_ctx->request_headers); + + RewriteOptions* options = NULL; + + //const char* host = intercept_ctx->request_headers->Lookup1(HttpAttributes::kHost); + const char* host = intercept_ctx->request_ctx->gurl->HostAndPort().as_string().c_str(); + if (host != NULL && strlen(host) > 0) { + intercept_ctx->request_ctx->options = get_host_options(host); + } + + // TODO(oschaaf): directory options should be coming from configuration! + bool ok = ps_determine_options(server_context, + intercept_ctx->request_ctx->options, + intercept_ctx->request_ctx->base_fetch->request_headers(), + intercept_ctx->request_ctx->base_fetch->response_headers(), + &options, + intercept_ctx->request_ctx->gurl); + + // Take ownership of custom_options. + scoped_ptr<RewriteOptions> custom_options(options); + + if (!ok) { + TSError("Failure while determining request options for psol resource"); + // options = server_context->global_options(); + } else { + // ps_determine_options modified url, removing any ModPagespeedFoo=Bar query + // parameters. Keep url_string in sync with url. + // TODO(oschaaf): we really should determine if we have to do the lookup + intercept_ctx->request_ctx->gurl->Spec().CopyToString(intercept_ctx->request_ctx->url_string); + } + + // The url we have here is already checked for IsWebValid() + net_instaweb::ResourceFetch::Start( + GoogleUrl(*intercept_ctx->request_ctx->url_string), + custom_options.release() /* null if there aren't custom options */, + false /* using_spdy */, server_context, intercept_ctx->request_ctx->base_fetch); + } else { + int64_t numBytesToWrite, numBytesWritten; + numBytesToWrite = intercept_ctx->response->size(); + numBytesWritten = TSIOBufferWrite(intercept_ctx->resp_buffer, + intercept_ctx->response->c_str(), numBytesToWrite); + + if (numBytesWritten == numBytesToWrite) { + TSVConnWrite(intercept_ctx->vconn, cont, intercept_ctx->resp_reader, numBytesToWrite); + } else { + TSError("Not all output could be written in one go"); + DCHECK(false); + } + } + } break; + case TS_EVENT_VCONN_EOS: + TSVConnShutdown(intercept_ctx->vconn, 1, 0); + break; + case TS_EVENT_VCONN_READ_COMPLETE: { + TSVConnShutdown(intercept_ctx->vconn, 1, 0); + } break; + case TS_EVENT_VCONN_WRITE_READY: + break; + case TS_EVENT_VCONN_WRITE_COMPLETE: + shutDown = true; + break; + case TS_EVENT_ERROR: + TSError("vconn event: error %s", intercept_ctx->request_ctx->url_string->c_str()); + shutDown = true; + break; + case TS_EVENT_NET_ACCEPT_FAILED: + TSError("vconn event: accept failed"); + shutDown = true; + break; + case TS_EVENT_IMMEDIATE: + case TS_EVENT_TIMEOUT: + break; + default: + TSError("default clause event: %d", event); + break; + } + + if (shutDown) { + shutdown(cont, intercept_ctx); + } + + return 1; +} + +// We intercept here because serving from ats's own cache is faster +// then serving from pagespeed's cache. (which needs to be looked in to) +static int +read_cache_header_callback(TSCont cont, TSEvent event, void *edata) +{ + TSHttpTxn txn = static_cast<TSHttpTxn>(edata); + TransformCtx* ctx = get_transaction_context(txn); + + if (ctx == NULL) { + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + return 0; + } + if (!ctx->resource_request) { + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + return 0; + } + // TODO(oschaaf): FIXME: Ownership of ctx has become too mucky. + // This is because I realised too late that the intercepts + // are able to outlive the transaction, which I hacked + // to work. + if (TSHttpIsInternalRequest(txn) == TS_SUCCESS) { + ats_ctx_destroy(ctx); + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + return 0; + } + + if (cache_hit(txn)) { + ats_ctx_destroy(ctx); + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + return 0; + } + + AtsServerContext* server_context = ctx->server_context; + AtsRewriteDriverFactory* factory = (AtsRewriteDriverFactory*)server_context->factory(); + GoogleString output; + StringWriter writer(&output); + HttpStatus::Code status = HttpStatus::kOK; + ContentType content_type = kContentTypeHtml; + StringPiece cache_control = HttpAttributes::kNoCache; + const char* error_message = NULL; + StringPiece request_uri_path = ctx->gurl->PathAndLeaf(); + + if (false && ctx->gurl->PathSansQuery() == "/robots.txt") { + content_type = kContentTypeText; + writer.Write("User-agent: *\n", server_context->message_handler()); + writer.Write("Disallow: /\n", server_context->message_handler()); + } + + // TODO(oschaaf): /pagespeed_admin handling + else { + // Optimized resource are highly cacheable (1 year expiry) + // TODO(oschaaf): configuration + TSHttpTxnRespCacheableSet(txn, 1); + TSHttpTxnReqCacheableSet(txn, 1); + + TSMBuffer reqp; + TSMLoc req_hdr_loc; + if (TSHttpTxnClientReqGet(ctx->txn, &reqp, &req_hdr_loc) != TS_SUCCESS) { + TSError("Error TSHttpTxnClientReqGet for resource!"); + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + return 0; + } + + TSCont interceptCont = TSContCreate(resource_intercept, TSMutexCreate()); + InterceptCtx *intercept_ctx = new InterceptCtx(); + intercept_ctx->request_ctx = ctx; + intercept_ctx->request_headers = new RequestHeaders(); + copy_request_headers_to_psol(reqp, req_hdr_loc, intercept_ctx->request_headers); + TSHandleMLocRelease(reqp, TS_NULL_MLOC, req_hdr_loc); + + + TSContDataSet(interceptCont, intercept_ctx); + TSHttpTxnServerIntercept(interceptCont, txn); + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + return 0; + } + + if (error_message != NULL) { + status = HttpStatus::kNotFound; + content_type = kContentTypeHtml; + output = error_message; + } + + ResponseHeaders response_headers; + response_headers.SetStatusAndReason(status); + response_headers.set_major_version(1); + response_headers.set_minor_version(0); + + response_headers.Add(HttpAttributes::kContentType, content_type.mime_type()); + + int64 now_ms = factory->timer()->NowMs(); + response_headers.SetDate(now_ms); + response_headers.SetLastModified(now_ms); + response_headers.Add(HttpAttributes::kCacheControl, cache_control); + + if (FindIgnoreCase(cache_control, "private") == + static_cast<int>(StringPiece::npos)) { + response_headers.Add(HttpAttributes::kEtag, "W/\"0\""); + } + + GoogleString header; + StringWriter header_writer(&header); + response_headers.WriteAsHttp(&header_writer, server_context->message_handler()); + + TSCont interceptCont = TSContCreate(resource_intercept, TSMutexCreate()); + InterceptCtx *intercept_ctx = new InterceptCtx(); + intercept_ctx->request_ctx = ctx; + header.append(output); + TSHttpTxnRespCacheableSet(txn, 0); + TSHttpTxnReqCacheableSet(txn, 0); + TSContDataSet(interceptCont, intercept_ctx); + TSHttpTxnServerIntercept(interceptCont, txn); + intercept_ctx->response->append(header); + + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + return 0; +} + +void setup_resource_intercept() +{ + TSCont cont = TSContCreate(read_cache_header_callback, NULL); + TSHttpHookAdd(TS_HTTP_CACHE_LOOKUP_COMPLETE_HOOK, cont); +} + http://git-wip-us.apache.org/repos/asf/trafficserver/blob/e8b899b0/plugins/experimental/ats_pagespeed/ats_resource_intercept.h ---------------------------------------------------------------------- diff --git a/plugins/experimental/ats_pagespeed/ats_resource_intercept.h b/plugins/experimental/ats_pagespeed/ats_resource_intercept.h new file mode 100644 index 0000000..933f20f --- /dev/null +++ b/plugins/experimental/ats_pagespeed/ats_resource_intercept.h @@ -0,0 +1,29 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef ATS_RESOURCE_INTERCEPT_H +#define ATS_RESOURCE_INTERCEPT_H + +void setup_resource_intercept(); + +#endif // ATS_INTERCEPT_H http://git-wip-us.apache.org/repos/asf/trafficserver/blob/e8b899b0/plugins/experimental/ats_pagespeed/ats_rewrite_driver_factory.cc ---------------------------------------------------------------------- diff --git a/plugins/experimental/ats_pagespeed/ats_rewrite_driver_factory.cc b/plugins/experimental/ats_pagespeed/ats_rewrite_driver_factory.cc new file mode 100644 index 0000000..4ca3d87 --- /dev/null +++ b/plugins/experimental/ats_pagespeed/ats_rewrite_driver_factory.cc @@ -0,0 +1,196 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "ats_rewrite_driver_factory.h" + +#include <cstdio> +#include <vector> + +#include "ats_thread_system.h" +#include "ats_message_handler.h" +#include "ats_server_context.h" + +#include "net/instaweb/http/public/content_type.h" +#include "net/instaweb/http/public/rate_controller.h" +#include "net/instaweb/http/public/rate_controlling_url_async_fetcher.h" +#include "net/instaweb/http/public/wget_url_fetcher.h" +#include "net/instaweb/rewriter/public/rewrite_driver.h" +#include "net/instaweb/rewriter/public/rewrite_driver_factory.h" +#include "net/instaweb/rewriter/public/server_context.h" +#include "net/instaweb/rewriter/public/static_asset_manager.h" +#include "net/instaweb/system/public/in_place_resource_recorder.h" +#include "net/instaweb/system/public/serf_url_async_fetcher.h" +#include "net/instaweb/system/public/system_caches.h" +#include "net/instaweb/system/public/system_rewrite_options.h" +#include "net/instaweb/util/public/google_message_handler.h" +#include "net/instaweb/util/public/null_shared_mem.h" +#include "net/instaweb/util/public/posix_timer.h" +#include "net/instaweb/util/public/property_cache.h" +#include "net/instaweb/util/public/pthread_shared_mem.h" +#include "net/instaweb/util/public/scheduler_thread.h" +#include "net/instaweb/util/public/shared_circular_buffer.h" +#include "net/instaweb/util/public/shared_mem_statistics.h" +#include "net/instaweb/util/public/slow_worker.h" +#include "net/instaweb/util/public/stdio_file_system.h" +#include "net/instaweb/util/public/string.h" +#include "net/instaweb/util/public/string_util.h" +#include "net/instaweb/util/public/thread_system.h" + + +namespace net_instaweb { + + + AtsRewriteDriverFactory::AtsRewriteDriverFactory( + const ProcessContext& process_context, + AtsThreadSystem* thread_system, + StringPiece hostname, int port) + : SystemRewriteDriverFactory(process_context, + thread_system, NULL /*default shared mem runtime*/, + "" /*hostname, not used*/, -1/*port, not used*/) + , ats_message_handler_(new AtsMessageHandler(thread_system->NewMutex())) + , ats_html_parse_message_handler_(new AtsMessageHandler(thread_system->NewMutex())) + , use_per_vhost_statistics_(false) + , threads_started_(false) + { + InitializeDefaultOptions(); + default_options()->set_beacon_url("/ats_pagespeed_beacon"); + default_options()->set_enabled(RewriteOptions::kEnabledOn); + default_options()->SetRewriteLevel(RewriteOptions::kCoreFilters); + + SystemRewriteOptions* system_options = dynamic_cast<SystemRewriteOptions*>( + default_options()); + system_options->set_log_dir("/tmp/ps_log/"); + system_options->set_statistics_logging_enabled(true); + + system_options->set_file_cache_clean_inode_limit(500000); + system_options->set_file_cache_clean_size_kb(1024*10000);// 10 GB + system_options->set_avoid_renaming_introspective_javascript(true); + system_options->set_file_cache_path("/tmp/ats_ps/"); + system_options->set_lru_cache_byte_limit(163840); + system_options->set_lru_cache_kb_per_process(1024*500);//500 MB + + system_options->set_flush_html(true); + + AtsRewriteOptions* ats_options = (AtsRewriteOptions*)system_options; + std::vector<std::string> args; + args.push_back("RateLimitBackgroundFetches"); + args.push_back("on"); + global_settings settings; + const char* msg = ats_options->ParseAndSetOptions(args, ats_message_handler_, settings); + CHECK(!msg); + + set_message_buffer_size(1024*128); + set_message_handler(ats_message_handler_); + set_html_parse_message_handler(ats_html_parse_message_handler_); + StartThreads(); + } + + AtsRewriteDriverFactory::~AtsRewriteDriverFactory() { + ShutDown(); + delete ats_message_handler_; + ats_message_handler_ = NULL; + delete ats_html_parse_message_handler_; + ats_html_parse_message_handler_ = NULL; + STLDeleteElements(&uninitialized_server_contexts_); + } + + void AtsRewriteDriverFactory::InitStaticAssetManager(StaticAssetManager* static_js_manager) { + static_js_manager->set_library_url_prefix("/ats_pagespeed_static/"); + } + + Hasher* AtsRewriteDriverFactory::NewHasher() { + return new MD5Hasher; + } + + MessageHandler* AtsRewriteDriverFactory::DefaultHtmlParseMessageHandler() { + return ats_html_parse_message_handler_; + } + + MessageHandler* AtsRewriteDriverFactory::DefaultMessageHandler() { + return ats_message_handler_; + } + + FileSystem* AtsRewriteDriverFactory::DefaultFileSystem() { + return new StdioFileSystem(); + } + + Timer* AtsRewriteDriverFactory::DefaultTimer() { + return new PosixTimer; + } + + NamedLockManager* AtsRewriteDriverFactory::DefaultLockManager() { + CHECK(false) << "default lock manager should not be called"; + return NULL; + } + + RewriteOptions* AtsRewriteDriverFactory::NewRewriteOptions() { + AtsRewriteOptions* options = new AtsRewriteOptions(thread_system()); + options->SetRewriteLevel(RewriteOptions::kCoreFilters); + return options; + } + + ServerContext* AtsRewriteDriverFactory::NewDecodingServerContext() { + ServerContext* sc = new AtsServerContext(this); + InitStubDecodingServerContext(sc); + return sc; + } + + void AtsRewriteDriverFactory::InitStats(Statistics* statistics) { + // Init standard PSOL stats. + SystemRewriteDriverFactory::InitStats(statistics); + // Init Ats-specific stats. + AtsServerContext::InitStats(statistics); + } + + + AtsServerContext* AtsRewriteDriverFactory::MakeAtsServerContext() { + AtsServerContext* server_context = new AtsServerContext(this); + uninitialized_server_contexts_.insert(server_context); + return server_context; + } + + ServerContext* AtsRewriteDriverFactory::NewServerContext() { + LOG(DFATAL) << "MakeAtsServerContext should be used instead"; + return NULL; + } + +net_instaweb::QueuedWorkerPool* AtsRewriteDriverFactory::CreateWorkerPool(net_instaweb::RewriteDriverFactory::WorkerPoolCategory pool, + StringPiece name) { + int tc = 8; + TSDebug("ats_pagespeed", "Created new QueuedWorkerPool of type %d named '%s' of size %d", pool, name.data(), tc); + net_instaweb::QueuedWorkerPool *q_pool = new net_instaweb::QueuedWorkerPool(tc, name, thread_system()); + return q_pool; +} + +void AtsRewriteDriverFactory::StartThreads() { + if (threads_started_) { + CHECK(false) << "threads already started"; + } + SchedulerThread* thread = new SchedulerThread(thread_system(), scheduler()); + bool ok = thread->Start(); + CHECK(ok) << "Unable to start scheduler thread"; + defer_cleanup(thread->MakeDeleter()); + threads_started_ = true; +} + +} // namespace net_instaweb http://git-wip-us.apache.org/repos/asf/trafficserver/blob/e8b899b0/plugins/experimental/ats_pagespeed/ats_rewrite_driver_factory.h ---------------------------------------------------------------------- diff --git a/plugins/experimental/ats_pagespeed/ats_rewrite_driver_factory.h b/plugins/experimental/ats_pagespeed/ats_rewrite_driver_factory.h new file mode 100644 index 0000000..de18a28 --- /dev/null +++ b/plugins/experimental/ats_pagespeed/ats_rewrite_driver_factory.h @@ -0,0 +1,113 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef ATS_REWRITE_DRIVER_FACTORY_H_ +#define ATS_REWRITE_DRIVER_FACTORY_H_ + +#include <set> + +#include "net/instaweb/system/public/system_rewrite_driver_factory.h" +#include "net/instaweb/util/public/md5_hasher.h" +#include "net/instaweb/util/public/scoped_ptr.h" + + +namespace net_instaweb { + + + class AbstractSharedMem; + //class NgxMessageHandler; + //class NgxRewriteOptions; + class AtsServerContext; + class AtsThreadSystem; + class GoogleMessageHandler; + //class NgxUrlAsyncFetcher; + class SharedCircularBuffer; + class SharedMemRefererStatistics; + class SharedMemStatistics; + class SlowWorker; + class StaticAssetManager; + class Statistics; + class StaticAssetManager; + //class SystemCaches; + +class AtsRewriteDriverFactory : public SystemRewriteDriverFactory { + public: + explicit AtsRewriteDriverFactory(const ProcessContext& process_context, + AtsThreadSystem* thread_system, + StringPiece hostname, int port); + virtual ~AtsRewriteDriverFactory(); + + virtual Hasher* NewHasher(); + virtual MessageHandler* DefaultHtmlParseMessageHandler(); + virtual MessageHandler* DefaultMessageHandler(); + virtual FileSystem* DefaultFileSystem(); + virtual Timer* DefaultTimer(); + virtual NamedLockManager* DefaultLockManager(); + virtual RewriteOptions* NewRewriteOptions(); + virtual ServerContext* NewDecodingServerContext(); + + virtual bool UseBeaconResultsInFilters() const { + return true; + } + + virtual void InitStaticAssetManager(StaticAssetManager* static_js_manager); + + // Initializes all the statistics objects created transitively by + // AtsRewriteDriverFactory, including nginx-specific and + // platform-independent statistics. + static void InitStats(Statistics* statistics); + + virtual net_instaweb::QueuedWorkerPool* CreateWorkerPool(WorkerPoolCategory pool, + StringPiece name); + virtual void NonStaticInitStats(Statistics* statistics) { + InitStats(statistics); + } + + AtsServerContext* MakeAtsServerContext(); + ServerContext* NewServerContext(); + //AbstractSharedMem* shared_mem_runtime() const { + // return shared_mem_runtime_.get(); + //} + + // Starts pagespeed threads if they've not been started already. Must be + // called after the caller has finished any forking it intends to do. + void StartThreads(); + bool use_per_vhost_statistics() const { + return use_per_vhost_statistics_; + } + void set_use_per_vhost_statistics(bool x) { + use_per_vhost_statistics_ = x; + } + + protected: + private: + //scoped_ptr<AbstractSharedMem> shared_mem_runtime_; + GoogleMessageHandler* ats_message_handler_; + GoogleMessageHandler* ats_html_parse_message_handler_; + bool use_per_vhost_statistics_; + bool threads_started_; +}; + +} // namespace net_instaweb + +#endif // ATS_REWRITE_DRIVER_FACTORY_H_ http://git-wip-us.apache.org/repos/asf/trafficserver/blob/e8b899b0/plugins/experimental/ats_pagespeed/ats_rewrite_options.cc ---------------------------------------------------------------------- diff --git a/plugins/experimental/ats_pagespeed/ats_rewrite_options.cc b/plugins/experimental/ats_pagespeed/ats_rewrite_options.cc new file mode 100644 index 0000000..172db83 --- /dev/null +++ b/plugins/experimental/ats_pagespeed/ats_rewrite_options.cc @@ -0,0 +1,263 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "ats_rewrite_options.h" + +#include "net/instaweb/public/version.h" +#include "net/instaweb/rewriter/public/rewrite_options.h" +#include "net/instaweb/util/public/timer.h" + +#include "net/instaweb/util/public/message_handler.h" +#include "net/instaweb/rewriter/public/file_load_policy.h" + +#include "net/instaweb/util/public/stdio_file_system.h" + +#include "ats_message_handler.h" +#include "ats_rewrite_driver_factory.h" + +using namespace std; + +namespace net_instaweb { + + +RewriteOptions::Properties* AtsRewriteOptions::ats_properties_ = NULL; + +AtsRewriteOptions::AtsRewriteOptions(ThreadSystem* thread_system) + : SystemRewriteOptions(thread_system) { + + Init(); +} + +void AtsRewriteOptions::Init() { + DCHECK(ats_properties_ != NULL) + << "Call AtsRewriteOptions::Initialize() before construction"; + InitializeOptions(ats_properties_); +} + +void AtsRewriteOptions::AddProperties() { + MergeSubclassProperties(ats_properties_); + AtsRewriteOptions dummy_config(NULL); + + dummy_config.set_default_x_header_value(MOD_PAGESPEED_VERSION_STRING "-" LASTCHANGE_STRING); +} + +void AtsRewriteOptions::Initialize() { + if (Properties::Initialize(&ats_properties_)) { + SystemRewriteOptions::Initialize(); + AddProperties(); + } +} + +void AtsRewriteOptions::Terminate() { + if (Properties::Terminate(&ats_properties_)) { + SystemRewriteOptions::Terminate(); + } +} + +bool AtsRewriteOptions::IsDirective(StringPiece config_directive, + StringPiece compare_directive) { + return StringCaseEqual(config_directive, compare_directive); +} + +RewriteOptions::OptionSettingResult AtsRewriteOptions::ParseAndSetOptions0( + StringPiece directive, GoogleString* msg, MessageHandler* handler) { + if (IsDirective(directive, "on")) { + set_enabled(RewriteOptions::kEnabledOn); + } else if (IsDirective(directive, "off")) { + set_enabled(RewriteOptions::kEnabledOff); + } else if (IsDirective(directive, "unplugged")) { + set_enabled(RewriteOptions::kEnabledUnplugged); + } else { + return RewriteOptions::kOptionNameUnknown; + } + return RewriteOptions::kOptionOk; +} + + +RewriteOptions::OptionSettingResult +AtsRewriteOptions::ParseAndSetOptionFromName1( + StringPiece name, StringPiece arg, + GoogleString* msg, MessageHandler* handler) { + // FileCachePath needs error checking. + if (StringCaseEqual(name, kFileCachePath)) { + if (!StringCaseStartsWith(arg, "/")) { + *msg = "must start with a slash"; + return RewriteOptions::kOptionValueInvalid; + } + } + + return SystemRewriteOptions::ParseAndSetOptionFromName1( + name, arg, msg, handler); +} + +bool AtsRewriteOptions::SetBoolFlag(bool* v, StringPiece arg) { + if (IsDirective(arg, "on")) { + *v=true; + return true; + } else if (IsDirective(arg, "off")) { + *v=false; + return true; + } + return false; +} + +const char* +AtsRewriteOptions::ParseAndSetOptions( + vector<string> args, MessageHandler* handler, global_settings& global_config) { + int n_args = args.size(); + CHECK_GE(n_args, 1); + + StringPiece directive = args[0]; + + // Remove initial "ModPagespeed" if there is one. + StringPiece mod_pagespeed("ModPagespeed"); + if (StringCaseStartsWith(directive, mod_pagespeed)) { + directive.remove_prefix(mod_pagespeed.size()); + } + + GoogleString msg; + OptionSettingResult result; + if (n_args == 1) { + result = ParseAndSetOptions0(directive, &msg, handler); + } else if (n_args == 2) { + StringPiece arg = args[1]; + if (IsDirective(directive, "UsePerVHostStatistics")) { + if (!SetBoolFlag(&global_config.use_per_vhost_statistics,arg)) { + msg = "Failed to set UsePerVHostStatistics value"; + result = RewriteOptions::kOptionValueInvalid; + } else { + result = RewriteOptions::kOptionOk; + } + } /* else if (IsDirective(directive, "InstallCrashHandler")) { + // Not applicable + } */ else if (IsDirective(directive, "MessageBufferSize")) { + int message_buffer_size; + bool ok = StringToInt(arg.as_string(), &message_buffer_size); + if (ok && message_buffer_size >= 0) { + global_config.message_buffer_size = message_buffer_size; + result = RewriteOptions::kOptionOk; + } else { + msg = "Failed to set MessageBufferSize value"; + result = RewriteOptions::kOptionValueInvalid; + } + } else if (IsDirective(directive, "UseNativeFetcher")) { + if (!SetBoolFlag(&global_config.info_urls_local_only,arg)) { + msg = "Failed to set UseNativeFetcher value"; + result = RewriteOptions::kOptionValueInvalid; + } else { + msg = "Native fetcher is not available in this release"; + + result = RewriteOptions::kOptionValueInvalid; + } + } else if (IsDirective(directive, "InfoUrlsLocalOnly")) { + if (!SetBoolFlag(&global_config.info_urls_local_only, arg)) { + msg = "Failed to set InfoUrlsLocalOnly value"; + result = RewriteOptions::kOptionValueInvalid; + } else { + result = RewriteOptions::kOptionOk; + } + }/* else if (IsDirective(directive, "RateLimitBackgroundFetches")) { + if (!SetBoolFlag(&global_config.rate_limit_background_fetches, arg)) { + msg = "Failed to set RateLimitBackgroundFetches value"; + result = RewriteOptions::kOptionValueInvalid; + } else { + result = RewriteOptions::kOptionOk; + } + } else if (IsDirective(directive, "ForceCaching")) { + if (!SetBoolFlag(&global_config.force_caching, arg)) { + msg = "Failed to set ForceCaching value"; + result = RewriteOptions::kOptionValueInvalid; + } else { + result = RewriteOptions::kOptionOk; + } + } else if (IsDirective(directive, "ListOutstandingUrlsOnError")) { + if (!SetBoolFlag(&global_config.list_outstanding_urls_on_error, arg)) { + msg = "Failed to set ListOutstandingUrlsOnError value"; + result = RewriteOptions::kOptionValueInvalid; + } else { + result = RewriteOptions::kOptionOk; + } + } else if (IsDirective(directive, "TrackOriginalContentLength")) { + if (!SetBoolFlag(&global_config.track_original_content_length, arg)) { + msg = "Failed to set TrackOriginalContentLength value"; + result = RewriteOptions::kOptionValueInvalid; + } else { + result = RewriteOptions::kOptionOk; + } + } */else { + result = ParseAndSetOptionFromName1(directive, args[1], &msg, handler); + } + } else if (n_args == 3) { + if (StringCaseEqual(directive, "CreateSharedMemoryMetadataCache")) { + int64 kb = 0; + if (!StringToInt64(args[2], &kb) || kb < 0) { + result = RewriteOptions::kOptionValueInvalid; + msg = "size_kb must be a positive 64-bit integer"; + } else { + global_config.shm_cache_size_kb = kb; + result = kOptionOk; + //bool ok = driver_factory->caches()->CreateShmMetadataCache( + // args[1].as_string(), kb, &msg); + //result = ok ? kOptionOk : kOptionValueInvalid; + } + } else { + result = ParseAndSetOptionFromName2(directive, args[1], args[2], + &msg, handler); + } + } else if (n_args == 4) { + result = ParseAndSetOptionFromName3( + directive, args[1], args[2], args[3], &msg, handler); + } else { + return "unknown option"; + } + + if (msg.size()) { + handler->Message(kWarning, "Error handling config line [%s]: [%s]", + JoinString(args, ' ').c_str(), msg.c_str()); + } + + switch (result) { + case RewriteOptions::kOptionOk: + return NULL; + case RewriteOptions::kOptionNameUnknown: + handler->Message(kWarning, "%s", JoinString(args, ' ').c_str()); + return "unknown option"; + case RewriteOptions::kOptionValueInvalid: { + handler->Message(kWarning, "%s", JoinString(args, ' ').c_str()); + return "Invalid value"; + } + } + + CHECK(false); + return NULL; +} + +AtsRewriteOptions* AtsRewriteOptions::Clone() const { + AtsRewriteOptions* options = new AtsRewriteOptions(this->thread_system()); + options->Merge(*this); + return options; +} + + +} // namespace net_instaweb +
