http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ats_resource_intercept.cc ---------------------------------------------------------------------- diff --git a/plugins/experimental/ats_speed/ats_resource_intercept.cc b/plugins/experimental/ats_speed/ats_resource_intercept.cc new file mode 100644 index 0000000..e54d15d --- /dev/null +++ b/plugins/experimental/ats_speed/ats_resource_intercept.cc @@ -0,0 +1,376 @@ +// Copyright 2013 We-Amp B.V. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: [email protected] (Otto van der Schaaf) +#include <ts/ts.h> + +#include <stdio.h> + +#include "ats_resource_intercept.h" + + +#include "ats_base_fetch.h" +#include "ats_rewrite_driver_factory.h" +#include "ats_rewrite_options.h" +#include "ats_server_context.h" +#include "ats_speed.h" + +#include "net/instaweb/http/public/request_context.h" +#include "net/instaweb/rewriter/public/resource_fetch.h" +#include "net/instaweb/rewriter/public/static_asset_manager.h" +#include "net/instaweb/system/public/system_request_context.h" + +#include "net/instaweb/util/public/string_writer.h" +#include "net/instaweb/system/public/handlers.h" + +using namespace net_instaweb; + +struct InterceptCtx +{ + TSVConn vconn; + TSIOBuffer req_buffer; + TSIOBufferReader req_reader; + TSIOBuffer resp_buffer; + TSIOBufferReader resp_reader; + GoogleString* response; + TransformCtx* request_ctx; + RequestHeaders* request_headers; + + InterceptCtx() + : vconn(NULL) + , req_buffer(NULL) + , req_reader(NULL) + , resp_buffer(NULL) + , resp_reader(NULL) + , response( new GoogleString() ) + , request_ctx(NULL) + , request_headers(NULL) + { + }; +}; + +static void +shutdown (TSCont cont, InterceptCtx * intercept_ctx) { + if (intercept_ctx->req_reader != NULL) { + TSIOBufferReaderFree(intercept_ctx->req_reader); + intercept_ctx->req_reader = NULL; + } + if (intercept_ctx->req_buffer != NULL) { + TSIOBufferDestroy(intercept_ctx->req_buffer); + intercept_ctx->req_buffer = NULL; + } + if (intercept_ctx->resp_reader != NULL) { + TSIOBufferReaderFree(intercept_ctx->resp_reader); + intercept_ctx->resp_reader = NULL; + } + if (intercept_ctx->resp_buffer != NULL) { + TSIOBufferDestroy(intercept_ctx->resp_buffer); + intercept_ctx->resp_buffer = NULL; + } + if (intercept_ctx->vconn != NULL) { + TSVConnShutdown(intercept_ctx->vconn, 0, 1); + TSVConnClose(intercept_ctx->vconn); + intercept_ctx->vconn = NULL; + } + if (intercept_ctx->response != NULL) { + delete intercept_ctx->response; + intercept_ctx->response = NULL; + } + // TODO(oschaaf): think the ordering of this one through. + if (intercept_ctx->request_ctx) { + ats_ctx_destroy(intercept_ctx->request_ctx); + intercept_ctx->request_ctx = NULL; + } + if (intercept_ctx->request_headers != NULL) { + delete intercept_ctx->request_headers; + intercept_ctx->request_headers = NULL; + } + delete intercept_ctx; + TSContDestroy(cont); +} + +static int +resource_intercept(TSCont cont, TSEvent event, void *edata) +{ + InterceptCtx *intercept_ctx = static_cast<InterceptCtx *>(TSContDataGet(cont)); + bool shutDown = false; + + // TODO(oschaaf): have a look at https://github.com/apache/trafficserver/blob/master/plugins/experimental/esi/serverIntercept.c + // and see if we have any edge cases we should fix. + switch (event) { + case TS_EVENT_NET_ACCEPT: { + intercept_ctx->vconn = static_cast<TSVConn>(edata); + intercept_ctx->req_buffer = TSIOBufferCreate(); + intercept_ctx->req_reader = TSIOBufferReaderAlloc(intercept_ctx->req_buffer); + intercept_ctx->resp_buffer = TSIOBufferCreate(); + intercept_ctx->resp_reader = TSIOBufferReaderAlloc(intercept_ctx->resp_buffer); + TSVConnRead(intercept_ctx->vconn, cont, intercept_ctx->req_buffer, 0x7fffffff); + } break; + case TS_EVENT_VCONN_READ_READY: { + CHECK(intercept_ctx->request_ctx->base_fetch == NULL) << "Base fetch must not be set!"; + CHECK(intercept_ctx->request_ctx->url_string != NULL) << "Url must be set!"; + + TSVConnShutdown(intercept_ctx->vconn, 1, 0); + + // response will already have a size for internal pages at this point. + // resources, however, will have to be fetched. + // TODO(oschaaf): this is extremely ugly. + if (intercept_ctx->response->size() == 0) { + // TODO(oschaaf): unused - must we close / clean this up? + TSVIO downstream_vio = TSVConnWrite( + intercept_ctx->vconn, cont, intercept_ctx->resp_reader, 0x7fffffff); + + AtsServerContext* server_context = intercept_ctx->request_ctx->server_context; + + // TODO:(oschaaf) host/port + SystemRequestContext* system_request_context = + new SystemRequestContext(server_context->thread_system()->NewMutex(), + server_context->timer(), + 80, + "127.0.0.1"); + + intercept_ctx->request_ctx->base_fetch = new AtsBaseFetch( + server_context, RequestContextPtr(system_request_context), + downstream_vio, intercept_ctx->resp_buffer, true); + intercept_ctx->request_ctx->base_fetch->set_request_headers( + intercept_ctx->request_headers); + + RewriteOptions* options = NULL; + + //const char* host = intercept_ctx->request_headers->Lookup1(HttpAttributes::kHost); + const char* host = intercept_ctx->request_ctx->gurl->HostAndPort().as_string().c_str(); + if (host != NULL && strlen(host) > 0) { + intercept_ctx->request_ctx->options = get_host_options(host); + } + + // TODO(oschaaf): directory options should be coming from configuration! + bool ok = ps_determine_options(server_context, + intercept_ctx->request_ctx->options, + intercept_ctx->request_ctx->base_fetch->request_headers(), + intercept_ctx->request_ctx->base_fetch->response_headers(), + &options, + intercept_ctx->request_ctx->gurl); + + // Take ownership of custom_options. + scoped_ptr<RewriteOptions> custom_options(options); + + if (!ok) { + TSError("Failure while determining request options for psol resource"); + // options = server_context->global_options(); + } else { + // ps_determine_options modified url, removing any ModPagespeedFoo=Bar query + // parameters. Keep url_string in sync with url. + // TODO(oschaaf): we really should determine if we have to do the lookup + intercept_ctx->request_ctx->gurl->Spec().CopyToString(intercept_ctx->request_ctx->url_string); + } + + // The url we have here is already checked for IsWebValid() + net_instaweb::ResourceFetch::Start( + GoogleUrl(*intercept_ctx->request_ctx->url_string), + custom_options.release() /* null if there aren't custom options */, + false /* using_spdy */, server_context, intercept_ctx->request_ctx->base_fetch); + } else { + int64_t numBytesToWrite, numBytesWritten; + numBytesToWrite = intercept_ctx->response->size(); + numBytesWritten = TSIOBufferWrite(intercept_ctx->resp_buffer, + intercept_ctx->response->c_str(), numBytesToWrite); + + if (numBytesWritten == numBytesToWrite) { + TSVConnWrite(intercept_ctx->vconn, cont, intercept_ctx->resp_reader, numBytesToWrite); + } else { + TSError("Not all output could be written in one go"); + DCHECK(false); + } + } + } break; + case TS_EVENT_VCONN_EOS: + TSVConnShutdown(intercept_ctx->vconn, 1, 0); + break; + case TS_EVENT_VCONN_READ_COMPLETE: { + TSVConnShutdown(intercept_ctx->vconn, 1, 0); + } break; + case TS_EVENT_VCONN_WRITE_READY: + break; + case TS_EVENT_VCONN_WRITE_COMPLETE: + shutDown = true; + break; + case TS_EVENT_ERROR: + TSError("vconn event: error %s", intercept_ctx->request_ctx->url_string->c_str()); + shutDown = true; + break; + case TS_EVENT_NET_ACCEPT_FAILED: + TSError("vconn event: accept failed"); + shutDown = true; + break; + case TS_EVENT_IMMEDIATE: + case TS_EVENT_TIMEOUT: + break; + default: + TSError("default clause event: %d", event); + break; + } + + if (shutDown) { + shutdown(cont, intercept_ctx); + } + + return 1; +} + +// We intercept here because serving from ats's own cache is faster +// then serving from pagespeed's cache. (which needs to be looked in to) +static int +read_cache_header_callback(TSCont cont, TSEvent event, void *edata) +{ + TSHttpTxn txn = static_cast<TSHttpTxn>(edata); + TransformCtx* ctx = get_transaction_context(txn); + + if (ctx == NULL) { + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + return 0; + } + if (!ctx->resource_request) { + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + return 0; + } + // TODO(oschaaf): FIXME: Ownership of ctx has become too mucky. + // This is because I realised too late that the intercepts + // are able to outlive the transaction, which I hacked + // to work. + if (TSHttpIsInternalRequest(txn) == TS_SUCCESS) { + ats_ctx_destroy(ctx); + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + return 0; + } + + if (cache_hit(txn)) { + ats_ctx_destroy(ctx); + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + return 0; + } + + AtsServerContext* server_context = ctx->server_context; + AtsRewriteDriverFactory* factory = (AtsRewriteDriverFactory*)server_context->factory(); + GoogleString output; + StringWriter writer(&output); + HttpStatus::Code status = HttpStatus::kOK; + ContentType content_type = kContentTypeHtml; + StringPiece cache_control = HttpAttributes::kNoCache; + const char* error_message = NULL; + StringPiece request_uri_path = ctx->gurl->PathAndLeaf(); + + if (ctx->gurl->PathSansQuery() == "/robots.txt") { + content_type = kContentTypeText; + writer.Write("User-agent: *\n", server_context->message_handler()); + writer.Write("Disallow: /\n", server_context->message_handler()); + } else if (ctx->gurl->PathSansQuery() == "/pagespeed_message") { + // TODO(oschaaf)... let's wait for a bit with this one. + } else if (ctx->gurl->PathSansQuery() == "/pagespeed_statistics" || ctx->gurl->PathSansQuery() == "/pagespeed_global_statistics") { + error_message = StatisticsHandler( + factory, + server_context, + NULL, // No SPDY-specific config in ats_pagespeed. + !factory->use_per_vhost_statistics() || StringCaseStartsWith( + request_uri_path, "/pagespeed_global_statistics"), + StringPiece(ctx->gurl->Query().as_string().c_str()), + &content_type, + &writer, + server_context->message_handler()); + } else if (ctx->gurl->PathSansLeaf() == "/ats_speed_static/") { + StringPiece file_contents; + if (server_context->static_asset_manager()->GetAsset( + request_uri_path.substr( + strlen("/ats_speed_static/")), + &file_contents, &content_type, &cache_control)) { + file_contents.CopyToString(&output); + } else { + error_message = "Static asset not found"; + } + } else if (ctx->gurl->PathSansQuery() == "/pagespeed_console") { + ConsoleHandler(server_context, server_context->config(), &writer, server_context->message_handler()); + } else { + // Optimized resource are highly cacheable (1 year expiry) + // TODO(oschaaf): configuration + TSHttpTxnRespCacheableSet(txn, 1); + TSHttpTxnReqCacheableSet(txn, 1); + + TSMBuffer reqp; + TSMLoc req_hdr_loc; + if (TSHttpTxnClientReqGet(ctx->txn, &reqp, &req_hdr_loc) != TS_SUCCESS) { + TSError("Error TSHttpTxnClientReqGet for resource!"); + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + return 0; + } + + TSCont interceptCont = TSContCreate(resource_intercept, TSMutexCreate()); + InterceptCtx *intercept_ctx = new InterceptCtx(); + intercept_ctx->request_ctx = ctx; + intercept_ctx->request_headers = new RequestHeaders(); + copy_request_headers_to_psol(reqp, req_hdr_loc, intercept_ctx->request_headers); + TSHandleMLocRelease(reqp, TS_NULL_MLOC, req_hdr_loc); + + + TSContDataSet(interceptCont, intercept_ctx); + TSHttpTxnServerIntercept(interceptCont, txn); + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + return 0; + } + + if (error_message != NULL) { + status = HttpStatus::kNotFound; + content_type = kContentTypeHtml; + output = error_message; + } + + ResponseHeaders response_headers; + response_headers.SetStatusAndReason(status); + response_headers.set_major_version(1); + response_headers.set_minor_version(0); + + response_headers.Add(HttpAttributes::kContentType, content_type.mime_type()); + + int64 now_ms = factory->timer()->NowMs(); + response_headers.SetDate(now_ms); + response_headers.SetLastModified(now_ms); + response_headers.Add(HttpAttributes::kCacheControl, cache_control); + + if (FindIgnoreCase(cache_control, "private") == + static_cast<int>(StringPiece::npos)) { + response_headers.Add(HttpAttributes::kEtag, "W/\"0\""); + } + + GoogleString header; + StringWriter header_writer(&header); + response_headers.WriteAsHttp(&header_writer, server_context->message_handler()); + + TSCont interceptCont = TSContCreate(resource_intercept, TSMutexCreate()); + InterceptCtx *intercept_ctx = new InterceptCtx(); + intercept_ctx->request_ctx = ctx; + header.append(output); + TSHttpTxnRespCacheableSet(txn, 0); + TSHttpTxnReqCacheableSet(txn, 0); + TSContDataSet(interceptCont, intercept_ctx); + TSHttpTxnServerIntercept(interceptCont, txn); + intercept_ctx->response->append(header); + + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + return 0; +} + +void setup_resource_intercept() +{ + TSCont cont = TSContCreate(read_cache_header_callback, NULL); + TSHttpHookAdd(TS_HTTP_CACHE_LOOKUP_COMPLETE_HOOK, cont); +} +
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ats_resource_intercept.h ---------------------------------------------------------------------- diff --git a/plugins/experimental/ats_speed/ats_resource_intercept.h b/plugins/experimental/ats_speed/ats_resource_intercept.h new file mode 100644 index 0000000..31f4208 --- /dev/null +++ b/plugins/experimental/ats_speed/ats_resource_intercept.h @@ -0,0 +1,21 @@ +// Copyright 2013 We-Amp B.V. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: [email protected] (Otto van der Schaaf) +#ifndef ATS_RESOURCE_INTERCEPT_H +#define ATS_RESOURCE_INTERCEPT_H + +void setup_resource_intercept(); + +#endif // ATS_INTERCEPT_H http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ats_rewrite_driver_factory.cc ---------------------------------------------------------------------- diff --git a/plugins/experimental/ats_speed/ats_rewrite_driver_factory.cc b/plugins/experimental/ats_speed/ats_rewrite_driver_factory.cc new file mode 100644 index 0000000..d04502d --- /dev/null +++ b/plugins/experimental/ats_speed/ats_rewrite_driver_factory.cc @@ -0,0 +1,178 @@ +// Copyright 2013 We-Amp B.V. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: [email protected] (Otto van der Schaaf) +#include "ats_rewrite_driver_factory.h" + +#include <cstdio> +#include <vector> + +#include "ats_thread_system.h" +#include "ats_message_handler.h" +#include "ats_server_context.h" + +#include "net/instaweb/http/public/content_type.h" +#include "net/instaweb/http/public/rate_controller.h" +#include "net/instaweb/http/public/rate_controlling_url_async_fetcher.h" +#include "net/instaweb/http/public/wget_url_fetcher.h" +#include "net/instaweb/rewriter/public/rewrite_driver.h" +#include "net/instaweb/rewriter/public/rewrite_driver_factory.h" +#include "net/instaweb/rewriter/public/server_context.h" +#include "net/instaweb/rewriter/public/static_asset_manager.h" +#include "net/instaweb/system/public/in_place_resource_recorder.h" +#include "net/instaweb/system/public/serf_url_async_fetcher.h" +#include "net/instaweb/system/public/system_caches.h" +#include "net/instaweb/system/public/system_rewrite_options.h" +#include "net/instaweb/util/public/google_message_handler.h" +#include "net/instaweb/util/public/null_shared_mem.h" +#include "net/instaweb/util/public/posix_timer.h" +#include "net/instaweb/util/public/property_cache.h" +#include "net/instaweb/util/public/pthread_shared_mem.h" +#include "net/instaweb/util/public/scheduler_thread.h" +#include "net/instaweb/util/public/shared_circular_buffer.h" +#include "net/instaweb/util/public/shared_mem_statistics.h" +#include "net/instaweb/util/public/slow_worker.h" +#include "net/instaweb/util/public/stdio_file_system.h" +#include "net/instaweb/util/public/string.h" +#include "net/instaweb/util/public/string_util.h" +#include "net/instaweb/util/public/thread_system.h" + + +namespace net_instaweb { + + + AtsRewriteDriverFactory::AtsRewriteDriverFactory(AtsThreadSystem* thread_system) + // TODO(oschaaf): fix hostname/port (?) + : SystemRewriteDriverFactory(thread_system, new PthreadSharedMem(), StringPiece("foohost"), 8080) + , ats_message_handler_(new AtsMessageHandler(thread_system->NewMutex())) + , ats_html_parse_message_handler_(new AtsMessageHandler(thread_system->NewMutex())) + , use_per_vhost_statistics_(false) + , threads_started_(false) + { + InitializeDefaultOptions(); + default_options()->set_beacon_url("/ats_speed_beacon"); + default_options()->set_enabled(RewriteOptions::kEnabledOn); + default_options()->SetRewriteLevel(RewriteOptions::kCoreFilters); + + SystemRewriteOptions* system_options = dynamic_cast<SystemRewriteOptions*>( + default_options()); + system_options->set_log_dir("/tmp/ps_log/"); + system_options->set_statistics_logging_enabled(true); + + system_options->set_file_cache_clean_inode_limit(500000); + system_options->set_file_cache_clean_size_kb(1024*10000);// 10 GB + system_options->set_avoid_renaming_introspective_javascript(true); + system_options->set_file_cache_path("/tmp/ats_ps/"); + system_options->set_lru_cache_byte_limit(163840); + system_options->set_lru_cache_kb_per_process(1024*500);//500 MB + + system_options->set_flush_html(true); + + AtsRewriteOptions* ats_options = (AtsRewriteOptions*)system_options; + std::vector<std::string> args; + args.push_back("RateLimitBackgroundFetches"); + args.push_back("on"); + global_settings settings; + const char* msg = ats_options->ParseAndSetOptions(args, ats_message_handler_, settings); + CHECK(!msg); + + set_message_buffer_size(1024*128); + set_message_handler(ats_message_handler_); + set_html_parse_message_handler(ats_html_parse_message_handler_); + StartThreads(); + } + + AtsRewriteDriverFactory::~AtsRewriteDriverFactory() { + ShutDown(); + delete ats_message_handler_; + ats_message_handler_ = NULL; + delete ats_html_parse_message_handler_; + ats_html_parse_message_handler_ = NULL; + STLDeleteElements(&uninitialized_server_contexts_); + } + + void AtsRewriteDriverFactory::InitStaticAssetManager(StaticAssetManager* static_js_manager) { + static_js_manager->set_library_url_prefix("/ats_speed_static/"); + } + + Hasher* AtsRewriteDriverFactory::NewHasher() { + return new MD5Hasher; + } + + MessageHandler* AtsRewriteDriverFactory::DefaultHtmlParseMessageHandler() { + return ats_html_parse_message_handler_; + } + + MessageHandler* AtsRewriteDriverFactory::DefaultMessageHandler() { + return ats_message_handler_; + } + + FileSystem* AtsRewriteDriverFactory::DefaultFileSystem() { + return new StdioFileSystem(); + } + + Timer* AtsRewriteDriverFactory::DefaultTimer() { + return new PosixTimer; + } + + NamedLockManager* AtsRewriteDriverFactory::DefaultLockManager() { + CHECK(false) << "default lock manager should not be called"; + return NULL; + } + + RewriteOptions* AtsRewriteDriverFactory::NewRewriteOptions() { + AtsRewriteOptions* options = new AtsRewriteOptions(thread_system()); + options->SetRewriteLevel(RewriteOptions::kCoreFilters); + return options; + } + + void AtsRewriteDriverFactory::InitStats(Statistics* statistics) { + // Init standard PSOL stats. + SystemRewriteDriverFactory::InitStats(statistics); + // Init Ats-specific stats. + AtsServerContext::InitStats(statistics); + } + + + AtsServerContext* AtsRewriteDriverFactory::MakeAtsServerContext() { + AtsServerContext* server_context = new AtsServerContext(this); + uninitialized_server_contexts_.insert(server_context); + return server_context; + } + + ServerContext* AtsRewriteDriverFactory::NewServerContext() { + LOG(DFATAL) << "MakeAtsServerContext should be used instead"; + return NULL; + } + +net_instaweb::QueuedWorkerPool* AtsRewriteDriverFactory::CreateWorkerPool(net_instaweb::RewriteDriverFactory::WorkerPoolCategory pool, + StringPiece name) { + int tc = 8; + TSDebug("ats_speed", "Created new QueuedWorkerPool of type %d named '%s' of size %d", pool, name.data(), tc); + net_instaweb::QueuedWorkerPool *q_pool = new net_instaweb::QueuedWorkerPool(tc, name, thread_system()); + return q_pool; +} + +void AtsRewriteDriverFactory::StartThreads() { + if (threads_started_) { + CHECK(false) << "threads already started"; + } + SchedulerThread* thread = new SchedulerThread(thread_system(), scheduler()); + bool ok = thread->Start(); + CHECK(ok) << "Unable to start scheduler thread"; + defer_cleanup(thread->MakeDeleter()); + threads_started_ = true; +} + +} // namespace net_instaweb http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ats_rewrite_driver_factory.h ---------------------------------------------------------------------- diff --git a/plugins/experimental/ats_speed/ats_rewrite_driver_factory.h b/plugins/experimental/ats_speed/ats_rewrite_driver_factory.h new file mode 100644 index 0000000..800cd34 --- /dev/null +++ b/plugins/experimental/ats_speed/ats_rewrite_driver_factory.h @@ -0,0 +1,102 @@ +// Copyright 2013 We-Amp B.V. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: [email protected] (Otto van der Schaaf) +#ifndef ATS_REWRITE_DRIVER_FACTORY_H_ +#define ATS_REWRITE_DRIVER_FACTORY_H_ + +#include <set> + +#include "net/instaweb/system/public/system_rewrite_driver_factory.h" +#include "net/instaweb/util/public/md5_hasher.h" +#include "net/instaweb/util/public/scoped_ptr.h" + + +namespace net_instaweb { + + + class AbstractSharedMem; + //class NgxMessageHandler; + //class NgxRewriteOptions; + class AtsServerContext; + class AtsThreadSystem; + class GoogleMessageHandler; + //class NgxUrlAsyncFetcher; + class SharedCircularBuffer; + class SharedMemRefererStatistics; + class SharedMemStatistics; + class SlowWorker; + class StaticAssetManager; + class Statistics; + class StaticAssetManager; + //class SystemCaches; + +class AtsRewriteDriverFactory : public SystemRewriteDriverFactory { + public: + explicit AtsRewriteDriverFactory(AtsThreadSystem* thread_system); + virtual ~AtsRewriteDriverFactory(); + + virtual Hasher* NewHasher(); + virtual MessageHandler* DefaultHtmlParseMessageHandler(); + virtual MessageHandler* DefaultMessageHandler(); + virtual FileSystem* DefaultFileSystem(); + virtual Timer* DefaultTimer(); + virtual NamedLockManager* DefaultLockManager(); + virtual RewriteOptions* NewRewriteOptions(); + + virtual bool UseBeaconResultsInFilters() const { + return true; + } + + virtual void InitStaticAssetManager(StaticAssetManager* static_js_manager); + + // Initializes all the statistics objects created transitively by + // AtsRewriteDriverFactory, including nginx-specific and + // platform-independent statistics. + static void InitStats(Statistics* statistics); + + virtual net_instaweb::QueuedWorkerPool* CreateWorkerPool(WorkerPoolCategory pool, + StringPiece name); + virtual void NonStaticInitStats(Statistics* statistics) { + InitStats(statistics); + } + + AtsServerContext* MakeAtsServerContext(); + ServerContext* NewServerContext(); + //AbstractSharedMem* shared_mem_runtime() const { + // return shared_mem_runtime_.get(); + //} + + // Starts pagespeed threads if they've not been started already. Must be + // called after the caller has finished any forking it intends to do. + void StartThreads(); + bool use_per_vhost_statistics() const { + return use_per_vhost_statistics_; + } + void set_use_per_vhost_statistics(bool x) { + use_per_vhost_statistics_ = x; + } + + protected: + private: + //scoped_ptr<AbstractSharedMem> shared_mem_runtime_; + GoogleMessageHandler* ats_message_handler_; + GoogleMessageHandler* ats_html_parse_message_handler_; + bool use_per_vhost_statistics_; + bool threads_started_; +}; + +} // namespace net_instaweb + +#endif // ATS_REWRITE_DRIVER_FACTORY_H_ http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ats_rewrite_options.cc ---------------------------------------------------------------------- diff --git a/plugins/experimental/ats_speed/ats_rewrite_options.cc b/plugins/experimental/ats_speed/ats_rewrite_options.cc new file mode 100644 index 0000000..cb4f3c1 --- /dev/null +++ b/plugins/experimental/ats_speed/ats_rewrite_options.cc @@ -0,0 +1,256 @@ +// Copyright 2013 We-Amp B.V. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: [email protected] (Otto van der Schaaf) +#include "ats_rewrite_options.h" + +#include "net/instaweb/public/version.h" +#include "net/instaweb/rewriter/public/rewrite_options.h" +#include "net/instaweb/util/public/timer.h" + +#include "net/instaweb/util/public/message_handler.h" +#include "net/instaweb/rewriter/public/file_load_policy.h" + +#include "net/instaweb/util/public/stdio_file_system.h" + +#include "ats_message_handler.h" +#include "ats_rewrite_driver_factory.h" + +using namespace std; + +namespace net_instaweb { + + +RewriteOptions::Properties* AtsRewriteOptions::ats_properties_ = NULL; + +AtsRewriteOptions::AtsRewriteOptions(ThreadSystem* thread_system) + : SystemRewriteOptions(thread_system) { + + Init(); +} + +void AtsRewriteOptions::Init() { + DCHECK(ats_properties_ != NULL) + << "Call AtsRewriteOptions::Initialize() before construction"; + InitializeOptions(ats_properties_); + statistics_handler_path_.set_default("/pagespeed_global_statistics"); +} + +void AtsRewriteOptions::AddProperties() { + MergeSubclassProperties(ats_properties_); + AtsRewriteOptions dummy_config(NULL); + + dummy_config.set_default_x_header_value(MOD_PAGESPEED_VERSION_STRING "-" LASTCHANGE_STRING); +} + +void AtsRewriteOptions::Initialize() { + if (Properties::Initialize(&ats_properties_)) { + SystemRewriteOptions::Initialize(); + AddProperties(); + } +} + +void AtsRewriteOptions::Terminate() { + if (Properties::Terminate(&ats_properties_)) { + SystemRewriteOptions::Terminate(); + } +} + +bool AtsRewriteOptions::IsDirective(StringPiece config_directive, + StringPiece compare_directive) { + return StringCaseEqual(config_directive, compare_directive); +} + +RewriteOptions::OptionSettingResult AtsRewriteOptions::ParseAndSetOptions0( + StringPiece directive, GoogleString* msg, MessageHandler* handler) { + if (IsDirective(directive, "on")) { + set_enabled(RewriteOptions::kEnabledOn); + } else if (IsDirective(directive, "off")) { + set_enabled(RewriteOptions::kEnabledOff); + } else if (IsDirective(directive, "unplugged")) { + set_enabled(RewriteOptions::kEnabledUnplugged); + } else { + return RewriteOptions::kOptionNameUnknown; + } + return RewriteOptions::kOptionOk; +} + + +RewriteOptions::OptionSettingResult +AtsRewriteOptions::ParseAndSetOptionFromName1( + StringPiece name, StringPiece arg, + GoogleString* msg, MessageHandler* handler) { + // FileCachePath needs error checking. + if (StringCaseEqual(name, kFileCachePath)) { + if (!StringCaseStartsWith(arg, "/")) { + *msg = "must start with a slash"; + return RewriteOptions::kOptionValueInvalid; + } + } + + return SystemRewriteOptions::ParseAndSetOptionFromName1( + name, arg, msg, handler); +} + +bool AtsRewriteOptions::SetBoolFlag(bool* v, StringPiece arg) { + if (IsDirective(arg, "on")) { + *v=true; + return true; + } else if (IsDirective(arg, "off")) { + *v=false; + return true; + } + return false; +} + +const char* +AtsRewriteOptions::ParseAndSetOptions( + vector<string> args, MessageHandler* handler, global_settings& global_config) { + int n_args = args.size(); + CHECK_GE(n_args, 1); + + StringPiece directive = args[0]; + + // Remove initial "ModPagespeed" if there is one. + StringPiece mod_pagespeed("ModPagespeed"); + if (StringCaseStartsWith(directive, mod_pagespeed)) { + directive.remove_prefix(mod_pagespeed.size()); + } + + GoogleString msg; + OptionSettingResult result; + if (n_args == 1) { + result = ParseAndSetOptions0(directive, &msg, handler); + } else if (n_args == 2) { + StringPiece arg = args[1]; + if (IsDirective(directive, "UsePerVHostStatistics")) { + if (!SetBoolFlag(&global_config.use_per_vhost_statistics,arg)) { + msg = "Failed to set UsePerVHostStatistics value"; + result = RewriteOptions::kOptionValueInvalid; + } else { + result = RewriteOptions::kOptionOk; + } + } /* else if (IsDirective(directive, "InstallCrashHandler")) { + // Not applicable + } */ else if (IsDirective(directive, "MessageBufferSize")) { + int message_buffer_size; + bool ok = StringToInt(arg.as_string(), &message_buffer_size); + if (ok && message_buffer_size >= 0) { + global_config.message_buffer_size = message_buffer_size; + result = RewriteOptions::kOptionOk; + } else { + msg = "Failed to set MessageBufferSize value"; + result = RewriteOptions::kOptionValueInvalid; + } + } else if (IsDirective(directive, "UseNativeFetcher")) { + if (!SetBoolFlag(&global_config.info_urls_local_only,arg)) { + msg = "Failed to set UseNativeFetcher value"; + result = RewriteOptions::kOptionValueInvalid; + } else { + msg = "Native fetcher is not available in this release"; + + result = RewriteOptions::kOptionValueInvalid; + } + } else if (IsDirective(directive, "InfoUrlsLocalOnly")) { + if (!SetBoolFlag(&global_config.info_urls_local_only, arg)) { + msg = "Failed to set InfoUrlsLocalOnly value"; + result = RewriteOptions::kOptionValueInvalid; + } else { + result = RewriteOptions::kOptionOk; + } + }/* else if (IsDirective(directive, "RateLimitBackgroundFetches")) { + if (!SetBoolFlag(&global_config.rate_limit_background_fetches, arg)) { + msg = "Failed to set RateLimitBackgroundFetches value"; + result = RewriteOptions::kOptionValueInvalid; + } else { + result = RewriteOptions::kOptionOk; + } + } else if (IsDirective(directive, "ForceCaching")) { + if (!SetBoolFlag(&global_config.force_caching, arg)) { + msg = "Failed to set ForceCaching value"; + result = RewriteOptions::kOptionValueInvalid; + } else { + result = RewriteOptions::kOptionOk; + } + } else if (IsDirective(directive, "ListOutstandingUrlsOnError")) { + if (!SetBoolFlag(&global_config.list_outstanding_urls_on_error, arg)) { + msg = "Failed to set ListOutstandingUrlsOnError value"; + result = RewriteOptions::kOptionValueInvalid; + } else { + result = RewriteOptions::kOptionOk; + } + } else if (IsDirective(directive, "TrackOriginalContentLength")) { + if (!SetBoolFlag(&global_config.track_original_content_length, arg)) { + msg = "Failed to set TrackOriginalContentLength value"; + result = RewriteOptions::kOptionValueInvalid; + } else { + result = RewriteOptions::kOptionOk; + } + } */else { + result = ParseAndSetOptionFromName1(directive, args[1], &msg, handler); + } + } else if (n_args == 3) { + if (StringCaseEqual(directive, "CreateSharedMemoryMetadataCache")) { + int64 kb = 0; + if (!StringToInt64(args[2], &kb) || kb < 0) { + result = RewriteOptions::kOptionValueInvalid; + msg = "size_kb must be a positive 64-bit integer"; + } else { + global_config.shm_cache_size_kb = kb; + result = kOptionOk; + //bool ok = driver_factory->caches()->CreateShmMetadataCache( + // args[1].as_string(), kb, &msg); + //result = ok ? kOptionOk : kOptionValueInvalid; + } + } else { + result = ParseAndSetOptionFromName2(directive, args[1], args[2], + &msg, handler); + } + } else if (n_args == 4) { + result = ParseAndSetOptionFromName3( + directive, args[1], args[2], args[3], &msg, handler); + } else { + return "unknown option"; + } + + if (msg.size()) { + handler->Message(kWarning, "Error handling config line [%s]: [%s]", + JoinString(args, ' ').c_str(), msg.c_str()); + } + + switch (result) { + case RewriteOptions::kOptionOk: + return NULL; + case RewriteOptions::kOptionNameUnknown: + handler->Message(kWarning, "%s", JoinString(args, ' ').c_str()); + return "unknown option"; + case RewriteOptions::kOptionValueInvalid: { + handler->Message(kWarning, "%s", JoinString(args, ' ').c_str()); + return "Invalid value"; + } + } + + CHECK(false); + return NULL; +} + +AtsRewriteOptions* AtsRewriteOptions::Clone() const { + AtsRewriteOptions* options = new AtsRewriteOptions(this->thread_system()); + options->Merge(*this); + return options; +} + + +} // namespace net_instaweb + http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ats_rewrite_options.h ---------------------------------------------------------------------- diff --git a/plugins/experimental/ats_speed/ats_rewrite_options.h b/plugins/experimental/ats_speed/ats_rewrite_options.h new file mode 100644 index 0000000..f7cfde3 --- /dev/null +++ b/plugins/experimental/ats_speed/ats_rewrite_options.h @@ -0,0 +1,95 @@ +// Copyright 2013 We-Amp B.V. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: [email protected] (Otto van der Schaaf) +#ifndef ATS_REWRITE_OPTIONS_H_ +#define ATS_REWRITE_OPTIONS_H_ + +#include <string> +#include <vector> + +#include "net/instaweb/util/public/string.h" +#include "net/instaweb/util/public/string_util.h" +#include "net/instaweb/rewriter/public/rewrite_options.h" +#include "net/instaweb/system/public/system_rewrite_options.h" + + +//#include "ats_configuration.h" + + +namespace net_instaweb { + +class ThreadSystem; + +struct global_settings { + global_settings() + : info_urls_local_only(true) + , use_native_fetcher(false) + , use_per_vhost_statistics(true) + , message_buffer_size(1024*128) + , shm_cache_size_kb(0) + //, rate_limit_background_fetches(true) + //, force_caching(false) + //, list_outstanding_urls_on_error(false) + //, track_original_content_length(false) + { + } + bool info_urls_local_only; + bool use_native_fetcher; + bool use_per_vhost_statistics; + int message_buffer_size; + //bool rate_limit_background_fetches; + //bool force_caching; + //bool list_outstanding_urls_on_error; + //bool track_original_content_length; + int shm_cache_size_kb; +}; + + +class AtsRewriteOptions : public SystemRewriteOptions { + public: + // See rewrite_options::Initialize and ::Terminate + static void Initialize(); + static void Terminate(); + + AtsRewriteOptions(ThreadSystem* thread_system); + virtual ~AtsRewriteOptions() { + } + + const char* ParseAndSetOptions( + std::vector<std::string> args, MessageHandler* handler, global_settings& global_config); + + virtual AtsRewriteOptions* Clone() const; + OptionSettingResult ParseAndSetOptions0( + StringPiece directive, GoogleString* msg, MessageHandler* handler); + + virtual OptionSettingResult ParseAndSetOptionFromName1( + StringPiece name, StringPiece arg, + GoogleString* msg, MessageHandler* handler); + + + private: + bool SetBoolFlag(bool* v, StringPiece arg); + static Properties* ats_properties_; + static void AddProperties(); + void Init(); + + bool IsDirective(StringPiece config_directive, StringPiece compare_directive); + + DISALLOW_COPY_AND_ASSIGN(AtsRewriteOptions); + }; + +} // namespace net_instaweb + +#endif // ATS_REWRITE_OPTIONS_H_ http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ats_server_context.cc ---------------------------------------------------------------------- diff --git a/plugins/experimental/ats_speed/ats_server_context.cc b/plugins/experimental/ats_speed/ats_server_context.cc new file mode 100644 index 0000000..91c3bb7 --- /dev/null +++ b/plugins/experimental/ats_speed/ats_server_context.cc @@ -0,0 +1,38 @@ +// Copyright 2013 We-Amp B.V. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: [email protected] (Otto van der Schaaf) +#include "ats_server_context.h" +#include "ats_rewrite_driver_factory.h" +#include "ats_rewrite_options.h" + +#include "net/instaweb/system/public/system_caches.h" + + +using namespace net_instaweb; + +AtsRewriteOptions* AtsServerContext::config() { + return (AtsRewriteOptions*)global_options(); +} + +AtsServerContext::AtsServerContext(AtsRewriteDriverFactory* factory) : + // TODO(oschaaf): host/port + SystemServerContext(factory, "foo.com" /*hostname*/, 8080/*port*/), + initialized_(false), + ats_factory_(factory) { +} + +AtsServerContext::~AtsServerContext() { + +} http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ats_server_context.h ---------------------------------------------------------------------- diff --git a/plugins/experimental/ats_speed/ats_server_context.h b/plugins/experimental/ats_speed/ats_server_context.h new file mode 100644 index 0000000..51c5e6b --- /dev/null +++ b/plugins/experimental/ats_speed/ats_server_context.h @@ -0,0 +1,48 @@ +// Copyright 2013 We-Amp B.V. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: [email protected] (Otto van der Schaaf) +#ifndef ATS_SERVER_CONTEXT_H_ +#define ATS_SERVER_CONTEXT_H_ + +#include "ats_rewrite_options.h" + +#include "net/instaweb/system/public/system_server_context.h" +#include "net/instaweb/util/public/statistics.h" + +namespace net_instaweb { + +class AtsRewriteOptions; +class AtsRewriteDriverFactory; + +class AtsServerContext : public net_instaweb::SystemServerContext { + public: + explicit AtsServerContext(AtsRewriteDriverFactory* factory); + virtual ~AtsServerContext(); + + virtual bool ProxiesHtml() const { + return true; + } + + AtsRewriteOptions *config(); + AtsRewriteDriverFactory *ats_rewrite_driver_factory() { return ats_factory_; } + + private: + bool initialized_; + AtsRewriteDriverFactory* ats_factory_; +}; + +} /* ats_pagespeed */ + +#endif /* ATS_SERVER_CONTEXT_H_ */ http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ats_speed.cc ---------------------------------------------------------------------- diff --git a/plugins/experimental/ats_speed/ats_speed.cc b/plugins/experimental/ats_speed/ats_speed.cc new file mode 100644 index 0000000..afefa37 --- /dev/null +++ b/plugins/experimental/ats_speed/ats_speed.cc @@ -0,0 +1,1083 @@ +// Copyright 2013 We-Amp B.V. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: [email protected] (Otto van der Schaaf) +// TODO(oschaaf): remove what isn't used +#ifndef __STDC_LIMIT_MACROS +#define __STDC_LIMIT_MACROS +#endif +#include <stdlib.h> +#include <errno.h> +#include <stdio.h> +#include <limits.h> +#include <stdint.h> +#include <sys/inotify.h> +#include <unistd.h> + +#include <ts/ts.h> + +#include <vector> +#include <set> + + +#include "ats_speed.h" + +#include "ats_config.h" +#include "ats_demo_filter.h" +#include "ats_header_utils.h" +#include "ats_rewrite_options.h" +#include "ats_log_message_handler.h" + +#include "base/logging.h" +#include "net/instaweb/http/public/response_headers.h" +#include "net/instaweb/util/public/string_util.h" + +#include "ats_base_fetch.h" +#include "ats_resource_intercept.h" +#include "ats_beacon_intercept.h" +#include "ats_process_context.h" +#include "ats_rewrite_driver_factory.h" +#include "ats_rewrite_options.h" +#include "ats_server_context.h" + +#include "net/instaweb/rewriter/public/rewrite_stats.h" +#include "net/instaweb/system/public/in_place_resource_recorder.h" + +#include "net/instaweb/automatic/public/proxy_fetch.h" +#include "net/instaweb/http/public/content_type.h" +#include "net/instaweb/http/public/request_context.h" +#include "net/instaweb/rewriter/public/experiment_matcher.h" +#include "net/instaweb/rewriter/public/experiment_util.h" +#include "net/instaweb/rewriter/public/process_context.h" +#include "net/instaweb/rewriter/public/resource_fetch.h" +#include "net/instaweb/rewriter/public/rewrite_driver.h" +#include "net/instaweb/rewriter/public/static_asset_manager.h" +#include "net/instaweb/system/public/handlers.h" +#include "net/instaweb/public/global_constants.h" +#include "net/instaweb/public/version.h" +#include "net/instaweb/util/public/google_message_handler.h" +#include "net/instaweb/util/public/google_url.h" +#include "net/instaweb/util/public/gzip_inflater.h" +#include "net/instaweb/util/public/query_params.h" +#include "net/instaweb/util/public/statistics_logger.h" +#include "net/instaweb/util/public/stdio_file_system.h" +#include "net/instaweb/util/public/string.h" +#include "net/instaweb/util/public/string_writer.h" +#include "net/instaweb/util/public/time_util.h" +#include "net/instaweb/util/stack_buffer.h" +#include "net/instaweb/system/public/system_request_context.h" + + +#include <dirent.h> + +using namespace net_instaweb; + +static AtsProcessContext* ats_process_context; +static const char* DEBUG_TAG = "ats_speed_transform"; +static int TXN_INDEX_ARG; +static int TXN_INDEX_OWNED_ARG; +static int TXN_INDEX_OWNED_ARG_SET; +static int TXN_INDEX_OWNED_ARG_UNSET; +TSMutex config_mutex = TSMutexCreate(); +AtsConfig* config = NULL; +TransformCtx* get_transaction_context(TSHttpTxn txnp) { + return (TransformCtx *) TSHttpTxnArgGet(txnp, TXN_INDEX_ARG); +} + +static TransformCtx * +ats_ctx_alloc() +{ + TransformCtx *ctx; + + ctx = (TransformCtx *) TSmalloc(sizeof(TransformCtx)); + ctx->downstream_vio = NULL; + ctx->downstream_buffer = NULL; + ctx->downstream_length = 0; + ctx->state = transform_state_initialized; + + ctx->base_fetch = NULL; + ctx->proxy_fetch = NULL; + + ctx->inflater = NULL; + ctx->url_string = NULL; + ctx->gurl = NULL; + ctx->write_pending = false; + ctx->fetch_done = false; + ctx->resource_request = false; + ctx->beacon_request = false; + ctx->transform_added = false; + ctx->mps_user_agent = false; + ctx->user_agent = NULL; + ctx->server_context = NULL; + ctx->html_rewrite = false; + ctx->request_method = NULL; + ctx->alive = 0xaaaa; + ctx->options = NULL; + ctx->to_host = NULL; + return ctx; +} + +void +ats_ctx_destroy(TransformCtx * ctx) +{ + TSReleaseAssert(ctx); + CHECK(ctx->alive == 0xaaaa) << "Already dead!"; + ctx->alive = 0xbbbb; + + if (ctx->base_fetch != NULL) { + ctx->base_fetch->Release(); + ctx->base_fetch = NULL; + } + + if (ctx->proxy_fetch != NULL) { + ctx->proxy_fetch->Done(false /* failure */); + ctx->proxy_fetch = NULL; + } + + if (ctx->inflater != NULL) { + delete ctx->inflater; + ctx->inflater = NULL; + } + + if (ctx->downstream_buffer) { + TSIOBufferDestroy(ctx->downstream_buffer); + } + + if (ctx->url_string != NULL) { + delete ctx->url_string; + ctx->url_string = NULL; + } + + if (ctx->gurl != NULL) { + delete ctx->gurl; + ctx->gurl = NULL; + } + if (ctx->user_agent != NULL) { + delete ctx->user_agent; + ctx->user_agent = NULL; + } + ctx->request_method = NULL; + if (ctx->options != NULL) { + delete ctx->options; + ctx->options = NULL; + } + if (ctx->to_host != NULL) { + delete ctx->to_host; + ctx->to_host = NULL; + } + TSfree(ctx); +} + +RewriteOptions* ps_determine_request_options( + ServerContext* server_context, + RequestHeaders* request_headers, + ResponseHeaders* response_headers, + GoogleUrl* url) { + // Stripping ModPagespeed query params before the property cache lookup to + // make cache key consistent for both lookup and storing in cache. + // + // Sets option from request headers and url. + ServerContext::OptionsBoolPair query_options_success = + server_context->GetQueryOptions(url, request_headers, + response_headers); + bool get_query_options_success = query_options_success.second; + if (!get_query_options_success) { + // Failed to parse query params or request headers. Treat this as if there + // were no query params given. + TSError("ps_route rerquest: parsing headers or query params failed."); + return NULL; + } + + // Will be NULL if there aren't any options set with query params or in + // headers. + return query_options_success.first; +} + +bool ps_determine_options(ServerContext* server_context, + // Directory-specific options, usually null. They've already been rebased off + // of the global options as part of the configuration process. + RewriteOptions* directory_options, + RequestHeaders* request_headers, + ResponseHeaders* response_headers, + RewriteOptions** options, + GoogleUrl* url) { + // Global options for this server. Never null. + RewriteOptions* global_options = server_context->global_options(); + + // Request-specific options, nearly always null. If set they need to be + // rebased on the directory options or the global options. + RewriteOptions* request_options = ps_determine_request_options( + server_context, request_headers, response_headers, url); + + // Because the caller takes ownership of any options we return, the only + // situation in which we can avoid allocating a new RewriteOptions is if the + // global options are ok as are. + if (directory_options == NULL && request_options == NULL && + !global_options->running_experiment()) { + return true; + } + + // Start with directory options if we have them, otherwise request options. + if (directory_options != NULL) { + //*options = directory_options->Clone(); + // OS: HACK! TODO! + *options = global_options->Clone(); + (*options)->Merge(*directory_options); + } else { + *options = global_options->Clone(); + } + + // Modify our options in response to request options or experiment settings, + // if we need to. If there are request options then ignore the experiment + // because we don't want experiments to be contaminated with unexpected + // settings. + if (request_options != NULL) { + (*options)->Merge(*request_options); + delete request_options; + } + // TODO(oschaaf): experiments + /*else if ((*options)->running_experiment()) { + bool ok = ps_set_experiment_state_and_cookie( + r, request_headers, *options, url->Host()); + if (!ok) { + delete *options; + *options = NULL; + return false; + } + }*/ + + return true; +} + +void +handle_send_response_headers(TSHttpTxn txnp) { + TransformCtx* ctx = get_transaction_context(txnp); + // TODO(oschaaf): Fix the response headers!! + bool is_owned = TSHttpTxnArgGet(txnp, TXN_INDEX_OWNED_ARG) == &TXN_INDEX_OWNED_ARG_SET; + if (!is_owned) { + return; + } + CHECK(ctx->alive == 0xaaaa) << "Already dead !"; + if (ctx->html_rewrite) { + TSMBuffer bufp = NULL; + TSMLoc hdr_loc = NULL; + if (ctx->base_fetch == NULL) { + // TODO(oschaaf): figure out when this happens. + return; + } + + if (TSHttpTxnClientRespGet(txnp, &bufp, &hdr_loc) == TS_SUCCESS) { + ResponseHeaders* pagespeed_headers = + ctx->base_fetch->response_headers(); + for (int i = 0 ; i < pagespeed_headers->NumAttributes() ; i++) { + const GoogleString& name_gs = pagespeed_headers->Name(i); + const GoogleString& value_gs = pagespeed_headers->Value(i); + + // We should avoid touching these fields, as ATS will drop keepalive when we do. + if ( StringCaseEqual(name_gs, "Connection") || StringCaseEqual(name_gs, "Transfer-Encoding") ) { + continue; + } + + TSMLoc field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, name_gs.data(), name_gs.size()); + if (field_loc != NULL) { + TSMimeHdrFieldValuesClear(bufp, hdr_loc, field_loc); + TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, field_loc, -1, + value_gs.data(), value_gs.size()); + } else if (TSMimeHdrFieldCreate(bufp, hdr_loc, &field_loc) == TS_SUCCESS) { + if (TSMimeHdrFieldNameSet(bufp, hdr_loc, field_loc, name_gs.data(), name_gs.size()) == TS_SUCCESS) { + TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, field_loc, -1, + value_gs.data(), value_gs.size()); + TSMimeHdrFieldAppend(bufp, hdr_loc, field_loc); + } else { + CHECK(false) << "Field name set failure"; + } + TSHandleMLocRelease(bufp, hdr_loc, field_loc); + } else { + CHECK(false) << "Field create failure"; + } + } + + TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc); + } else { + DCHECK(false) << "Could not get response headers?!"; + } + } +} + +static void +copy_response_headers_to_psol(TSMBuffer bufp, TSMLoc hdr_loc, ResponseHeaders* psol_headers) { + int n_mime_headers = TSMimeHdrFieldsCount(bufp, hdr_loc); + TSMLoc field_loc; + const char *name, *value; + int name_len, value_len; + GoogleString header; + for (int i = 0; i < n_mime_headers; ++i) { + field_loc = TSMimeHdrFieldGet(bufp, hdr_loc, i); + if (!field_loc) { + TSDebug(DEBUG_TAG, "[%s] Error while obtaining header field #%d", __FUNCTION__, i); + continue; + } + name = TSMimeHdrFieldNameGet(bufp, hdr_loc, field_loc, &name_len); + StringPiece s_name(name, name_len); + int n_field_values = TSMimeHdrFieldValuesCount(bufp, hdr_loc, field_loc); + for (int j = 0; j < n_field_values; ++j) { + value = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, field_loc, j, &value_len); + if ( NULL == value || !value_len ) { + TSDebug(DEBUG_TAG, "[%s] Error while getting value #%d of header [%.*s]", + __FUNCTION__, j, name_len, name); + } else { + StringPiece s_value(value, value_len); + psol_headers->Add(s_name, s_value); + //TSDebug(DEBUG_TAG, "Add response header [%.*s:%.*s]",name_len, name, value_len, value); + } + } + TSHandleMLocRelease(bufp, hdr_loc, field_loc); + } +} + +void +copy_request_headers_to_psol(TSMBuffer bufp, TSMLoc hdr_loc, RequestHeaders* psol_headers) { + int n_mime_headers = TSMimeHdrFieldsCount(bufp, hdr_loc); + TSMLoc field_loc; + const char *name, *value; + int name_len, value_len; + GoogleString header; + for (int i = 0; i < n_mime_headers; ++i) { + field_loc = TSMimeHdrFieldGet(bufp, hdr_loc, i); + if (!field_loc) { + TSDebug(DEBUG_TAG, "[%s] Error while obtaining header field #%d", __FUNCTION__, i); + continue; + } + name = TSMimeHdrFieldNameGet(bufp, hdr_loc, field_loc, &name_len); + StringPiece s_name(name, name_len); + int n_field_values = TSMimeHdrFieldValuesCount(bufp, hdr_loc, field_loc); + for (int j = 0; j < n_field_values; ++j) { + value = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, field_loc, j, &value_len); + if ( NULL == value || !value_len ) { + TSDebug(DEBUG_TAG, "[%s] Error while getting value #%d of header [%.*s]", + __FUNCTION__, j, name_len, name); + } else { + StringPiece s_value(value, value_len); + psol_headers->Add(s_name, s_value); + //TSDebug(DEBUG_TAG, "Add request header [%.*s:%.*s]",name_len, name, value_len, value); + } + } + TSHandleMLocRelease(bufp, hdr_loc, field_loc); + } +} + +// TODO(oschaaf): this is not sustainable when we get more +// configuration options like this. +bool get_override_expiry(const StringPiece& host) { + TSMutexLock(config_mutex); + AtsHostConfig* hc = config->Find(host.data(), host.size()); + TSMutexUnlock(config_mutex); + return hc->override_expiry(); +} + +AtsRewriteOptions* get_host_options(const StringPiece& host) { + TSMutexLock(config_mutex); + AtsRewriteOptions* r = NULL; + AtsHostConfig* hc = config->Find(host.data(), host.size()); + if (hc->options() != NULL) { + // We return a clone here to avoid having to thing about + // configuration reloads and outstanding options + r = hc->options()->Clone(); + } + TSMutexUnlock(config_mutex); + return r; +} + +std::string get_remapped_host(TSHttpTxn txn) { + TSMBuffer server_req_buf; + TSMLoc server_req_loc; + std::string to_host; + if (TSHttpTxnServerReqGet(txn, &server_req_buf, &server_req_loc) == TS_SUCCESS + || TSHttpTxnCachedReqGet(txn, &server_req_buf, &server_req_loc) == TS_SUCCESS) { + to_host = get_header(server_req_buf, server_req_loc, "Host"); + TSHandleMLocRelease(server_req_buf, TS_NULL_MLOC, server_req_loc); + } else { + fprintf(stderr, "@@@@@@@ FAILED \n"); + } + return to_host; +} + +static void +ats_transform_init(TSCont contp, TransformCtx * ctx) +{ + //prepare the downstream for transforming + TSVConn downstream_conn; + TSMBuffer bufp; + TSMLoc hdr_loc; + TSMBuffer reqp; + TSMLoc req_hdr_loc; + ctx->state = transform_state_output; + + + // TODO: check cleanup flow + if (TSHttpTxnTransformRespGet(ctx->txn, &bufp, &hdr_loc) != TS_SUCCESS) { + TSError("Error TSHttpTxnTransformRespGet"); + return; + } + if (TSHttpTxnClientReqGet(ctx->txn, &reqp, &req_hdr_loc) != TS_SUCCESS) { + TSError("Error TSHttpTxnClientReqGet"); + return; + } + + AtsServerContext* server_context = ats_process_context->server_context(); + if (server_context->IsPagespeedResource(*ctx->gurl)) { + CHECK(false) << "PageSpeed resource should not get here!"; + } + + downstream_conn = TSTransformOutputVConnGet(contp); + ctx->downstream_buffer = TSIOBufferCreate(); + ctx->downstream_vio = TSVConnWrite(downstream_conn, contp, TSIOBufferReaderAlloc(ctx->downstream_buffer), INT64_MAX); + + // TODO(oschaaf): fix host/ip(?) + SystemRequestContext* system_request_context = + new SystemRequestContext(server_context->thread_system()->NewMutex(), + server_context->timer(), + 80, + "127.0.0.1"); + + ctx->base_fetch = new AtsBaseFetch(server_context, RequestContextPtr(system_request_context), + ctx->downstream_vio, ctx->downstream_buffer, false); + + + RewriteOptions* options = NULL; + RequestHeaders* request_headers = new RequestHeaders(); + ctx->base_fetch->SetRequestHeadersTakingOwnership(request_headers); + copy_request_headers_to_psol(reqp, req_hdr_loc, request_headers); + + TSHttpStatus status = TSHttpHdrStatusGet(bufp, hdr_loc); + // TODO(oschaaf): http version + ctx->base_fetch->response_headers()->set_status_code(status); + copy_response_headers_to_psol(bufp, hdr_loc, ctx->base_fetch->response_headers()); + ctx->base_fetch->response_headers()->ComputeCaching(); + const char* host = ctx->gurl->HostAndPort().as_string().c_str(); + //request_headers->Lookup1(HttpAttributes::kHost); + if (host != NULL && strlen(host) > 0) { + ctx->options = get_host_options(host); + } + bool ok = ps_determine_options(server_context, + ctx->options, + request_headers, + ctx->base_fetch->response_headers(), + &options, + ctx->gurl); + + // Take ownership of custom_options. + scoped_ptr<RewriteOptions> custom_options(options); + + if (!ok) { + TSError("Failure while determining request options for psol"); + options = server_context->global_options(); + } else { + // ps_determine_options modified url, removing any ModPagespeedFoo=Bar query + // parameters. Keep url_string in sync with url. + ctx->gurl->Spec().CopyToString(ctx->url_string); + } + + RewriteDriver* driver; + if (custom_options.get() == NULL) { + driver = server_context->NewRewriteDriver(ctx->base_fetch->request_context()); + } else { + driver = server_context->NewCustomRewriteDriver(custom_options.release(), ctx->base_fetch->request_context()); + } + + driver->SetUserAgent(ctx->user_agent->c_str()); + driver->SetRequestHeaders(*request_headers); + + bool page_callback_added = false; + scoped_ptr<ProxyFetchPropertyCallbackCollector> + property_callback( + ProxyFetchFactory::InitiatePropertyCacheLookup( + false /* is resource fetch?*/, + *ctx->gurl, + server_context, + options, + ctx->base_fetch, + false /* requires_blink_cohort (no longer unused) */, + &page_callback_added)); + + ctx->proxy_fetch = + ats_process_context->proxy_fetch_factory()->CreateNewProxyFetch( + *(ctx->url_string), ctx->base_fetch, driver, + property_callback.release(), + NULL /* original_content_fetch */); + + TSHandleMLocRelease(reqp, TS_NULL_MLOC, req_hdr_loc); + TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc); +} + +static void +ats_transform_one(TransformCtx * ctx, TSIOBufferReader upstream_reader, int amount) +{ + TSIOBufferBlock downstream_blkp; + const char *upstream_buffer; + int64_t upstream_length; + + while (amount > 0) { + downstream_blkp = TSIOBufferReaderStart(upstream_reader); + if (!downstream_blkp) { + TSError("couldn't get from IOBufferBlock"); + return; + } + + upstream_buffer = TSIOBufferBlockReadStart(downstream_blkp, upstream_reader, &upstream_length); + if (!upstream_buffer) { + TSError("couldn't get from TSIOBufferBlockReadStart"); + return; + } + + if (upstream_length > amount) { + upstream_length = amount; + } + + TSDebug("ats-speed", "transform!"); + // TODO(oschaaf): use at least the message handler from the server conrtext here? + if (ctx->inflater == NULL) { + ctx->proxy_fetch->Write(StringPiece((char*)upstream_buffer, upstream_length), ats_process_context->message_handler()); + } else { + char buf[net_instaweb::kStackBufferSize]; + + ctx->inflater->SetInput((char*)upstream_buffer, upstream_length); + + while (ctx->inflater->HasUnconsumedInput()) { + int num_inflated_bytes = ctx->inflater->InflateBytes( + buf, net_instaweb::kStackBufferSize); + if (num_inflated_bytes < 0) { + TSError("Corrupted inflation"); + } else if (num_inflated_bytes > 0) { + ctx->proxy_fetch->Write(StringPiece(buf, num_inflated_bytes), + ats_process_context->message_handler()); + } + } + } + //ctx->proxy_fetch->Flush(NULL); + TSIOBufferReaderConsume(upstream_reader, upstream_length); + amount -= upstream_length; + } + // TODO(oschaaf): get the output from the base fetch, and send it downstream. + // This would require proper locking around the base fetch buffer + // We could also have a look at directly writing to the traffic server buffers +} + + +static void +ats_transform_finish(TransformCtx * ctx) +{ + if (ctx->state == transform_state_output) { + ctx->state = transform_state_finished; + ctx->proxy_fetch->Done(true); + ctx->proxy_fetch = NULL; + } +} + +static void +ats_transform_do(TSCont contp) +{ + TSVIO upstream_vio; + TransformCtx *ctx; + int64_t upstream_todo; + int64_t upstream_avail; + int64_t downstream_bytes_written; + + ctx = (TransformCtx*)TSContDataGet(contp); + + if (ctx->state == transform_state_initialized) { + ats_transform_init(contp, ctx); + } + + upstream_vio = TSVConnWriteVIOGet(contp); + downstream_bytes_written = ctx->downstream_length; + + if (!TSVIOBufferGet(upstream_vio)) { + ats_transform_finish(ctx); + return; + } + + upstream_todo = TSVIONTodoGet(upstream_vio); + + if (upstream_todo > 0) { + upstream_avail = TSIOBufferReaderAvail(TSVIOReaderGet(upstream_vio)); + + if (upstream_todo > upstream_avail) { + upstream_todo = upstream_avail; + } + + if (upstream_todo > 0) { + ats_transform_one(ctx, TSVIOReaderGet(upstream_vio), upstream_todo); + TSVIONDoneSet(upstream_vio, TSVIONDoneGet(upstream_vio) + upstream_todo); + } + } + + if (TSVIONTodoGet(upstream_vio) > 0) { + if (upstream_todo > 0) { + if (ctx->downstream_length > downstream_bytes_written) { + TSVIOReenable(ctx->downstream_vio); + } + TSContCall(TSVIOContGet(upstream_vio), TS_EVENT_VCONN_WRITE_READY, upstream_vio); + } + } else { + ats_transform_finish(ctx); + TSContCall(TSVIOContGet(upstream_vio), TS_EVENT_VCONN_WRITE_COMPLETE, upstream_vio); + } +} + + +static int +ats_speed_transform(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */) +{ + if (TSVConnClosedGet(contp)) { + //ats_ctx_destroy((TransformCtx*)TSContDataGet(contp)); + TSContDestroy(contp); + return 0; + } else { + switch (event) { + case TS_EVENT_ERROR:{ + fprintf(stderr, "ats speed transform event: [%d] TS EVENT ERROR?!\n", event); + TSVIO upstream_vio = TSVConnWriteVIOGet(contp); + TSContCall(TSVIOContGet(upstream_vio), TS_EVENT_ERROR, upstream_vio); + } + break; + case TS_EVENT_VCONN_WRITE_COMPLETE: + TSVConnShutdown(TSTransformOutputVConnGet(contp), 0, 1); + break; + case TS_EVENT_VCONN_WRITE_READY: + ats_transform_do(contp); + break; + case TS_EVENT_IMMEDIATE: + ats_transform_do(contp); + break; + default: + DCHECK(false) << "unknown event: " << event; + ats_transform_do(contp); + break; + } + } + + return 0; +} + +static void +ats_speed_transform_add(TSHttpTxn txnp) +{ + TransformCtx* ctx = get_transaction_context(txnp); + CHECK(ctx); + if (ctx->transform_added) { // Happens with a stale cache hit + return; + } else { + ctx->transform_added = true; + } + + TSHttpTxnUntransformedRespCache(txnp, 1); + TSHttpTxnTransformedRespCache(txnp, 0); + + TSVConn connp; + + connp = TSTransformCreate(ats_speed_transform, txnp); + TSContDataSet(connp, ctx); + TSHttpTxnHookAdd(txnp, TS_HTTP_RESPONSE_TRANSFORM_HOOK, connp); +} + +// Returns true if a server intercept was set up +// Which means we should not attempt any further transformation +void +handle_read_request_header(TSHttpTxn txnp) { + TSMBuffer reqp = NULL; + TSMLoc hdr_loc = NULL; + char *url = NULL; + int url_length = -1; + + TransformCtx* ctx = ats_ctx_alloc(); + ctx->txn = txnp; + TSHttpTxnArgSet(txnp, TXN_INDEX_ARG, (void*) ctx); + TSHttpTxnArgSet(txnp, TXN_INDEX_OWNED_ARG, &TXN_INDEX_OWNED_ARG_SET); + + if (TSHttpTxnClientReqGet(txnp, &reqp, &hdr_loc) == TS_SUCCESS) { + url = TSHttpTxnEffectiveUrlStringGet(txnp, &url_length); + if (!url || url_length <= 0) { + DCHECK(false) << "Could not get url!"; + } else { + std::string s_url = std::string(url,url_length); + GoogleUrl gurl(s_url); + + ctx->url_string = new GoogleString(url, url_length); + ctx->gurl = new GoogleUrl(*(ctx->url_string)); + CHECK(ctx->gurl->IsWebValid()) << "Invalid URL!"; + const char * method; + int method_len; + method = TSHttpHdrMethodGet(reqp, hdr_loc, &method_len); + bool head_or_get = method == TS_HTTP_METHOD_GET || method == TS_HTTP_METHOD_HEAD; + ctx->request_method = method; + GoogleString user_agent = get_header(reqp, hdr_loc, "User-Agent"); + ctx->user_agent = new GoogleString(user_agent); + ctx->server_context = ats_process_context->server_context(); + if (user_agent.find(kModPagespeedSubrequestUserAgent) != user_agent.npos) { + ctx->mps_user_agent = true; + } + if (ats_process_context->server_context()->IsPagespeedResource(gurl)) { + if (head_or_get && !ctx->mps_user_agent) { + ctx->resource_request = true; + TSHttpTxnArgSet(txnp, TXN_INDEX_OWNED_ARG, &TXN_INDEX_OWNED_ARG_UNSET); + } + } else if (ctx->gurl->PathSansQuery() == "/pagespeed_message" + || ctx->gurl->PathSansQuery() == "/pagespeed_statistics" + || ctx->gurl->PathSansQuery() == "/pagespeed_global_statistics" + || ctx->gurl->PathSansQuery() == "/pagespeed_console" + || ctx->gurl->PathSansLeaf() == "/ats_speed_static/" + || ctx->gurl->PathSansQuery() == "/robots.txt" + ) { + ctx->resource_request = true; + TSHttpTxnArgSet(txnp, TXN_INDEX_OWNED_ARG, &TXN_INDEX_OWNED_ARG_UNSET); + } + else if (StringCaseEqual(gurl.PathSansQuery() ,"/ats_speed_beacon")) { + ctx->beacon_request = true; + TSHttpTxnArgSet(txnp, TXN_INDEX_OWNED_ARG, &TXN_INDEX_OWNED_ARG_UNSET); + hook_beacon_intercept(txnp); + } + TSfree((void*)url); + } + TSHandleMLocRelease(reqp, TS_NULL_MLOC, hdr_loc); + } else { + DCHECK(false) << "Could not get client request header\n"; + } + + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); +} + +bool +cache_hit(TSHttpTxn txnp) { + int obj_status; + if (TSHttpTxnCacheLookupStatusGet(txnp, &obj_status) == TS_ERROR) { + // TODO(oschaaf): log warning + return false; + } + return obj_status == TS_CACHE_LOOKUP_HIT_FRESH; +} + +static int +transform_plugin(TSCont contp, TSEvent event, void *edata) +{ + TSHttpTxn txn = (TSHttpTxn) edata; + + CHECK(event == TS_EVENT_HTTP_READ_RESPONSE_HDR || event == TS_EVENT_HTTP_READ_CACHE_HDR + || event == TS_EVENT_HTTP_SEND_REQUEST_HDR || event == TS_EVENT_HTTP_READ_REQUEST_HDR + || event == TS_EVENT_HTTP_TXN_CLOSE || event == TS_EVENT_HTTP_SEND_RESPONSE_HDR) + << "Invalid transform event"; + + if (event != TS_EVENT_HTTP_READ_REQUEST_HDR) { + // Bail if an intercept is running + bool is_owned = TSHttpTxnArgGet(txn, TXN_INDEX_OWNED_ARG) == &TXN_INDEX_OWNED_ARG_SET; + if (!is_owned) { + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + return 0; + } + } + + if (event == TS_EVENT_HTTP_SEND_RESPONSE_HDR) { + handle_send_response_headers(txn); + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + return 0; + } if (event == TS_EVENT_HTTP_TXN_CLOSE) { + TransformCtx* ctx = get_transaction_context(txn); + //if (ctx != NULL && !ctx->resource_request && !ctx->beacon_request && !ctx->html_rewrite) { + // For intercepted requests like beacons and resource requests, we don't own the + // ctx here - the interceptor does. + + if (ctx != NULL) { + bool is_owned = TSHttpTxnArgGet(txn, TXN_INDEX_OWNED_ARG) == &TXN_INDEX_OWNED_ARG_SET; + if (is_owned) { + ats_ctx_destroy(ctx); + } + } + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + return 0; + } if (event == TS_EVENT_HTTP_READ_REQUEST_HDR) { + handle_read_request_header(txn); + return 0; + } else if (event == TS_EVENT_HTTP_SEND_REQUEST_HDR) { + TSMBuffer request_header_buf = NULL; + TSMLoc request_header_loc = NULL; + + if (TSHttpTxnServerReqGet(txn, &request_header_buf, &request_header_loc) == TS_SUCCESS) { + hide_accept_encoding(request_header_buf, request_header_loc, "@xxAccept-Encoding"); + // Turn off pagespeed optimization at the origin + set_header(request_header_buf, request_header_loc, "PageSpeed", "off"); + TSHandleMLocRelease(request_header_buf, TS_NULL_MLOC, request_header_loc); + } else { + CHECK(false) << "Could not find server request header"; + } + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + return 0; + } else if (event == TS_EVENT_HTTP_READ_RESPONSE_HDR) { + TSMBuffer request_header_buf = NULL; + TSMLoc request_header_loc = NULL; + + if (TSHttpTxnServerReqGet(txn, &request_header_buf, &request_header_loc) == TS_SUCCESS) { + restore_accept_encoding(request_header_buf, request_header_loc, "@xxAccept-Encoding"); + TSHandleMLocRelease(request_header_buf, TS_NULL_MLOC, request_header_loc); + } else { + CHECK(false) << "Could not find server request header"; + } + } + + CHECK(event == TS_EVENT_HTTP_READ_RESPONSE_HDR || event == TS_EVENT_HTTP_READ_CACHE_HDR); + + TransformCtx* ctx = get_transaction_context(txn); + if (ctx == NULL) { + // TODO(oschaaf): document how and when this happens. + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + return 0; + } + std::string* to_host = new std::string(); + to_host->append(get_remapped_host(ctx->txn)); + ctx->to_host = to_host; + TSMBuffer response_header_buf = NULL; + TSMLoc response_header_loc = NULL; + + // TODO(oschaaf): from configuration! + bool override_expiry = false; + + const char* host = ctx->gurl->HostAndPort().as_string().c_str(); + //request_headers->Lookup1(HttpAttributes::kHost); + if (host != NULL && strlen(host) > 0) { + override_expiry = get_override_expiry(host); + } + + + if (ctx->mps_user_agent && override_expiry) { + if (TSHttpTxnServerRespGet(txn, &response_header_buf, &response_header_loc) == TS_SUCCESS) { + // TODO => set cacheable. + unset_header(response_header_buf, response_header_loc, "Cache-Control"); + unset_header(response_header_buf, response_header_loc, "Expires"); + unset_header(response_header_buf, response_header_loc, "Age"); + set_header(response_header_buf, response_header_loc, "Cache-Control", "public, max-age=3600"); + TSHandleMLocRelease(response_header_buf, TS_NULL_MLOC, response_header_loc); + } + } + bool ok = !(ctx->resource_request || ctx->beacon_request || ctx->mps_user_agent); + if (!ok) { + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + return 0; + } + + bool have_response_header = false; + + if (TSHttpTxnServerRespGet(txn, &response_header_buf, &response_header_loc) == TS_SUCCESS) { + have_response_header = true; + if (override_expiry) { + unset_header(response_header_buf, response_header_loc, "Cache-Control"); + unset_header(response_header_buf, response_header_loc, "Expires"); + unset_header(response_header_buf, response_header_loc, "Age"); + set_header(response_header_buf, response_header_loc, "Cache-Control", "public, max-age=3600"); + } + } + else if (TSHttpTxnCachedRespGet(txn, &response_header_buf, &response_header_loc) == TS_SUCCESS) { + have_response_header = true; + } + if (!have_response_header) { + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + return 0; + } + + if (ok) { + if (ctx->request_method != TS_HTTP_METHOD_GET && ctx->request_method != TS_HTTP_METHOD_HEAD + && ctx->request_method != TS_HTTP_METHOD_POST) { + ok = false; + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + return 0; + } + } + + TSHttpStatus status = TSHttpHdrStatusGet(response_header_buf, response_header_loc); + if (ok) { + if (!(status == TS_HTTP_STATUS_OK || status == TS_HTTP_STATUS_NOT_FOUND)) { + ok = false; + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + return 0; + } + } + if (ok) { + StringPiece s_content_type = get_header(response_header_buf, response_header_loc, "Content-Type"); + const net_instaweb::ContentType* content_type = + net_instaweb::MimeTypeToContentType(s_content_type); + + if ((content_type == NULL || !content_type->IsHtmlLike())) { + ok = false; + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + return 0; + } + } + + if (ok) { + StringPiece content_encoding = get_header(response_header_buf, response_header_loc, "Content-Encoding"); + net_instaweb::GzipInflater::InflateType inflate_type; + bool is_encoded = false; + + if (StringCaseEqual(content_encoding, "deflate")) { + is_encoded = true; + inflate_type = GzipInflater::kDeflate; + } else if (StringCaseEqual(content_encoding, "gzip")) { + is_encoded = true; + inflate_type = GzipInflater::kGzip; + } + + if (is_encoded) { + ctx->inflater = new GzipInflater(inflate_type); + ctx->inflater->Init(); + } + TSDebug(DEBUG_TAG, "Will optimize [%s]", ctx->url_string->c_str()); + ctx->html_rewrite = true; + set_header(response_header_buf,response_header_loc,"@gzip_nocache","0"); + ats_speed_transform_add(txn); + } + + TSHandleMLocRelease(response_header_buf, TS_NULL_MLOC, response_header_loc); + TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE); + + return 0; +} + +bool RegisterPlugin() { + TSPluginRegistrationInfo info; + + info.plugin_name = (char *)"ats_speed"; + info.vendor_name = (char *)"We-Amp B.V."; + info.support_email = (char *)"[email protected]"; + + if (TSPluginRegister(TS_SDK_VERSION_3_0, &info) != TS_SUCCESS) { + TSError("Failed to register ATSSpeed"); + return false; + } + + return true; +} + +void cleanup_process() { + delete ats_process_context; + AtsRewriteDriverFactory::Terminate(); + AtsRewriteOptions::Terminate(); +} + +static void +process_configuration() +{ + AtsConfig* new_config = new AtsConfig((AtsThreadSystem*)ats_process_context->server_context()->thread_system()); + DIR *dir; + struct dirent *ent; + + if ((dir = opendir ("/usr/local/etc/trafficserver/psol/")) != NULL) { + while ((ent = readdir (dir)) != NULL) { + size_t len = strlen(ent->d_name); + if (len <= 0) continue; + if (ent->d_name[0] == '.') continue; + if (ent->d_name[len-1] == '~') continue; + if (ent->d_name[0] == '#') continue; + GoogleString s("/usr/local/etc/trafficserver/psol/"); + s.append(ent->d_name); + fprintf (stderr, "parse [%s]\n", s.c_str()); + if (!new_config->Parse(s.c_str())) { + TSError("Error parsing %s", s.c_str()); + } + } + closedir (dir); + } + + AtsConfig* old_config; + TSMutexLock(config_mutex); + fprintf(stderr, "Update configuration\n"); + old_config = config; + config = new_config; + TSMutexUnlock(config_mutex); + if (old_config != NULL) { + delete old_config; + } +} + +static void * +config_notification_callback(void *data) +{ + int BUF_MAX = 1024 * (sizeof(struct inotify_event) + 16); + char buf[BUF_MAX]; + int fd,wd; + + fd = inotify_init(); + + if (fd < 0) { + perror( "inotify_init" ); + CHECK(false) << "Failed to initialize inotify"; + } + + wd = inotify_add_watch(fd, "/usr/local/etc/trafficserver/psol/", IN_MODIFY | IN_CREATE | IN_DELETE); + + while (1) { + int len = read(fd, buf, BUF_MAX); + int i = 0; + bool do_update = false; + while ( i < len ) { + struct inotify_event *event = ( struct inotify_event * ) &buf[ i ]; + if ( event->len ) { + if (!(event->mask & IN_ISDIR)) { + const char* name = event->name; + size_t name_len = strlen(event->name); + if (name_len > 0 && name[0] != '.' && name[0] != '#' && name[name_len-1] != '~' ) { + do_update = true; + } + } + } + i += ( sizeof (struct inotify_event) ) + event->len; + } + if (do_update) { + process_configuration(); + } + } + + inotify_rm_watch( fd, wd ); + close( fd ); + + return NULL; +} + + +void TSPluginInit(int argc, const char *argv[]) { + if (RegisterPlugin() == true) { + if (TSHttpArgIndexReserve("ats_speed", "Stores the transaction context", &TXN_INDEX_ARG) != TS_SUCCESS) { + CHECK(false) << "failed to reserve an argument index"; + } + if (TSHttpArgIndexReserve("ats_speed", "Stores the transaction context", &TXN_INDEX_OWNED_ARG) != TS_SUCCESS) { + CHECK(false) << "failed to reserve an argument index"; + } + + AtsRewriteOptions::Initialize(); + AtsRewriteDriverFactory::Initialize(); + net_instaweb::log_message_handler::Install(); + atexit(cleanup_process); + ats_process_context = new AtsProcessContext(); + process_configuration(); + TSCont transform_contp = TSContCreate(transform_plugin, NULL); + TSHttpHookAdd(TS_HTTP_READ_RESPONSE_HDR_HOOK, transform_contp); + TSHttpHookAdd(TS_HTTP_READ_CACHE_HDR_HOOK, transform_contp); + TSHttpHookAdd(TS_HTTP_SEND_REQUEST_HDR_HOOK, transform_contp); + TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, transform_contp); + TSHttpHookAdd(TS_HTTP_TXN_CLOSE_HOOK, transform_contp); + TSHttpHookAdd(TS_HTTP_SEND_RESPONSE_HDR_HOOK, transform_contp); + + setup_resource_intercept(); + CHECK(TSThreadCreate(config_notification_callback, NULL)) << ""; + ats_process_context->message_handler()->Message( + kInfo, "TSPluginInit OK"); + } +} http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ats_speed.h ---------------------------------------------------------------------- diff --git a/plugins/experimental/ats_speed/ats_speed.h b/plugins/experimental/ats_speed/ats_speed.h new file mode 100644 index 0000000..99b9bd4 --- /dev/null +++ b/plugins/experimental/ats_speed/ats_speed.h @@ -0,0 +1,94 @@ +// Copyright 2013 We-Amp B.V. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: [email protected] (Otto van der Schaaf) +#ifndef ATS_SPEED_H_ +#define ATS_SPEED_H_ + +#include <string> + +#include <ts/ts.h> + +#include "net/instaweb/util/public/google_url.h" +#include "net/instaweb/util/public/string.h" +#include "net/instaweb/util/public/string_util.h" + +namespace net_instaweb { + +class AtsBaseFetch; +class AtsRewriteOptions; +class AtsServerContext; +class GzipInflater; +class ProxyFetch; +class RewriteOptions; +class RequestHeaders; +class ResponseHeaders; +class ServerContext; + +} // namespace net_instaweb + +enum transform_state { + transform_state_initialized, + transform_state_output, + transform_state_finished +}; + +typedef struct +{ + TSHttpTxn txn; + TSVIO downstream_vio; + TSIOBuffer downstream_buffer; + int64_t downstream_length; + enum transform_state state; + + net_instaweb::AtsBaseFetch* base_fetch; + net_instaweb::ProxyFetch* proxy_fetch; + net_instaweb::GzipInflater* inflater; + + bool write_pending; + bool fetch_done; + GoogleString* url_string; + bool beacon_request; + bool resource_request; + bool mps_user_agent; + bool transform_added; + net_instaweb::GoogleUrl* gurl; + net_instaweb::AtsServerContext* server_context; + GoogleString* user_agent; + bool html_rewrite; + const char* request_method; + int alive; + net_instaweb::AtsRewriteOptions* options; + // TODO: Use GoogleString* + std::string* to_host; +} TransformCtx; + +TransformCtx* get_transaction_context(TSHttpTxn txnp); +void ats_ctx_destroy(TransformCtx * ctx); +bool cache_hit(TSHttpTxn txnp); + +bool ps_determine_options(net_instaweb::ServerContext* server_context, + // Directory-specific options, usually null. They've already been rebased off + // of the global options as part of the configuration process. + net_instaweb::RewriteOptions* directory_options, + net_instaweb::RequestHeaders* request_headers, + net_instaweb::ResponseHeaders* response_headers, + net_instaweb::RewriteOptions** options, + net_instaweb::GoogleUrl* url); + +void copy_request_headers_to_psol(TSMBuffer bufp, TSMLoc hdr_loc, net_instaweb::RequestHeaders* psol_headers); +// You will own options returned by this: +net_instaweb::AtsRewriteOptions* get_host_options(const StringPiece& host); + +#endif /* ATS_SPEED_H_ */ http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ats_thread_system.h ---------------------------------------------------------------------- diff --git a/plugins/experimental/ats_speed/ats_thread_system.h b/plugins/experimental/ats_speed/ats_thread_system.h new file mode 100644 index 0000000..ebbf5e9 --- /dev/null +++ b/plugins/experimental/ats_speed/ats_thread_system.h @@ -0,0 +1,43 @@ +// Copyright 2013 We-Amp B.V. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: [email protected] (Otto van der Schaaf) + +#ifndef ATS_THREAD_SYSTEM_H_ +#define ATS_THREAD_SYSTEM_H_ + +#include <pthread.h> + +#include <ts/ts.h> +#include "net/instaweb/system/public/system_thread_system.h" +#include "net/instaweb/util/public/thread.h" +#include "net/instaweb/util/public/thread_system.h" +#include "net/instaweb/util/public/pthread_rw_lock.h" +#include "net/instaweb/util/public/condvar.h" + +namespace net_instaweb { + +class AtsThreadSystem : public net_instaweb::SystemThreadSystem { + public: + virtual void BeforeThreadRunHook() { + TSThreadInit(); + } + + virtual ~AtsThreadSystem() { } +}; + +} // net_instaweb + + +#endif // ATS_THREAD_SYSTEM_H_ http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/ethread.patch ---------------------------------------------------------------------- diff --git a/plugins/experimental/ats_speed/ethread.patch b/plugins/experimental/ats_speed/ethread.patch new file mode 100644 index 0000000..3cc5e1c --- /dev/null +++ b/plugins/experimental/ats_speed/ethread.patch @@ -0,0 +1,13 @@ +diff --git a/iocore/eventsystem/I_EThread.h b/iocore/eventsystem/I_EThread.h +index e5f1c56..3d2f9c7 100644 +--- a/iocore/eventsystem/I_EThread.h ++++ b/iocore/eventsystem/I_EThread.h +@@ -297,7 +297,7 @@ public: + virtual ~EThread(); + + Event *schedule_spawn(Continuation *cont); +- Event *schedule(Event *e, bool fast_signal = false); ++ Event *schedule(Event *e, bool fast_signal = true); + + /** Block of memory to allocate thread specific data e.g. stat system arrays. */ + char thread_private[PER_THREAD_DATA]; http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/Makefile ---------------------------------------------------------------------- diff --git a/plugins/experimental/ats_speed/gzip/Makefile b/plugins/experimental/ats_speed/gzip/Makefile new file mode 100644 index 0000000..0b8d12e --- /dev/null +++ b/plugins/experimental/ats_speed/gzip/Makefile @@ -0,0 +1,9 @@ +TSXS?=tsxs + +all: + $(TSXS) -o gzip.so -v -C *.cc +install: + $(TSXS) -v -i -o gzip.so +clean: + rm -f *.lo *.so +
