ats_pagespeed: rename ats_speed -> ats_pagespeed Fixes TS-3005
Project: http://git-wip-us.apache.org/repos/asf/trafficserver/repo Commit: http://git-wip-us.apache.org/repos/asf/trafficserver/commit/e8b899b0 Tree: http://git-wip-us.apache.org/repos/asf/trafficserver/tree/e8b899b0 Diff: http://git-wip-us.apache.org/repos/asf/trafficserver/diff/e8b899b0 Branch: refs/heads/5.1.x Commit: e8b899b0fb847ca81eed290fa9c34f95ca6255eb Parents: afa93fa Author: Otto van der Schaaf <[email protected]> Authored: Wed Aug 20 22:52:56 2014 +0200 Committer: Alan M. Carroll <[email protected]> Committed: Thu Aug 21 09:53:32 2014 -0500 ---------------------------------------------------------------------- plugins/experimental/ats_pagespeed/.gitignore | 15 + plugins/experimental/ats_pagespeed/Makefile | 83 ++ .../ats_pagespeed/Makefile.psol_source | 49 + plugins/experimental/ats_pagespeed/README.md | 52 + .../ats_pagespeed/ats_base_fetch.cc | 142 +++ .../experimental/ats_pagespeed/ats_base_fetch.h | 88 ++ .../ats_pagespeed/ats_beacon_intercept.cc | 364 ++++++ .../ats_pagespeed/ats_beacon_intercept.h | 31 + .../experimental/ats_pagespeed/ats_config.cc | 204 ++++ plugins/experimental/ats_pagespeed/ats_config.h | 90 ++ .../ats_pagespeed/ats_header_utils.cc | 96 ++ .../ats_pagespeed/ats_header_utils.h | 41 + .../ats_pagespeed/ats_log_message_handler.cc | 101 ++ .../ats_pagespeed/ats_log_message_handler.h | 36 + .../ats_pagespeed/ats_message_handler.cc | 114 ++ .../ats_pagespeed/ats_message_handler.h | 75 ++ .../experimental/ats_pagespeed/ats_pagespeed.cc | 1093 ++++++++++++++++++ .../experimental/ats_pagespeed/ats_pagespeed.h | 102 ++ .../ats_pagespeed/ats_process_context.cc | 86 ++ .../ats_pagespeed/ats_process_context.h | 58 + .../ats_pagespeed/ats_resource_intercept.cc | 363 ++++++ .../ats_pagespeed/ats_resource_intercept.h | 29 + .../ats_pagespeed/ats_rewrite_driver_factory.cc | 196 ++++ .../ats_pagespeed/ats_rewrite_driver_factory.h | 113 ++ .../ats_pagespeed/ats_rewrite_options.cc | 263 +++++ .../ats_pagespeed/ats_rewrite_options.h | 103 ++ .../ats_pagespeed/ats_server_context.cc | 46 + .../ats_pagespeed/ats_server_context.h | 56 + .../ats_pagespeed/ats_thread_system.h | 50 + .../experimental/ats_pagespeed/gzip/Makefile | 24 + plugins/experimental/ats_pagespeed/gzip/README | 4 + .../ats_pagespeed/gzip/configuration.cc | 264 +++++ .../ats_pagespeed/gzip/configuration.h | 84 ++ .../ats_pagespeed/gzip/debug_macros.h | 59 + plugins/experimental/ats_pagespeed/gzip/gzip.cc | 826 +++++++++++++ .../experimental/ats_pagespeed/gzip/gzip.config | 6 + plugins/experimental/ats_pagespeed/gzip/misc.cc | 197 ++++ plugins/experimental/ats_pagespeed/gzip/misc.h | 84 ++ .../ats_pagespeed/scripts/prepare_psol.sh | 93 ++ plugins/experimental/ats_speed/.gitignore | 15 - plugins/experimental/ats_speed/Makefile | 83 -- .../experimental/ats_speed/Makefile.psol_source | 49 - plugins/experimental/ats_speed/README.md | 52 - .../experimental/ats_speed/ats_base_fetch.cc | 142 --- plugins/experimental/ats_speed/ats_base_fetch.h | 88 -- .../ats_speed/ats_beacon_intercept.cc | 364 ------ .../ats_speed/ats_beacon_intercept.h | 31 - plugins/experimental/ats_speed/ats_config.cc | 204 ---- plugins/experimental/ats_speed/ats_config.h | 90 -- .../experimental/ats_speed/ats_header_utils.cc | 96 -- .../experimental/ats_speed/ats_header_utils.h | 41 - .../ats_speed/ats_log_message_handler.cc | 101 -- .../ats_speed/ats_log_message_handler.h | 36 - .../ats_speed/ats_message_handler.cc | 114 -- .../ats_speed/ats_message_handler.h | 75 -- .../ats_speed/ats_process_context.cc | 86 -- .../ats_speed/ats_process_context.h | 58 - .../ats_speed/ats_resource_intercept.cc | 363 ------ .../ats_speed/ats_resource_intercept.h | 29 - .../ats_speed/ats_rewrite_driver_factory.cc | 196 ---- .../ats_speed/ats_rewrite_driver_factory.h | 113 -- .../ats_speed/ats_rewrite_options.cc | 263 ----- .../ats_speed/ats_rewrite_options.h | 103 -- .../ats_speed/ats_server_context.cc | 46 - .../experimental/ats_speed/ats_server_context.h | 56 - plugins/experimental/ats_speed/ats_speed.cc | 1093 ------------------ plugins/experimental/ats_speed/ats_speed.h | 102 -- .../experimental/ats_speed/ats_thread_system.h | 50 - plugins/experimental/ats_speed/gzip/Makefile | 24 - plugins/experimental/ats_speed/gzip/README | 4 - .../ats_speed/gzip/configuration.cc | 264 ----- .../experimental/ats_speed/gzip/configuration.h | 84 -- .../experimental/ats_speed/gzip/debug_macros.h | 59 - plugins/experimental/ats_speed/gzip/gzip.cc | 826 ------------- plugins/experimental/ats_speed/gzip/gzip.config | 6 - plugins/experimental/ats_speed/gzip/misc.cc | 197 ---- plugins/experimental/ats_speed/gzip/misc.h | 84 -- .../ats_speed/scripts/prepare_psol.sh | 93 -- 78 files changed, 5780 insertions(+), 5780 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/trafficserver/blob/e8b899b0/plugins/experimental/ats_pagespeed/.gitignore ---------------------------------------------------------------------- diff --git a/plugins/experimental/ats_pagespeed/.gitignore b/plugins/experimental/ats_pagespeed/.gitignore new file mode 100644 index 0000000..a12b1d2 --- /dev/null +++ b/plugins/experimental/ats_pagespeed/.gitignore @@ -0,0 +1,15 @@ +# Compiled Object files +*.slo +*.lo +*.o + +# Compiled Dynamic libraries +*.so +*.dylib + +# Compiled Static libraries +*.lai +*.la +*.a +*.gz +*~ \ No newline at end of file http://git-wip-us.apache.org/repos/asf/trafficserver/blob/e8b899b0/plugins/experimental/ats_pagespeed/Makefile ---------------------------------------------------------------------- diff --git a/plugins/experimental/ats_pagespeed/Makefile b/plugins/experimental/ats_pagespeed/Makefile new file mode 100644 index 0000000..9177b44 --- /dev/null +++ b/plugins/experimental/ats_pagespeed/Makefile @@ -0,0 +1,83 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SHELL := /bin/bash +TSXS?=tsxs +# Specify BUILDTYPE=Debug for a debug build +BUILDTYPE?=Release +MOD_PAGESPEED_DIR=$(shell pwd)/psol/include/ +PAGESPEED_OUT=$(shell pwd)/psol/lib/$(BUILDTYPE)/linux/x64/ + + +os_name=unknown_os +arch_name=ia32 +uname_os=$(shell uname) +uname_arch=$(shell uname -m) + +ifeq ($(uname_os),Linux) + os_name=linux +endif + +ifeq ($(uname_arch), x86_64) + arch_name=x64 +endif +ifeq ($(uname_arch), amd64) + arch_name=x64 +endif + +INC =-I$(MOD_PAGESPEED_DIR)\ + -I$(MOD_PAGESPEED_DIR)third_party/chromium/src/\ + -I$(MOD_PAGESPEED_DIR)third_party/google-sparsehash/src\ + -I$(MOD_PAGESPEED_DIR)third_party/google-sparsehash/gen/arch/$(os_name)/$(arch_name)/include\ + -I$(MOD_PAGESPEED_DIR)third_party/protobuf/src\ + -I$(MOD_PAGESPEED_DIR)third_party/re2/src\ + -I$(MOD_PAGESPEED_DIR)third_party/out/$(BUILDTYPE)/obj/gen\ + -I$(MOD_PAGESPEED_DIR)third_party/apr/src/include/\ + -I$(MOD_PAGESPEED_DIR)third_party/aprutil/src/include/\ + -I$(MOD_PAGESPEED_DIR)third_party/apr/gen/arch/$(os_name)/$(arch_name)/include/\ + -I$(MOD_PAGESPEED_DIR)third_party/aprutil/gen/arch/$(os_name)/$(arch_name)/include/\ + -I$(MOD_PAGESPEED_DIR)out/$(BUILDTYPE)/obj/gen\ + -I$(MOD_PAGESPEED_DIR)out/$(BUILDTYPE)/obj/gen/protoc_out/instaweb + +PSOL_LIBS = $(PAGESPEED_OUT)pagespeed_automatic.a +#PSOL_LIBS = $(PAGESPEED_OUT)pagespeed_automatic.a $(PAGESPEED_OUT)libserf.a $(PAGESPEED_OUT)libaprutil.a $(PAGESPEED_OUT)libapr.a + +%.so: psol %.cc +# https://github.com/pagespeed/ngx_pagespeed/issues/433: it would be nice to have -Wall -Werror, only suppressing when needed. + g++ $(INC) -shared -o ats_pagespeed.so -g -pipe -O3 -fpic *.cc -lstdc++ -lstdc++ -lpthread $(PSOL_LIBS) -lrt + +all: psol gzip/gzip.so ats_pagespeed.so + +1.8.31.4.tar.gz: + wget --no-check-certificate https://dl.google.com/dl/page-speed/psol/1.8.31.4.tar.gz + +psol/: 1.8.31.4.tar.gz + tar -xzvf 1.8.31.4.tar.gz + +gzip/gzip.so: + cd gzip && make + +install: all + $(TSXS) -i -o ats_pagespeed.so + cd gzip && make install + +cleanpsol: + rm -rf psol/ + rm *.gz + +clean: + rm -f *.lo *.so *.o + rm -f gzip/*.lo gzip/*.so gzip/*.o http://git-wip-us.apache.org/repos/asf/trafficserver/blob/e8b899b0/plugins/experimental/ats_pagespeed/Makefile.psol_source ---------------------------------------------------------------------- diff --git a/plugins/experimental/ats_pagespeed/Makefile.psol_source b/plugins/experimental/ats_pagespeed/Makefile.psol_source new file mode 100755 index 0000000..f4c35723 --- /dev/null +++ b/plugins/experimental/ats_pagespeed/Makefile.psol_source @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +TSXS?=tsxs +BUILDTYPE=Release +MOD_PAGESPEED_DIR=$(HOME)/code/google/mod_pagespeed/src/ +PAGESPEED_OUT=$(MOD_PAGESPEED_DIR)out/$(BUILDTYPE)/ + +INC =-I$(MOD_PAGESPEED_DIR)\ + -I$(MOD_PAGESPEED_DIR)third_party/chromium/src/\ + -I$(MOD_PAGESPEED_DIR)third_party/google-sparsehash/src\ + -I$(MOD_PAGESPEED_DIR)third_party/google-sparsehash/gen/arch/linux/x64/include\ + -I$(MOD_PAGESPEED_DIR)third_party/protobuf/src\ + -I$(MOD_PAGESPEED_DIR)third_party/re2/src\ + -I$(MOD_PAGESPEED_DIR)third_party/out/$(BUILDTYPE)/obj/gen\ + -I$(MOD_PAGESPEED_DIR)third_party/apr/src/include/\ + -I$(MOD_PAGESPEED_DIR)third_party/aprutil/src/include/\ + -I$(MOD_PAGESPEED_DIR)third_party/apr/gen/arch/linux/x64/include/\ + -I$(MOD_PAGESPEED_DIR)third_party/aprutil/gen/arch/linux/x64/include/\ + -I$(PAGESPEED_OUT)obj/gen/\ + -I$(PAGESPEED_OUT)obj/gen/protoc_out/instaweb/ + +PSOL_LIBS = $(MOD_PAGESPEED_DIR)net/instaweb/automatic/pagespeed_automatic.a + +%.so: %.cc + g++ $(INC) -shared -o ats_pagespeed.so -g -pipe -Wall -Werror -O3 -fpic *.cc -lstdc++ -lpthread -lrt $(PSOL_LIBS) + +all: gzip/gzip.so ats_pagespeed.so + +install: all + $(TSXS) -i -o ats_pagespeed.so + cp gzip/gzip.so ./ + $(TSXS) -i -o zip.so + +clean: + rm -f *.lo *.so *.o http://git-wip-us.apache.org/repos/asf/trafficserver/blob/e8b899b0/plugins/experimental/ats_pagespeed/README.md ---------------------------------------------------------------------- diff --git a/plugins/experimental/ats_pagespeed/README.md b/plugins/experimental/ats_pagespeed/README.md new file mode 100644 index 0000000..6c4ff15 --- /dev/null +++ b/plugins/experimental/ats_pagespeed/README.md @@ -0,0 +1,52 @@ + + +Apache Traffic Server web content optimization plugin powered by Google PageSpeed + +http://www.atsspeed.com/ + +To build, a simple 'make' should work. Use 'sudo make install' to install. +Optionally, patching ATS with ethread.patch helps with eliminating latency that +sometimes gets induced when synchronising ATS's and PSOL's thread pools. + +After that, update ATS's plugin.config with: +``` +ats_pagespeed.so +gzip.so /usr/local/etc/trafficserver/gzip.config +```` +gzip.so also is build with ats_pagespeed, as it currently is a slightly +modified version of the official one from the ATS repository. + +There are some hard-coded things in the plugin, these directories should exist: +- /tmp/ps_log/ to exist +- /tmp/ats_ps/ to exist + +Configuration files go into `/usr/local/etc/trafficserver/psol.` +That folder is monitored, and changes to files in there are picked +up immediately. A sample configuration: + +``` +# [host] +[192.168.185.185] +# Force traffic server to cache all origin responses +override_expiry +pagespeed FlushHtml on +pagespeed RewriteLevel CoreFilters +pagespeed EnableFilters rewrite_domains,trim_urls +pagespeed MapRewriteDomain http://192.168.185.185 http://www.foo.com +pagespeed MapOriginDomain http://192.168.185.185 http://www.foo.com +pagespeed EnableFilters prioritize_critical_css,move_css_to_head,move_css_above_scripts +pagespeed EnableFilters fallback_rewrite_css_urls,insert_img_dimensions,lazyload_images,local_storage_cache +pagespeed EnableFilters prioritize_critical_css,rewrite_css +pagespeed EnableFilters combine_javascript,combine_css +``` + +It also expects this in records.config from ATS to function: +`CONFIG proxy.config.url_remap.pristine_host_hdr INT 0` + +You can view debug output of the plugin using `traffic_server -T ".*speed.*"` + +The current state compiles against PSOL 1.7.30.4-beta. +Please note the this plugin will generate asserts when build against +the debug version of mps (option->Merge from a different thread). + http://git-wip-us.apache.org/repos/asf/trafficserver/blob/e8b899b0/plugins/experimental/ats_pagespeed/ats_base_fetch.cc ---------------------------------------------------------------------- diff --git a/plugins/experimental/ats_pagespeed/ats_base_fetch.cc b/plugins/experimental/ats_pagespeed/ats_base_fetch.cc new file mode 100644 index 0000000..769e79b --- /dev/null +++ b/plugins/experimental/ats_pagespeed/ats_base_fetch.cc @@ -0,0 +1,142 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "ats_base_fetch.h" + +#include <ts/ts.h> + +#include "ats_server_context.h" + +#include "net/instaweb/util/public/string_util.h" +#include "net/instaweb/util/public/string_writer.h" +#include "net/instaweb/util/public/google_message_handler.h" + + +using namespace net_instaweb; + +// TODO(oschaaf): rename is_resource_fetch -> write_raw_response_headers +AtsBaseFetch::AtsBaseFetch(AtsServerContext* server_context, + const net_instaweb::RequestContextPtr& request_ctx, + TSVIO downstream_vio, TSIOBuffer downstream_buffer, bool is_resource_fetch) : + AsyncFetch(request_ctx), + server_context_(server_context), + done_called_(false), + last_buf_sent_(false), + references_(2), + downstream_vio_(downstream_vio), + downstream_buffer_(downstream_buffer), + is_resource_fetch_(is_resource_fetch), + downstream_length_(0), + txn_mutex_(TSVIOMutexGet(downstream_vio)) { + buffer_.reserve(1024 * 32); +} + +AtsBaseFetch::~AtsBaseFetch() { + CHECK(references_ == 0); +} + +// Should be called from the event loop, +// and thus with the txn mutex held by ATS +void AtsBaseFetch::Release() { + DecrefAndDeleteIfUnreferenced(); +} + +void AtsBaseFetch::Lock(){ + TSMutexLock(txn_mutex_); +} + +void AtsBaseFetch::Unlock() { + TSMutexUnlock(txn_mutex_); +} + +bool AtsBaseFetch::HandleWrite(const StringPiece& sp, net_instaweb::MessageHandler* handler) { + ForwardData(sp, false, false); + return true; +} + +bool AtsBaseFetch::HandleFlush( net_instaweb::MessageHandler* handler ) { + ForwardData("", true, false); + return true; +} + +void AtsBaseFetch::HandleHeadersComplete() { + // oschaaf: ATS will currently send its response headers + // earlier than this will fire. So this has become a no-op. + // This implies that we can't support convert_meta_tags + TSDebug("ats-speed", "HeadersComplete()!"); + // For resource fetches, we need to output the headers in raw HTTP format. + if (is_resource_fetch_) { + GoogleMessageHandler mh; + GoogleString s; + StringWriter string_writer(&s); + response_headers()->Add("Connection", "Close"); + response_headers()->WriteAsHttp(&string_writer, &mh); + ForwardData(StringPiece(s.data(),s.size()), true, false); + } +} + +void AtsBaseFetch::ForwardData(const StringPiece& sp, bool reenable, bool last) { + TSIOBufferBlock downstream_blkp; + char *downstream_buffer; + int64_t downstream_length; + int64_t to_write = sp.size(); + + Lock(); + if (references_ == 2) { + while (to_write > 0) { + downstream_blkp = TSIOBufferStart(downstream_buffer_); + downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, &downstream_length); + int64_t bytes_written = to_write > downstream_length ? downstream_length : to_write; + memcpy(downstream_buffer, sp.data() + (sp.size() - to_write), bytes_written); + to_write -= bytes_written; + downstream_length_ += bytes_written; + TSIOBufferProduce(downstream_buffer_, bytes_written); + } + CHECK(to_write == 0) << "to_write failure"; + if (last) { + TSVIONBytesSet(downstream_vio_, downstream_length_); + } + if (reenable) { + TSVIOReenable(downstream_vio_); + } + } + Unlock(); +} + +void AtsBaseFetch::HandleDone(bool success) { + CHECK(!done_called_); + CHECK(downstream_vio_); + TSDebug("ats-speed", "Done()!"); + + Lock(); + done_called_ = true; + ForwardData("", true, true); + DecrefAndDeleteIfUnreferenced(); + Unlock(); +} + +void AtsBaseFetch::DecrefAndDeleteIfUnreferenced() { + if (__sync_add_and_fetch(&references_, -1) == 0) { + delete this; + } +} http://git-wip-us.apache.org/repos/asf/trafficserver/blob/e8b899b0/plugins/experimental/ats_pagespeed/ats_base_fetch.h ---------------------------------------------------------------------- diff --git a/plugins/experimental/ats_pagespeed/ats_base_fetch.h b/plugins/experimental/ats_pagespeed/ats_base_fetch.h new file mode 100644 index 0000000..63b952f --- /dev/null +++ b/plugins/experimental/ats_pagespeed/ats_base_fetch.h @@ -0,0 +1,88 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef ATS_BASE_FETCH_H_ +#define ATS_BASE_FETCH_H_ + +#include <string> + +#include <ts/ts.h> + +#include "ats_pagespeed.h" + +#include "net/instaweb/http/public/async_fetch.h" +#include "net/instaweb/http/public/headers.h" +#include "net/instaweb/util/public/string.h" + + +namespace net_instaweb { + +class AtsServerContext; +class AbstractMutex; + +class AtsBaseFetch : public net_instaweb::AsyncFetch { + +public: + // TODO(oschaaf): change this to take the downstream buffer and vio + // instead of AtsData*. Also, make the bytes send a property + // of the fetch itself instead of tracking it on data. + // Doing so, would allow us to share this with the server intercept + // code for resources. + AtsBaseFetch(AtsServerContext* server_context, + const net_instaweb::RequestContextPtr& request_ctx, + TSVIO downstream_vio, + TSIOBuffer downstream_buffer, + bool is_resource_fetch); + + virtual ~AtsBaseFetch(); + void Release(); +private: + virtual bool HandleWrite(const StringPiece& sp, net_instaweb::MessageHandler* handler); + virtual bool HandleFlush( net_instaweb::MessageHandler* handler); + virtual void HandleHeadersComplete(); + virtual void HandleDone(bool success); + void Lock(); + void Unlock(); + void DecrefAndDeleteIfUnreferenced(); + void ForwardData(const StringPiece& sp, bool reenable, bool last); + GoogleString buffer_; + AtsServerContext* server_context_; + bool done_called_; + bool last_buf_sent_; + + // How many active references there are to this fetch. Starts at two, + // decremented once when Done() is called and once when Release() is called. + int references_; + TSVIO downstream_vio_; + TSIOBuffer downstream_buffer_; + bool is_resource_fetch_; + int64_t downstream_length_; + + // We don't own this mutex + TSMutex txn_mutex_; +}; + +} /* ats_pagespeed */ + + +#endif /* ATS_BASE_FETCH_H_ */ http://git-wip-us.apache.org/repos/asf/trafficserver/blob/e8b899b0/plugins/experimental/ats_pagespeed/ats_beacon_intercept.cc ---------------------------------------------------------------------- diff --git a/plugins/experimental/ats_pagespeed/ats_beacon_intercept.cc b/plugins/experimental/ats_pagespeed/ats_beacon_intercept.cc new file mode 100644 index 0000000..88cf016 --- /dev/null +++ b/plugins/experimental/ats_pagespeed/ats_beacon_intercept.cc @@ -0,0 +1,364 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "ats_beacon_intercept.h" +#include "ats_pagespeed.h" +#include "ats_server_context.h" + +#include "net/instaweb/system/public/system_request_context.h" + +#include <string> +#include <limits.h> +#include <strings.h> +#include <stdio.h> + +using std::string; +using namespace net_instaweb; + +#define DEBUG_TAG "ats_pagespeed_beacon" + +struct InterceptCtx { + TSVConn net_vc; + TSCont contp; + + struct IoHandle { + TSVIO vio; + TSIOBuffer buffer; + TSIOBufferReader reader; + IoHandle() + : vio(0), buffer(0), reader(0) { }; + ~IoHandle() { + if (reader) { + TSIOBufferReaderFree(reader); + } + if (buffer) { + TSIOBufferDestroy(buffer); + } + }; + }; + + IoHandle input; + IoHandle output; + + TSHttpParser http_parser; + string body; + int req_content_len; + TSMBuffer req_hdr_bufp; + TSMLoc req_hdr_loc; + bool req_hdr_parsed; + bool initialized; + TransformCtx* request_context; + InterceptCtx(TSCont cont) + : net_vc(0), contp(cont), input(), output(), body(""), req_content_len(0), req_hdr_bufp(0), req_hdr_loc(0), + req_hdr_parsed(false), initialized(false) { + http_parser = TSHttpParserCreate(); + } + + bool init(TSVConn vconn); + + void setupWrite(); + + ~InterceptCtx() { + TSDebug(DEBUG_TAG, "[%s] Destroying continuation data", __FUNCTION__); + TSHttpParserDestroy(http_parser); + if (req_hdr_loc) { + TSHandleMLocRelease(req_hdr_bufp, TS_NULL_MLOC, req_hdr_loc); + } + if (req_hdr_bufp) { + TSMBufferDestroy(req_hdr_bufp); + } + if (request_context) { + ats_ctx_destroy(request_context); + request_context = NULL; + } + }; +}; + +bool +InterceptCtx::init(TSVConn vconn) +{ + if (initialized) { + TSError("[%s] InterceptCtx already initialized!", __FUNCTION__); + return false; + } + + net_vc = vconn; + + input.buffer = TSIOBufferCreate(); + input.reader = TSIOBufferReaderAlloc(input.buffer); + input.vio = TSVConnRead(net_vc, contp, input.buffer, INT_MAX); + + req_hdr_bufp = TSMBufferCreate(); + req_hdr_loc = TSHttpHdrCreate(req_hdr_bufp); + TSHttpHdrTypeSet(req_hdr_bufp, req_hdr_loc, TS_HTTP_TYPE_REQUEST); + + initialized = true; + TSDebug(DEBUG_TAG, "[%s] InterceptCtx initialized!", __FUNCTION__); + return true; +} + +void +InterceptCtx::setupWrite() { + TSAssert(output.buffer == 0); + output.buffer = TSIOBufferCreate(); + output.reader = TSIOBufferReaderAlloc(output.buffer); + output.vio = TSVConnWrite(net_vc, contp, output.reader, INT_MAX); +} + +// Parses out query params from the request. +void ps_query_params_handler(StringPiece unparsed_uri, StringPiece* data) { + stringpiece_ssize_type question_mark_index = unparsed_uri.find("?"); + if (question_mark_index == StringPiece::npos) { + *data = ""; + } else { + *data = unparsed_uri.substr( + question_mark_index+1, unparsed_uri.size() - (question_mark_index+1)); + } +} + +static bool +handleRead(InterceptCtx *cont_data, bool &read_complete) { + int avail = TSIOBufferReaderAvail(cont_data->input.reader); + if (avail == TS_ERROR) { + TSError("[%s] Error while getting number of bytes available", __FUNCTION__); + return false; + } + + TSDebug(DEBUG_TAG, "[%s] Parsed header, avail: %d", __FUNCTION__, avail); + + int consumed = 0; + if (avail > 0) { + int64_t data_len; + const char *data; + TSIOBufferBlock block = TSIOBufferReaderStart(cont_data->input.reader); + while (block != NULL) { + data = TSIOBufferBlockReadStart(block, cont_data->input.reader, &data_len); + if (!cont_data->req_hdr_parsed) { + const char *endptr = data + data_len; + if (TSHttpHdrParseReq(cont_data->http_parser, cont_data->req_hdr_bufp, cont_data->req_hdr_loc, + &data, endptr) == TS_PARSE_DONE) { + TSDebug(DEBUG_TAG, "[%s] Parsed header", __FUNCTION__); + TSMLoc content_len_loc = TSMimeHdrFieldFind(cont_data->req_hdr_bufp, cont_data->req_hdr_loc, + TS_MIME_FIELD_CONTENT_LENGTH, -1); + + /*if (!content_len_loc) { + TSError("[%s] Error while searching content length header [%s]", + __FUNCTION__, TS_MIME_FIELD_CONTENT_LENGTH); + return false; + } + if (!content_len_loc) { + TSError("[%s] request doesn't contain content length header [%s]", + __FUNCTION__, TS_MIME_FIELD_CONTENT_TYPE); + return false; + }*/ + if (!content_len_loc) { + cont_data->req_content_len = 0; + } else { + cont_data->req_content_len = TSMimeHdrFieldValueIntGet(cont_data->req_hdr_bufp, cont_data->req_hdr_loc, + content_len_loc, 0); + TSHandleMLocRelease(cont_data->req_hdr_bufp, cont_data->req_hdr_loc, content_len_loc); + } + TSDebug(DEBUG_TAG, "[%s] Got content length as %d", __FUNCTION__, cont_data->req_content_len); + if (cont_data->req_content_len < 0) { + TSError("[%s] Invalid content length [%d]", __FUNCTION__, cont_data->req_content_len); + return false; + } + if (endptr - data) { + TSDebug(DEBUG_TAG, "[%s] Appending %ld bytes to body", __FUNCTION__, static_cast<long int>(endptr - data)); + cont_data->body.append(data, endptr - data); + } + cont_data->req_hdr_parsed = true; + } + } else { + //TSDebug(DEBUG_TAG, "[%s] Appending %" PRId64" bytes to body", __FUNCTION__, data_len); + cont_data->body.append(data, data_len); + } + consumed += data_len; + block = TSIOBufferBlockNext(block); + } + } + + TSIOBufferReaderConsume(cont_data->input.reader, consumed); + + TSDebug(DEBUG_TAG, "[%s] Consumed %d bytes from input vio, avail: %d", __FUNCTION__, consumed, avail); + + // Modify the input VIO to reflect how much data we've completed. + TSVIONDoneSet(cont_data->input.vio, TSVIONDoneGet(cont_data->input.vio) + consumed); + + if (static_cast<int>(cont_data->body.size()) == cont_data->req_content_len) { + TSDebug(DEBUG_TAG, "[%s] Completely read body of size %d", __FUNCTION__, cont_data->req_content_len); + read_complete = true; + } else { + read_complete = false; + TSDebug(DEBUG_TAG, "[%s] Reenabling input vio as %ld bytes still need to be read", + __FUNCTION__, static_cast<long int>(cont_data->req_content_len - cont_data->body.size())); + TSVIOReenable(cont_data->input.vio); + } + return true; +} + +static bool +processRequest(InterceptCtx *cont_data) { + // OS: Looks like on 5.x we sometimes receive read complete / EOS events twice, + // which needs looking into. Probably this intercept is doing something it shouldn't + if (cont_data->output.buffer) { + TSDebug("ats_pagespeed", "Received read complete / EOS twice?!"); + return true; + } + string reply_header("HTTP/1.1 204 No Content\r\n"); + int body_size = static_cast<int>(cont_data->body.size()); + if (cont_data->req_content_len != body_size) { + TSError("[%s] Read only %d bytes of body; expecting %d bytes", __FUNCTION__, body_size, + cont_data->req_content_len); + } + + char buf[64]; + //snprintf(buf, 64, "%s: %d\r\n\r\n", TS_MIME_FIELD_CONTENT_LENGTH, body_size); + snprintf(buf, 64, "%s: %d\r\n\r\n", TS_MIME_FIELD_CONTENT_LENGTH, 0); + reply_header.append(buf); + reply_header.append("Cache-Control: max-age=0, no-cache"); + //TSError("[%s] reply header: \n%s", __FUNCTION__, reply_header.data()); + + StringPiece query_param_beacon_data; + ps_query_params_handler(cont_data->request_context->url_string->c_str(), &query_param_beacon_data); + + GoogleString beacon_data = net_instaweb::StrCat( + query_param_beacon_data, "&", cont_data->body); + ServerContext* server_context = cont_data->request_context->server_context; + + SystemRequestContext* system_request_context = + new SystemRequestContext(server_context->thread_system()->NewMutex(), + server_context->timer(), + // TODO(oschaaf): determine these for real. + "www.foo.com", + 80, + "127.0.0.1"); + + if (!server_context->HandleBeacon( + beacon_data, + cont_data->request_context->user_agent->c_str(), + net_instaweb::RequestContextPtr(system_request_context))) { + TSError("Beacon handling failure!"); + } else { + TSDebug(DEBUG_TAG, "Beacon post data processed OK: [%s]", beacon_data.c_str()); + } + + cont_data->setupWrite(); + if (TSIOBufferWrite(cont_data->output.buffer, reply_header.data(), reply_header.size()) == TS_ERROR) { + TSError("[%s] Error while writing reply header", __FUNCTION__); + return false; + } + /* + if (TSIOBufferWrite(cont_data->output.buffer, cont_data->body.data(), body_size) == TS_ERROR) { + TSError("[%s] Error while writing content", __FUNCTION__); + return false; + }*/ + int total_bytes_written = reply_header.size() + body_size; + TSDebug(DEBUG_TAG, "[%s] Wrote reply of size %d", __FUNCTION__, total_bytes_written); + TSVIONBytesSet(cont_data->output.vio, total_bytes_written); + + TSVIOReenable(cont_data->output.vio); + return true; +} + +static int +txn_intercept(TSCont contp, TSEvent event, void *edata) { + TSDebug(DEBUG_TAG, "[%s] Received event: %d", __FUNCTION__, (int)event); + + InterceptCtx *cont_data = static_cast<InterceptCtx *>(TSContDataGet(contp)); + bool read_complete = false; + bool shutdown = false; + switch (event) { + case TS_EVENT_NET_ACCEPT: + TSDebug(DEBUG_TAG, "[%s] Received net accept event", __FUNCTION__); + TSAssert(cont_data->initialized == false); + if (!cont_data->init(static_cast<TSVConn>(edata))) { + TSError("[%s] Could not initialize continuation data!", __FUNCTION__); + return 1; + } + break; + case TS_EVENT_VCONN_READ_READY: + TSDebug(DEBUG_TAG, "[%s] Received read ready event", __FUNCTION__); + if (!handleRead(cont_data, read_complete)) { + TSError("[%s] Error while reading from input vio", __FUNCTION__); + //return 0; + read_complete = true; + } + break; + case TS_EVENT_VCONN_READ_COMPLETE: + case TS_EVENT_VCONN_EOS: + // intentional fall-through + TSDebug(DEBUG_TAG, "[%s] Received read complete/eos event %d", __FUNCTION__, event); + read_complete = true; + break; + case TS_EVENT_VCONN_WRITE_READY: + TSDebug(DEBUG_TAG, "[%s] Received write ready event", __FUNCTION__); + break; + case TS_EVENT_VCONN_WRITE_COMPLETE: + TSDebug(DEBUG_TAG, "[%s] Received write complete event", __FUNCTION__); + shutdown = true; + break; + case TS_EVENT_ERROR: + // todo: do some error handling here + TSDebug(DEBUG_TAG, "[%s] Received error event; going to shutdown, event: %d", __FUNCTION__, event); + TSError("[%s] Received error event; going to shutdown, event: %d", __FUNCTION__, event); + shutdown = true; + break; + default: + break; + } + + if (read_complete) { + if (!processRequest(cont_data)) { + TSError("[%s] Failed to process process", __FUNCTION__); + } else { + TSDebug(DEBUG_TAG, "[%s] Processed request successfully", __FUNCTION__); + } + } + + if (shutdown) { + TSDebug(DEBUG_TAG, "[%s] Completed request processing. Shutting down...", __FUNCTION__); + if (cont_data->net_vc) { + TSVConnClose(cont_data->net_vc); + } + delete cont_data; + TSContDestroy(contp); + } + + return 1; +} + +bool +hook_beacon_intercept(TSHttpTxn txnp) { + TSCont contp = TSContCreate(txn_intercept, TSMutexCreate()); + if (!contp) { + TSError("[%s] Could not create intercept request", __FUNCTION__); + return false; + } + InterceptCtx *cont_data = new InterceptCtx(contp); + cont_data->request_context = get_transaction_context(txnp); + TSContDataSet(contp, cont_data); + TSHttpTxnIntercept(contp, txnp); + TSDebug(DEBUG_TAG, "[%s] Setup server intercept successfully", __FUNCTION__); + return true; +} http://git-wip-us.apache.org/repos/asf/trafficserver/blob/e8b899b0/plugins/experimental/ats_pagespeed/ats_beacon_intercept.h ---------------------------------------------------------------------- diff --git a/plugins/experimental/ats_pagespeed/ats_beacon_intercept.h b/plugins/experimental/ats_pagespeed/ats_beacon_intercept.h new file mode 100644 index 0000000..d53d81c --- /dev/null +++ b/plugins/experimental/ats_pagespeed/ats_beacon_intercept.h @@ -0,0 +1,31 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef _ATS_BEACON_INTERCEPT_H +#define _ATS_BEACON_INTERCEPT_H + +#include "ts/ts.h" + +bool hook_beacon_intercept(TSHttpTxn txnp); + +#endif http://git-wip-us.apache.org/repos/asf/trafficserver/blob/e8b899b0/plugins/experimental/ats_pagespeed/ats_config.cc ---------------------------------------------------------------------- diff --git a/plugins/experimental/ats_pagespeed/ats_config.cc b/plugins/experimental/ats_pagespeed/ats_config.cc new file mode 100644 index 0000000..e0adf42 --- /dev/null +++ b/plugins/experimental/ats_pagespeed/ats_config.cc @@ -0,0 +1,204 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "ats_config.h" + +#include <ts/ts.h> +#include <fstream> + +#include "net/instaweb/util/public/string_util.h" + +#include "ats_message_handler.h" +#include "ats_rewrite_options.h" + +namespace net_instaweb { + +using namespace std; + + + +void ltrim_if(string& s, int (* fp) (int)) { + for (size_t i = 0; i < s.size();) { + if (fp(s[i])) { + s.erase(i,1); + } else { + break; + } + } +} + +void rtrim_if(string& s, int (* fp) (int)) { + for (ssize_t i = (ssize_t)s.size() - 1; i >= 0; i--) { + if (fp(s[i])) { + s.erase(i,1); + } else { + break; + } + } +} + +void trim_if(string& s, int (* fp) (int)) { + ltrim_if(s, fp); + rtrim_if(s, fp); +} + +vector<string> tokenize(const string &s, int (* fp) (int)) { + vector<string> r; + string tmp; + + for (size_t i = 0; i < s.size(); i++) { + if ( fp(s[i]) ) { + if ( tmp.size() ) { + r.push_back(tmp); + tmp = ""; + } + } else { + tmp += s[i]; + } + } + + if ( tmp.size() ) { + r.push_back(tmp); + } + + return r; +} + +AtsConfig::AtsConfig(AtsThreadSystem* thread_system) + : thread_system_(thread_system) { + AddHostConfig(new AtsHostConfig(GoogleString("(XXXXXX)"), new AtsRewriteOptions(thread_system_))); +} + +AtsConfig::~AtsConfig() { + for (size_t i = 0; i < host_configurations_.size(); i++) { + delete host_configurations_[i]; + host_configurations_.clear(); + } +} + +void AtsConfig::AddHostConfig(AtsHostConfig* hc){ + host_configurations_.push_back(hc); +} + +AtsHostConfig::~AtsHostConfig() { + if (options_ != NULL) { + delete options_; + options_ = NULL; + } +} + +AtsHostConfig * AtsConfig::Find(const char * host, int host_length) { + AtsHostConfig * host_configuration = host_configurations_[0]; + + std::string shost(host, host_length); + + for (size_t i = 1; i < host_configurations_.size(); i++ ) { + if (host_configurations_[i]->host() == shost){ + host_configuration = host_configurations_[i]; + break; + } + } + + return host_configuration; +} + +bool AtsConfig::Parse(const char * path ) { + string pathstring(path); + + // If we have a path and it's not an absolute path, make it relative to the + // configuration directory. + if (!pathstring.empty() && pathstring[0] != '/') { + pathstring.assign(TSConfigDirGet()); + pathstring.append("/"); + pathstring.append(path); + } + + trim_if(pathstring, isspace); + + AtsHostConfig* current_host_configuration = host_configurations_[0]; + + if (pathstring.empty()) { + TSError("Empty path passed in AtsConfig::Parse"); + return false; + } + + path = pathstring.c_str(); + std::ifstream f; + + size_t lineno = 0; + + f.open(path, std::ios::in); + + if (!f.is_open()) { + TSError("could not open file [%s], skip",path); + return false; + } + + + while (!f.eof()) { + std::string line; + getline(f, line); + ++lineno; + + trim_if(line, isspace); + if (line.size() == 0) { + continue; + } + if (line[0] == '#') { + continue; + } + + vector<string> v = tokenize( line, isspace ); + if (v.size() == 0) + continue; + GoogleString msg; + AtsMessageHandler handler(thread_system_->NewMutex()); + if (v.size() == 1) { + string token = v[0]; + if ((token[0] == '[') && (token[token.size()-1] == ']')) { + GoogleString current_host = token.substr(1, token.size() - 2); + current_host_configuration = new AtsHostConfig(current_host, new AtsRewriteOptions(thread_system_)); + AddHostConfig(current_host_configuration); + } else if (StringCaseEqual(token,"override_expiry")) { + current_host_configuration->set_override_expiry(true); + } else { + msg = "unknown single token on a line"; + } + } else { + global_settings settings; + v.erase (v.begin()); + const char* err = current_host_configuration->options()->ParseAndSetOptions(v, &handler, settings); + if (err) { + msg.append(err); + } + } + if (msg.size() > 0) { + TSDebug("ats-speed", "Error parsing line [%s]: [%s]", line.c_str(), msg.c_str()); + } + } + + return true; +} + + +} // namespace net_instaweb http://git-wip-us.apache.org/repos/asf/trafficserver/blob/e8b899b0/plugins/experimental/ats_pagespeed/ats_config.h ---------------------------------------------------------------------- diff --git a/plugins/experimental/ats_pagespeed/ats_config.h b/plugins/experimental/ats_pagespeed/ats_config.h new file mode 100644 index 0000000..d3b0e40 --- /dev/null +++ b/plugins/experimental/ats_pagespeed/ats_config.h @@ -0,0 +1,90 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef ATS_CONFIG_H_ +#define ATS_CONFIG_H_ + +#include <string> +#include <vector> + +#include <ts/ts.h> + +#include "ats_thread_system.h" + +#include "net/instaweb/util/public/string.h" +#include "net/instaweb/util/public/string_util.h" + + +namespace net_instaweb { + +class AtsRewriteOptions; + +class AtsHostConfig { +public: + explicit AtsHostConfig(const GoogleString & host, AtsRewriteOptions* options) + : host_(host) + , options_(options) + { + } + virtual ~AtsHostConfig(); + + inline GoogleString host() { return host_; } + inline AtsRewriteOptions* options() { return options_; } + inline bool override_expiry() { return override_expiry_; } + inline void set_override_expiry(bool x) { override_expiry_ = x; } +private: + GoogleString host_; + AtsRewriteOptions* options_; + bool override_expiry_; + DISALLOW_COPY_AND_ASSIGN(AtsHostConfig); +}; // class AtsHostConfig + +class AtsConfig { + friend class AtsHostConfig; +public: + explicit AtsConfig(AtsThreadSystem* thread_system); + virtual ~AtsConfig(); + + // TODO(oschaaf): destructor?? + bool Parse(const char * path); + AtsHostConfig * Find(const char * host, int host_length); + inline AtsHostConfig * GlobalConfiguration() { + return host_configurations_[0]; + } + AtsThreadSystem* thread_system() { + return thread_system_; + } + +private: + void AddHostConfig(AtsHostConfig* hc); + + std::vector<AtsHostConfig *> host_configurations_; + AtsThreadSystem* thread_system_; + //todo: destructor. delete owned host configurations + DISALLOW_COPY_AND_ASSIGN(AtsConfig); +}; // class Configuration + + +} // namespace net_instaweb + +#endif // ATS_CONFIG_H http://git-wip-us.apache.org/repos/asf/trafficserver/blob/e8b899b0/plugins/experimental/ats_pagespeed/ats_header_utils.cc ---------------------------------------------------------------------- diff --git a/plugins/experimental/ats_pagespeed/ats_header_utils.cc b/plugins/experimental/ats_pagespeed/ats_header_utils.cc new file mode 100644 index 0000000..a61c784 --- /dev/null +++ b/plugins/experimental/ats_pagespeed/ats_header_utils.cc @@ -0,0 +1,96 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "ats_header_utils.h" + +GoogleString get_header(TSMBuffer bufp, TSMLoc hdr_loc, const char * header_name) +{ + const char * val = NULL; + int val_len; + TSMLoc field_loc = TSMimeHdrFieldFind( bufp, hdr_loc, header_name, -1); + + if (field_loc) { + val = TSMimeHdrFieldValueStringGet (bufp, hdr_loc, field_loc, 0, &val_len); + TSHandleMLocRelease(bufp,hdr_loc,field_loc); + return GoogleString(val,val_len); + } + + return GoogleString(""); +} + +void unset_header(TSMBuffer bufp, TSMLoc hdr_loc, const char * header_name) +{ + TSMLoc field_loc = TSMimeHdrFieldFind( bufp, hdr_loc, header_name, -1); + + if (field_loc) { + TSMimeHdrFieldDestroy(bufp, hdr_loc, field_loc); + TSHandleMLocRelease(bufp, hdr_loc, field_loc); + } +} + +void hide_accept_encoding(TSMBuffer reqp, TSMLoc hdr_loc, const char * hidden_header_name) +{ + TSMLoc field = TSMimeHdrFieldFind(reqp, hdr_loc, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING); + while (field) { + TSMLoc tmp; + tmp = TSMimeHdrFieldNextDup(reqp, hdr_loc, field); + TSMimeHdrFieldNameSet(reqp, hdr_loc, field, hidden_header_name, -1); + TSHandleMLocRelease(reqp, hdr_loc, field); + field = tmp; + } +} + +void restore_accept_encoding(TSMBuffer reqp, TSMLoc hdr_loc, const char * hidden_header_name) +{ + TSMLoc field = TSMimeHdrFieldFind(reqp, hdr_loc, hidden_header_name, -1); + + while (field) { + TSMLoc tmp; + tmp = TSMimeHdrFieldNextDup(reqp, hdr_loc, field); + TSMimeHdrFieldNameSet(reqp, hdr_loc, field, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING); + TSHandleMLocRelease(reqp, hdr_loc, field); + field = tmp; + } +} + +void set_header(TSMBuffer bufp, TSMLoc hdr_loc, const char * header_name, const char * header_value) +{ + TSMLoc field_loc = TSMimeHdrFieldFind( bufp, hdr_loc, header_name, -1); + + if (field_loc) { + TSMimeHdrFieldValueStringSet(bufp, hdr_loc, field_loc, -1, header_value, -1); + } else { + if ( TSMimeHdrFieldCreate(bufp, hdr_loc, &field_loc) == TS_SUCCESS ) { + TSMimeHdrFieldNameSet(bufp, hdr_loc, field_loc, header_name, -1); + TSMimeHdrFieldAppend(bufp, hdr_loc, field_loc); + TSMimeHdrFieldValueStringSet(bufp,hdr_loc,field_loc,-1,header_value,-1); + } else { + TSError("field creation error for field [%s]", header_name); + return; + } + } + + if (field_loc) { + TSHandleMLocRelease(bufp,hdr_loc,field_loc); + } +} http://git-wip-us.apache.org/repos/asf/trafficserver/blob/e8b899b0/plugins/experimental/ats_pagespeed/ats_header_utils.h ---------------------------------------------------------------------- diff --git a/plugins/experimental/ats_pagespeed/ats_header_utils.h b/plugins/experimental/ats_pagespeed/ats_header_utils.h new file mode 100644 index 0000000..1d6c567 --- /dev/null +++ b/plugins/experimental/ats_pagespeed/ats_header_utils.h @@ -0,0 +1,41 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef ATS_HEADER_UTILS_H +#define ATS_HEADER_UTILS_H + +#include <string> + +#include <ts/ts.h> + +#include "net/instaweb/util/public/string.h" +#include "net/instaweb/util/public/string_util.h" + + +GoogleString get_header(TSMBuffer bufp, TSMLoc hdr_loc, const char * header_name); +void unset_header(TSMBuffer bufp, TSMLoc hdr_loc, const char * header_name); +void hide_accept_encoding(TSMBuffer reqp, TSMLoc hdr_loc, const char * hidden_header_name); +void restore_accept_encoding(TSMBuffer reqp, TSMLoc hdr_loc, const char * hidden_header_name); +void set_header(TSMBuffer bufp, TSMLoc hdr_loc, const char * header_name, const char * header_value); + +#endif // ATS_HEADER_UTILS_H http://git-wip-us.apache.org/repos/asf/trafficserver/blob/e8b899b0/plugins/experimental/ats_pagespeed/ats_log_message_handler.cc ---------------------------------------------------------------------- diff --git a/plugins/experimental/ats_pagespeed/ats_log_message_handler.cc b/plugins/experimental/ats_pagespeed/ats_log_message_handler.cc new file mode 100644 index 0000000..f41b9cc --- /dev/null +++ b/plugins/experimental/ats_pagespeed/ats_log_message_handler.cc @@ -0,0 +1,101 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "ats_log_message_handler.h" + +#include <ts/ts.h> + +#include <unistd.h> + +#include <limits> +#include <string> + +#include "base/debug/debugger.h" +#include "base/debug/stack_trace.h" +#include "base/logging.h" +#include "net/instaweb/public/version.h" +#include "net/instaweb/util/public/string_util.h" + +// Make sure we don't attempt to use LOG macros here, since doing so +// would cause us to go into an infinite log loop. +#undef LOG +#define LOG USING_LOG_HERE_WOULD_CAUSE_INFINITE_RECURSION + +namespace { + +bool LogMessageHandler(int severity, const char* file, int line, + size_t message_start, const GoogleString& str) { + GoogleString message = str; + if (severity == logging::LOG_FATAL) { + if (base::debug::BeingDebugged()) { + base::debug::BreakDebugger(); + } else { + base::debug::StackTrace trace; + std::ostringstream stream; + trace.OutputToStream(&stream); + message.append(stream.str()); + } + } + + // Trim the newline off the end of the message string. + size_t last_msg_character_index = message.length() - 1; + if (message[last_msg_character_index] == '\n') { + message.resize(last_msg_character_index); + } + + TSDebug("ats-speed-vlog", "[%s] %s", + net_instaweb::kModPagespeedVersion, + message.c_str()); + + if (severity == logging::LOG_FATAL) { + // Crash the process to generate a dump. + base::debug::BreakDebugger(); + } + + return true; +} + +} // namespace + + +namespace net_instaweb { + +namespace log_message_handler { + + +const int kDebugLogLevel = -2; + +void Install() { + logging::SetLogMessageHandler(&LogMessageHandler); + + // All VLOG(2) and higher will be displayed as DEBUG logs if the nginx log + // level is DEBUG. + // TODO(oschaaf): from config + //if (log->log_level >= NGX_LOG_DEBUG) { + logging::SetMinLogLevel(-2); + //} +} + +} // namespace log_message_handler + +} // namespace net_instaweb http://git-wip-us.apache.org/repos/asf/trafficserver/blob/e8b899b0/plugins/experimental/ats_pagespeed/ats_log_message_handler.h ---------------------------------------------------------------------- diff --git a/plugins/experimental/ats_pagespeed/ats_log_message_handler.h b/plugins/experimental/ats_pagespeed/ats_log_message_handler.h new file mode 100644 index 0000000..bf57634 --- /dev/null +++ b/plugins/experimental/ats_pagespeed/ats_log_message_handler.h @@ -0,0 +1,36 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef ATS_LOG_MESSAGE_HANDLER_H_ +#define ATS_LOG_MESSAGE_HANDLER_H_ + + +namespace net_instaweb { + + namespace log_message_handler { + void Install(); + } // namespace log_message_handler + +} // namespace net_instaweb + +#endif // ATS_LOG_MESSAGE_HANDLER_H_ http://git-wip-us.apache.org/repos/asf/trafficserver/blob/e8b899b0/plugins/experimental/ats_pagespeed/ats_message_handler.cc ---------------------------------------------------------------------- diff --git a/plugins/experimental/ats_pagespeed/ats_message_handler.cc b/plugins/experimental/ats_pagespeed/ats_message_handler.cc new file mode 100644 index 0000000..370f317 --- /dev/null +++ b/plugins/experimental/ats_pagespeed/ats_message_handler.cc @@ -0,0 +1,114 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "ats_message_handler.h" + +#include <signal.h> +#include <unistd.h> + +#include "net/instaweb/util/public/abstract_mutex.h" +#include "net/instaweb/util/public/debug.h" +#include "net/instaweb/util/public/shared_circular_buffer.h" +#include "net/instaweb/util/public/string_util.h" +#include "net/instaweb/public/version.h" +#include "pagespeed/kernel/base/posix_timer.h" +#include "pagespeed/kernel/base/time_util.h" + + +namespace { + +// This will be prefixed to every logged message. +const char kModuleName[] = "ats_pagespeed"; + +} // namespace + +namespace net_instaweb { + +AtsMessageHandler::AtsMessageHandler(AbstractMutex* mutex) + : mutex_(mutex), + buffer_(NULL) { + SetPidString(static_cast<int64>(getpid())); +} + + +bool AtsMessageHandler::Dump(Writer* writer) { + // Can't dump before SharedCircularBuffer is set up. + if (buffer_ == NULL) { + return false; + } + return buffer_->Dump(writer, &handler_); +} + +void AtsMessageHandler::set_buffer(SharedCircularBuffer* buff) { + ScopedMutex lock(mutex_.get()); + buffer_ = buff; +} + +void AtsMessageHandler::MessageVImpl(MessageType type, const char* msg, + va_list args) { + GoogleString formatted_message = Format(msg, args); + + TSDebug("ats-speed", "[%s %s] %s", kModuleName, kModPagespeedVersion, + formatted_message.c_str()); + + // Prepare a log message for the SharedCircularBuffer only. + // Prepend time and severity to message. + // Format is [time] [severity] [pid] message. + GoogleString message; + GoogleString time; + PosixTimer timer; + if (!ConvertTimeToString(timer.NowMs(), &time)) { + time = "?"; + } + + StrAppend(&message, "[", time, "] ", + "[", MessageTypeToString(type), "] "); + StrAppend(&message, pid_string_, " ", formatted_message, "\n"); + { + ScopedMutex lock(mutex_.get()); + if (buffer_ != NULL) { + buffer_->Write(message); + } + } +} + +void AtsMessageHandler::FileMessageVImpl(MessageType type, const char* file, + int line, const char* msg, + va_list args) { + GoogleString formatted_message = Format(msg, args); + TSDebug("ats-speed", "[%s %s] %s:%d:%s", + kModuleName, kModPagespeedVersion, file, line, + formatted_message.c_str()); +} + +// TODO(sligocki): It'd be nice not to do so much string copying. +GoogleString AtsMessageHandler::Format(const char* msg, va_list args) { + GoogleString buffer; + + // Ignore the name of this routine: it formats with vsnprintf. + // See base/stringprintf.cc. + StringAppendV(&buffer, msg, args); + return buffer; +} + +} // namespace net_instaweb http://git-wip-us.apache.org/repos/asf/trafficserver/blob/e8b899b0/plugins/experimental/ats_pagespeed/ats_message_handler.h ---------------------------------------------------------------------- diff --git a/plugins/experimental/ats_pagespeed/ats_message_handler.h b/plugins/experimental/ats_pagespeed/ats_message_handler.h new file mode 100644 index 0000000..b8248cf --- /dev/null +++ b/plugins/experimental/ats_pagespeed/ats_message_handler.h @@ -0,0 +1,75 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef NGX_MESSAGE_HANDLER_H_ +#define NGX_MESSAGE_HANDLER_H_ + +#include <ts/ts.h> +#include <cstdarg> + +#include "net/instaweb/util/public/basictypes.h" +#include "net/instaweb/util/public/google_message_handler.h" +#include "net/instaweb/util/public/message_handler.h" +#include "net/instaweb/util/public/scoped_ptr.h" +#include "net/instaweb/util/public/string.h" +#include "net/instaweb/util/public/string_util.h" + +namespace net_instaweb { + + class AbstractMutex; + class SharedCircularBuffer; + class Timer; + class Writer; + + class AtsMessageHandler : public GoogleMessageHandler { + public: + explicit AtsMessageHandler(AbstractMutex* mutex); + + void set_buffer(SharedCircularBuffer* buff); + + void SetPidString(const int64 pid) { + pid_string_ = StrCat("[", Integer64ToString(pid), "]"); + } + // Dump contents of SharedCircularBuffer. + bool Dump(Writer* writer); + + protected: + virtual void MessageVImpl(MessageType type, const char* msg, va_list args); + + virtual void FileMessageVImpl(MessageType type, const char* filename, + int line, const char* msg, va_list args); + + private: + GoogleString Format(const char* msg, va_list args); + + scoped_ptr<AbstractMutex> mutex_; + GoogleString pid_string_; + GoogleMessageHandler handler_; + SharedCircularBuffer* buffer_; + + DISALLOW_COPY_AND_ASSIGN(AtsMessageHandler); + }; + +} // namespace net_instaweb + +#endif // NGX_MESSAGE_HANDLER_H_
