Repository: trafficserver Updated Branches: refs/heads/master 57fa0c3ab -> 5a54dd839
TS-3147 improvements to ESI pugin Project: http://git-wip-us.apache.org/repos/asf/trafficserver/repo Commit: http://git-wip-us.apache.org/repos/asf/trafficserver/commit/5a54dd83 Tree: http://git-wip-us.apache.org/repos/asf/trafficserver/tree/5a54dd83 Diff: http://git-wip-us.apache.org/repos/asf/trafficserver/diff/5a54dd83 Branch: refs/heads/master Commit: 5a54dd83997c6c04f6f428b3e8a6afa890aaa451 Parents: 57fa0c3 Author: Kit Chan <[email protected]> Authored: Sun Oct 19 23:52:43 2014 -0700 Committer: Kit Chan <[email protected]> Committed: Sun Oct 19 23:52:43 2014 -0700 ---------------------------------------------------------------------- plugins/experimental/esi/Makefile.am | 2 + plugins/experimental/esi/esi.cc | 57 +++++---- plugins/experimental/esi/lib/EsiGunzip.cc | 135 ++++++++++++++++++++++ plugins/experimental/esi/lib/EsiGunzip.h | 63 ++++++++++ plugins/experimental/esi/lib/EsiProcessor.cc | 8 +- 5 files changed, 229 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/trafficserver/blob/5a54dd83/plugins/experimental/esi/Makefile.am ---------------------------------------------------------------------- diff --git a/plugins/experimental/esi/Makefile.am b/plugins/experimental/esi/Makefile.am index 1677d51..017f154 100644 --- a/plugins/experimental/esi/Makefile.am +++ b/plugins/experimental/esi/Makefile.am @@ -33,6 +33,7 @@ libesicore_la_SOURCES = \ lib/DocNode.cc \ lib/EsiParser.cc \ lib/EsiGzip.cc \ + lib/EsiGunzip.cc \ lib/EsiProcessor.cc \ lib/Expression.cc \ lib/FailureInfo.cc \ @@ -51,6 +52,7 @@ libtest_la_SOURCES = \ lib/DocNode.cc \ lib/EsiParser.cc \ lib/EsiGzip.cc \ + lib/EsiGunzip.cc \ lib/EsiProcessor.cc \ lib/Expression.cc \ lib/FailureInfo.cc \ http://git-wip-us.apache.org/repos/asf/trafficserver/blob/5a54dd83/plugins/experimental/esi/esi.cc ---------------------------------------------------------------------- diff --git a/plugins/experimental/esi/esi.cc b/plugins/experimental/esi/esi.cc index c6858b4..be8cb60 100644 --- a/plugins/experimental/esi/esi.cc +++ b/plugins/experimental/esi/esi.cc @@ -40,6 +40,7 @@ #include "lib/Utils.h" #include "lib/gzip.h" #include "EsiGzip.h" +#include "EsiGunzip.h" #include "EsiProcessor.h" #include "HttpDataFetcher.h" #include "HandlerManager.h" @@ -65,6 +66,7 @@ static HandlerManager *gHandlerManager = NULL; #define DEBUG_TAG "plugin_esi" #define PROCESSOR_DEBUG_TAG "plugin_esi_processor" #define GZIP_DEBUG_TAG "plugin_esi_gzip" +#define GUNZIP_DEBUG_TAG "plugin_esi_gunzip" #define PARSER_DEBUG_TAG "plugin_esi_parser" #define FETCHER_DEBUG_TAG "plugin_esi_fetcher" #define VARS_DEBUG_TAG "plugin_esi_vars" @@ -100,6 +102,7 @@ struct ContData HttpDataFetcherImpl *data_fetcher; EsiProcessor *esi_proc; EsiGzip *esi_gzip; + EsiGunzip *esi_gunzip; TSCont contp; TSHttpTxn txnp; const struct OptionInfo *option_info; @@ -122,7 +125,7 @@ struct ContData ContData(TSCont contptr, TSHttpTxn tx) : curr_state(READING_ESI_DOC), input_vio(NULL), output_vio(NULL), output_buffer(NULL), output_reader(NULL), - esi_vars(NULL), data_fetcher(NULL), esi_proc(NULL), esi_gzip(NULL), + esi_vars(NULL), data_fetcher(NULL), esi_proc(NULL), esi_gzip(NULL), esi_gunzip(NULL), contp(contptr), txnp(tx), request_url(NULL), input_type(DATA_TYPE_RAW_ESI), packed_node_list(""), gzipped_data(""), gzip_output(false), @@ -229,7 +232,7 @@ ContData::init() // we don't know how much data we are going to write, so INT_MAX output_vio = TSVConnWrite(output_conn, contp, output_reader, INT64_MAX); - string fetcher_tag, vars_tag, expr_tag, proc_tag, gzip_tag; + string fetcher_tag, vars_tag, expr_tag, proc_tag, gzip_tag, gunzip_tag; if (!data_fetcher) { data_fetcher = new HttpDataFetcherImpl(contp, client_addr, createDebugTag(FETCHER_DEBUG_TAG, contp, fetcher_tag)); @@ -244,6 +247,7 @@ ContData::init() &TSDebug, &TSError, *data_fetcher, *esi_vars, *gHandlerManager); esi_gzip = new EsiGzip(createDebugTag(GZIP_DEBUG_TAG, contp, gzip_tag), &TSDebug, &TSError); + esi_gunzip = new EsiGunzip(createDebugTag(GUNZIP_DEBUG_TAG, contp, gunzip_tag), &TSDebug, &TSError); TSDebug(debug_tag, "[%s] Set input data type to [%s]", __FUNCTION__, DATA_TYPE_NAMES_[input_type]); @@ -491,6 +495,9 @@ ContData::~ContData() if(esi_gzip) { delete esi_gzip; } + if(esi_gunzip) { + delete esi_gunzip; + } } static int removeCacheHandler(TSCont contp, TSEvent /* event ATS_UNUSED */, void * /* edata ATS_UNUSED */) { @@ -664,7 +671,9 @@ transformData(TSCont contp) if (cont_data->input_type == DATA_TYPE_RAW_ESI) { cont_data->esi_proc->addParseData(data, data_len); } else if (cont_data->input_type == DATA_TYPE_GZIPPED_ESI) { - cont_data->gzipped_data.append(data, data_len); + string udata = ""; + cont_data->esi_gunzip->stream_decode(data, data_len, udata); + cont_data->esi_proc->addParseData(udata.data(), udata.size()); } else { cont_data->packed_node_list.append(data, data_len); } @@ -710,17 +719,11 @@ transformData(TSCont contp) } if (cont_data->input_type != DATA_TYPE_PACKED_ESI) { + bool gunzip_complete = true; if (cont_data->input_type == DATA_TYPE_GZIPPED_ESI) { - BufferList buf_list; - if (gunzip(cont_data->gzipped_data.data(), cont_data->gzipped_data.size(), buf_list)) { - for (BufferList::iterator iter = buf_list.begin(); iter != buf_list.end(); ++iter) { - cont_data->esi_proc->addParseData(iter->data(), iter->size()); - } - } else { - TSError("[%s] Error while gunzipping data", __FUNCTION__); - } + gunzip_complete = cont_data->esi_gunzip->stream_finish(); } - if (cont_data->esi_proc->completeParse()) { + if (cont_data->esi_proc->completeParse() && gunzip_complete) { if (cont_data->option_info->packed_node_support && cont_data->os_response_cacheable && !cont_data->cache_txn && !cont_data->head_only) { @@ -803,7 +806,7 @@ transformData(TSCont contp) } } - if ((cont_data->curr_state == ContData::FETCHING_DATA) && + if (((cont_data->curr_state == ContData::FETCHING_DATA)||(cont_data->curr_state == ContData::READING_ESI_DOC)) && (cont_data->option_info->first_byte_flush)) { // retest as state may have changed in previous block TSDebug(cont_data->debug_tag, "[%s] trying to process doc", __FUNCTION__); string out_data; @@ -811,7 +814,7 @@ transformData(TSCont contp) int overall_len; EsiProcessor::ReturnCode retval = cont_data->esi_proc->flush(out_data, overall_len); - if (cont_data->data_fetcher->isFetchComplete()) { + if ((cont_data->curr_state == ContData::FETCHING_DATA) && cont_data->data_fetcher->isFetchComplete()) { TSDebug(cont_data->debug_tag, "[%s] data ready; last process() will have finished the entire processing", __FUNCTION__); cont_data->curr_state = ContData::PROCESSING_COMPLETE; } @@ -863,6 +866,10 @@ transformData(TSCont contp) TSError("[%s] Error while finishing gzip", __FUNCTION__); return 0; } else { + if(TSVIOBufferGet(cont_data->output_vio) == NULL) { + TSError("[%s] Error while writing bytes to downstream VC", __FUNCTION__); + return 0; + } if (TSIOBufferWrite(TSVIOBufferGet(cont_data->output_vio), cdata.data(), cdata.size()) == TS_ERROR) { TSError("[%s] Error while writing bytes to downstream VC", __FUNCTION__); return 0; @@ -985,7 +992,8 @@ transformHandler(TSCont contp, TSEvent event, void *edata) if (is_fetch_event) { TSDebug(cont_debug_tag, "[%s] Handling fetch event %d...", __FUNCTION__, event); if (cont_data->data_fetcher->handleFetchEvent(event, edata)) { - if (cont_data->curr_state == ContData::FETCHING_DATA) { + if ((cont_data->curr_state == ContData::FETCHING_DATA) || + (cont_data->curr_state == ContData::READING_ESI_DOC)){ // there's a small chance that fetcher is ready even before // parsing is complete; hence we need to check the state too if(cont_data->option_info->first_byte_flush || @@ -1147,10 +1155,6 @@ modifyResponseHeader(TSCont contp, TSEvent event, void *edata) { TS_MIME_LEN_CONTENT_ENCODING, TS_HTTP_VALUE_GZIP, TS_HTTP_LEN_GZIP); } - if (!have_content_length) { - TSError("[%s] no Content-Length!", __FUNCTION__); - } - if (mod_data->option_info->packed_node_support && mod_data->cache_txn) { addMimeHeaderField(bufp, hdr_loc, TS_MIME_FIELD_VARY, TS_MIME_LEN_VARY, TS_MIME_FIELD_ACCEPT_ENCODING, @@ -1284,7 +1288,6 @@ isTxnTransformable(TSHttpTxn txnp, bool is_cache_txn, bool * intercept_header, b TSMBuffer bufp; TSMLoc hdr_loc; - TSHttpStatus resp_status; TSReturnCode header_obtained; bool retval = false; @@ -1334,16 +1337,6 @@ isTxnTransformable(TSHttpTxn txnp, bool is_cache_txn, bool * intercept_header, b break; // found internal header; no other detection required } - // allow response with valid status code to be transformable - resp_status = TSHttpHdrStatusGet(bufp, hdr_loc); - if (static_cast<int>(resp_status) == static_cast<int>(TS_ERROR)) { - TSError("[%s] Error while getting http status", __FUNCTION__); - break; - } - if (resp_status != TS_HTTP_STATUS_OK) { - TSDebug(DEBUG_TAG, "[%s] non-OK response status %d", __FUNCTION__, resp_status); - } - if ((!checkHeaderValue(bufp, hdr_loc, TS_MIME_FIELD_CONTENT_TYPE, TS_MIME_LEN_CONTENT_TYPE, "text/", 5, true)) && (!checkHeaderValue(bufp, hdr_loc, TS_MIME_FIELD_CONTENT_TYPE, TS_MIME_LEN_CONTENT_TYPE, @@ -1351,6 +1344,8 @@ isTxnTransformable(TSHttpTxn txnp, bool is_cache_txn, bool * intercept_header, b (!checkHeaderValue(bufp, hdr_loc, TS_MIME_FIELD_CONTENT_TYPE, TS_MIME_LEN_CONTENT_TYPE, "application/x-javascript", 24, true)) && (!checkHeaderValue(bufp, hdr_loc, TS_MIME_FIELD_CONTENT_TYPE, TS_MIME_LEN_CONTENT_TYPE, + "application/json", 16, true)) && + (!checkHeaderValue(bufp, hdr_loc, TS_MIME_FIELD_CONTENT_TYPE, TS_MIME_LEN_CONTENT_TYPE, "multipart/mixed", 15, true))) { TSDebug(DEBUG_TAG, "[%s] Not text content", __FUNCTION__); break; @@ -1374,7 +1369,7 @@ isCacheObjTransformable(TSHttpTxn txnp, bool * intercept_header, bool * head_onl TSError("[%s] Couldn't get cache status of object", __FUNCTION__); return false; } - if ((obj_status == TS_CACHE_LOOKUP_HIT_FRESH) || (obj_status == TS_CACHE_LOOKUP_HIT_STALE)) { + if (obj_status == TS_CACHE_LOOKUP_HIT_FRESH) { /* time_t respTime; if (TSHttpTxnCachedRespTimeGet(txnp, &respTime) == TS_SUCCESS) { http://git-wip-us.apache.org/repos/asf/trafficserver/blob/5a54dd83/plugins/experimental/esi/lib/EsiGunzip.cc ---------------------------------------------------------------------- diff --git a/plugins/experimental/esi/lib/EsiGunzip.cc b/plugins/experimental/esi/lib/EsiGunzip.cc new file mode 100644 index 0000000..2c3042b --- /dev/null +++ b/plugins/experimental/esi/lib/EsiGunzip.cc @@ -0,0 +1,135 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "EsiGunzip.h" +#include "gzip.h" +#include <ctype.h> +#include <stdint.h> + +using std::string; +using namespace EsiLib; + +EsiGunzip::EsiGunzip(const char *debug_tag, + ComponentBase::Debug debug_func, ComponentBase::Error error_func) + : ComponentBase(debug_tag, debug_func, error_func), + _downstream_length(0), + _total_data_length(0) { + _init = false; + _success = true; +} + +bool +EsiGunzip::stream_finish() { + if(_init) { + if (inflateEnd(&_zstrm) != Z_OK) { + _errorLog("[%s] inflateEnd failed!", __FUNCTION__); + _success = false; + } + _init = false; + } + + + return _success; +} + +bool +EsiGunzip::stream_decode(const char *data, int data_len, std::string &udata) { + + BufferList buf_list; + + if(!_init) { + + _zstrm.zalloc = Z_NULL; + _zstrm.zfree = Z_NULL; + _zstrm.opaque = Z_NULL; + _zstrm.next_in = 0; + _zstrm.avail_in = 0; + + if (inflateInit2(&_zstrm, MAX_WBITS + 16) != Z_OK) { + _errorLog("[%s] inflateInit2 failed!", __FUNCTION__); + _success = false; + return false; + } + _init = true; + } + + if(data && (data_len > 0)) { + _zstrm.next_in = reinterpret_cast<Bytef *>(const_cast<char *>(data)); + _zstrm.avail_in = data_len; + char raw_buf[BUF_SIZE]; + int inflate_result; + int32_t unzipped_data_size = 0; + int32_t curr_buf_size; + + do { + _zstrm.next_out = reinterpret_cast<Bytef *>(raw_buf); + _zstrm.avail_out = BUF_SIZE; + inflate_result = inflate(&_zstrm, Z_SYNC_FLUSH); + curr_buf_size = -1; + if ((inflate_result == Z_OK)||(inflate_result == Z_BUF_ERROR)) { + curr_buf_size = BUF_SIZE - _zstrm.avail_out; + } else if (inflate_result == Z_STREAM_END) { + curr_buf_size = BUF_SIZE - _zstrm.avail_out; + } + if (curr_buf_size > BUF_SIZE) { + _errorLog("[%s] buf too large", __FUNCTION__); + break; + } + if (curr_buf_size < 1) { + _errorLog("[%s] buf below zero", __FUNCTION__); + break; + } + unzipped_data_size += curr_buf_size; + + // push empty object onto list and add data to in-list object to + // avoid data copy for temporary + buf_list.push_back(string()); + string &curr_buf = buf_list.back(); + curr_buf.assign(raw_buf, curr_buf_size); + + if (inflate_result == Z_STREAM_END) { + break; + } + } while (_zstrm.avail_in > 0); + + _total_data_length += data_len; + } + + for (BufferList::iterator iter = buf_list.begin(); iter != buf_list.end(); ++iter) { + udata.append(iter->data(), iter->size()); + } + + return true; +} + + + +EsiGunzip::~EsiGunzip() { + _downstream_length = 0; + _total_data_length = 0; + _init = false; + _success = true; +} + + + http://git-wip-us.apache.org/repos/asf/trafficserver/blob/5a54dd83/plugins/experimental/esi/lib/EsiGunzip.h ---------------------------------------------------------------------- diff --git a/plugins/experimental/esi/lib/EsiGunzip.h b/plugins/experimental/esi/lib/EsiGunzip.h new file mode 100644 index 0000000..5ded024 --- /dev/null +++ b/plugins/experimental/esi/lib/EsiGunzip.h @@ -0,0 +1,63 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#ifndef _ESI_GUNZIP_H +#define _ESI_GUNZIP_H + +#include "ComponentBase.h" +#include <zlib.h> +#include <string> + +class EsiGunzip : private EsiLib::ComponentBase +{ + +public: + + EsiGunzip(const char *debug_tag, + EsiLib::ComponentBase::Debug debug_func, EsiLib::ComponentBase::Error error_func); + + virtual ~EsiGunzip(); + + bool stream_decode(const char *data, int data_len, std::string &udata); + + inline bool stream_decode(std::string data, std::string &udata) { + return stream_decode(data.data(), data.size(), udata); + } + + bool stream_finish(); + +private: + + int _downstream_length; + int _total_data_length; + z_stream _zstrm; + uLong _crc; + + bool _init; + bool _success; +}; + + +#endif // _ESI_GUNZIP_H + + http://git-wip-us.apache.org/repos/asf/trafficserver/blob/5a54dd83/plugins/experimental/esi/lib/EsiProcessor.cc ---------------------------------------------------------------------- diff --git a/plugins/experimental/esi/lib/EsiProcessor.cc b/plugins/experimental/esi/lib/EsiProcessor.cc index 77f13b0..74213d0 100644 --- a/plugins/experimental/esi/lib/EsiProcessor.cc +++ b/plugins/experimental/esi/lib/EsiProcessor.cc @@ -392,10 +392,6 @@ EsiProcessor::flush(string &data, int &overall_len) { data.assign(""); return SUCCESS; } - if (_curr_state != WAITING_TO_PROCESS) { - _errorLog("[%s] Processor has to finish parsing via completeParse() before process() call", __FUNCTION__); - return FAILURE; - } DocNodeList::iterator node_iter,iter; bool attempt_succeeded; bool attempt_pending; @@ -478,6 +474,7 @@ EsiProcessor::flush(string &data, int &overall_len) { } if (attempt_succeeded) { _debugLog(_debug_tag, "[%s] attempt section succeded; using attempt section", __FUNCTION__); + _n_prescanned_nodes = _n_prescanned_nodes + try_iter->attempt_nodes.size(); _node_list.splice(try_iter->pos, try_iter->attempt_nodes); } else { _debugLog(_debug_tag, "[%s] attempt section errored; trying except section", __FUNCTION__); @@ -485,6 +482,7 @@ EsiProcessor::flush(string &data, int &overall_len) { if (!_preprocess(try_iter->except_nodes, n_prescanned_nodes)) { _errorLog("[%s] Failed to preprocess except nodes", __FUNCTION__); } + _n_prescanned_nodes = _n_prescanned_nodes + try_iter->except_nodes.size(); _node_list.splice(try_iter->pos, try_iter->except_nodes); if (_fetcher.getNumPendingRequests()) { _debugLog(_debug_tag, "[%s] New fetch requests were triggered by except block; " @@ -531,7 +529,7 @@ EsiProcessor::flush(string &data, int &overall_len) { } } - if(!node_pending) { + if(!node_pending && (_curr_state == WAITING_TO_PROCESS)) { _curr_state = PROCESSED; _addFooterData(); }
