TS-1062: Implement dechunk supporting for extended FetchSM With this patch, we can let FetchSM to dechunk body content automatically by passing 'TS_FETCH_FLAGS_DECHUNK' option to TSFetchCreate();
Signed-off-by: Yunkai Zhang <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/trafficserver/repo Commit: http://git-wip-us.apache.org/repos/asf/trafficserver/commit/8a0bee4e Tree: http://git-wip-us.apache.org/repos/asf/trafficserver/tree/8a0bee4e Diff: http://git-wip-us.apache.org/repos/asf/trafficserver/diff/8a0bee4e Branch: refs/heads/master Commit: 8a0bee4e2b4e897c8e9b089bdbc409999338bdd2 Parents: 45f6553 Author: Yunkai Zhang <[email protected]> Authored: Mon Mar 3 18:35:37 2014 +0800 Committer: Yunkai Zhang <[email protected]> Committed: Wed Mar 12 11:02:15 2014 +0800 ---------------------------------------------------------------------- proxy/FetchSM.cc | 59 +++++++++++++++++++++++++++++++------------ proxy/FetchSM.h | 2 ++ proxy/http/HttpTunnel.cc | 52 ++++++++++++++++++++++++++++++++------ proxy/http/HttpTunnel.h | 15 ++++++++--- 4 files changed, 101 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/trafficserver/blob/8a0bee4e/proxy/FetchSM.cc ---------------------------------------------------------------------- diff --git a/proxy/FetchSM.cc b/proxy/FetchSM.cc index 16d4f4c..77d3197 100644 --- a/proxy/FetchSM.cc +++ b/proxy/FetchSM.cc @@ -47,6 +47,11 @@ void FetchSM::cleanUp() { Debug(DEBUG_TAG, "[%s] calling cleanup", __FUNCTION__); + + if (resp_is_chunked > 0 && (fetch_flags & TS_FETCH_FLAGS_DECHUNK)) { + chunked_handler.clear(); + } + free_MIOBuffer(req_buffer); free_MIOBuffer(resp_buffer); mutex.clear(); @@ -165,6 +170,13 @@ FetchSM::check_chunked() len = len > f->len ? f->len : len; if (!strncasecmp(f->str, "chunked", len)) { resp_is_chunked = 1; + if (fetch_flags & TS_FETCH_FLAGS_DECHUNK) { + ChunkedHandler *ch = &chunked_handler; + ch->init_by_action(resp_reader, ChunkedHandler::ACTION_DECHUNK); + ch->dechunked_reader = ch->dechunked_buffer->alloc_reader(); + ch->state = ChunkedHandler::CHUNK_READ_SIZE; + resp_reader->dealloc(); + } return true; } } @@ -178,13 +190,18 @@ FetchSM::dechunk_body() { ink_assert(resp_is_chunked > 0); // - // TODO: dechunk the body content. - // return: + // Return Value: // - 0: need to read more data. // - TS_FETCH_EVENT_EXT_BODY_READY. // - TS_FETCH_EVENT_EXT_BODY_DONE. // - return TS_FETCH_EVENT_EXT_BODY_DONE; + if (chunked_handler.process_chunked_content()) + return TS_FETCH_EVENT_EXT_BODY_DONE; + + if (chunked_handler.dechunked_reader->read_avail()) + return TS_FETCH_EVENT_EXT_BODY_READY; + + return 0; } void @@ -221,11 +238,16 @@ FetchSM::InvokePluginExt(int error_event) goto out; } - if (!resp_reader->read_avail()) - goto out; - Debug(DEBUG_TAG, "[%s] chunked:%d, content_len:%ld, recived_len:%ld, avail:%ld\n", - __FUNCTION__, resp_is_chunked, resp_content_length, resp_recived_body_len, resp_reader->read_avail()); + __FUNCTION__, resp_is_chunked, resp_content_length, resp_recived_body_len, + resp_is_chunked > 0 ? chunked_handler.chunked_reader->read_avail() : resp_reader->read_avail()); + + if (resp_is_chunked > 0) { + if (!chunked_handler.chunked_reader->read_avail()) + goto out; + } else if (!resp_reader->read_avail()) { + goto out; + } if (!check_chunked()) { if (!check_body_done()) @@ -233,14 +255,19 @@ FetchSM::InvokePluginExt(int error_event) else contp->handleEvent(TS_FETCH_EVENT_EXT_BODY_DONE, this); } else if (fetch_flags & TS_FETCH_FLAGS_DECHUNK){ - event = dechunk_body(); + do { + if (chunked_handler.state == ChunkedHandler::CHUNK_FLOW_CONTROL) { + chunked_handler.state = ChunkedHandler::CHUNK_READ_SIZE_START; + } - if (!event) { - read_vio->reenable(); - goto out; - } + event = dechunk_body(); + if (!event) { + read_vio->reenable(); + goto out; + } - contp->handleEvent(event, this); + contp->handleEvent(event, this); + } while (chunked_handler.state == ChunkedHandler::CHUNK_FLOW_CONTROL); } else if (check_body_done()){ contp->handleEvent(TS_FETCH_EVENT_EXT_BODY_DONE, this); } else { @@ -498,8 +525,8 @@ FetchSM::ext_read_data(char *buf, size_t len) if (!header_done) return 0; - if (check_chunked()) - reader = NULL; // TODO: asign dechunking reader + if (check_chunked() && (fetch_flags & TS_FETCH_FLAGS_DECHUNK)) + reader = (tsapi_bufferreader*)chunked_handler.dechunked_reader; else reader = (TSIOBufferReader)resp_reader; @@ -525,7 +552,7 @@ FetchSM::ext_read_data(char *buf, size_t len) } resp_recived_body_len += already; - resp_reader->consume(already); + TSIOBufferReaderConsume(reader, already); read_vio->reenable(); return already; http://git-wip-us.apache.org/repos/asf/trafficserver/blob/8a0bee4e/proxy/FetchSM.h ---------------------------------------------------------------------- diff --git a/proxy/FetchSM.h b/proxy/FetchSM.h index 1e312ef..4a07a72 100644 --- a/proxy/FetchSM.h +++ b/proxy/FetchSM.h @@ -33,6 +33,7 @@ #include "P_Net.h" #include "ts.h" #include "HttpSM.h" +#include "HttpTunnel.h" class FetchSM: public Continuation { @@ -151,6 +152,7 @@ private: Ptr<ProxyMutex> cont_mutex; HTTPParser http_parser; HTTPHdr client_response_hdr; + ChunkedHandler chunked_handler; TSFetchEvent callback_events; TSFetchWakeUpOptions callback_options; bool req_finished; http://git-wip-us.apache.org/repos/asf/trafficserver/blob/8a0bee4e/proxy/http/HttpTunnel.cc ---------------------------------------------------------------------- diff --git a/proxy/http/HttpTunnel.cc b/proxy/http/HttpTunnel.cc index ca2f60c..a22b1c1 100644 --- a/proxy/http/HttpTunnel.cc +++ b/proxy/http/HttpTunnel.cc @@ -70,27 +70,63 @@ ChunkedHandler::ChunkedHandler() void ChunkedHandler::init(IOBufferReader * buffer_in, HttpTunnelProducer * p) { + if (p->do_chunking) + init_by_action(buffer_in, ACTION_DOCHUNK); + else if (p->do_dechunking) + init_by_action(buffer_in, ACTION_DECHUNK); + else + init_by_action(buffer_in, ACTION_PASSTHRU); + return; +} + +void +ChunkedHandler::init_by_action(IOBufferReader *buffer_in, Action action) +{ running_sum = 0; num_digits = 0; cur_chunk_size = 0; bytes_left = 0; truncation = false; + this->action = action; - if (p->do_chunking) { + switch (action) { + case ACTION_DOCHUNK: dechunked_reader = buffer_in->mbuf->clone_reader(buffer_in); dechunked_reader->mbuf->water_mark = min_block_transfer_bytes; chunked_buffer = new_MIOBuffer(CHUNK_IOBUFFER_SIZE_INDEX); chunked_size = 0; - } else { - ink_assert(p->do_dechunking || p->do_chunked_passthru); + break; + case ACTION_DECHUNK: + chunked_reader = buffer_in->mbuf->clone_reader(buffer_in); + dechunked_buffer = new_MIOBuffer(BUFFER_SIZE_INDEX_256); + dechunked_size = 0; + break; + case ACTION_PASSTHRU: chunked_reader = buffer_in->mbuf->clone_reader(buffer_in); + break; + default: + ink_release_assert(!"Unknown action"); + } - if (p->do_dechunking) { - // This is the min_block_transfer_bytes value. - dechunked_buffer = new_MIOBuffer(BUFFER_SIZE_INDEX_256); - dechunked_size = 0; - } + return; +} + +void +ChunkedHandler::clear() +{ + switch (action) { + case ACTION_DOCHUNK: + free_MIOBuffer(chunked_buffer); + break; + case ACTION_DECHUNK: + free_MIOBuffer(dechunked_buffer); + break; + case ACTION_PASSTHRU: + default: + break; } + + return; } void http://git-wip-us.apache.org/repos/asf/trafficserver/blob/8a0bee4e/proxy/http/HttpTunnel.h ---------------------------------------------------------------------- diff --git a/proxy/http/HttpTunnel.h b/proxy/http/HttpTunnel.h index e4bdd71..81fac50 100644 --- a/proxy/http/HttpTunnel.h +++ b/proxy/http/HttpTunnel.h @@ -86,8 +86,7 @@ enum TunnelChunkingAction_t struct ChunkedHandler { - enum ChunkedState - { + enum ChunkedState { CHUNK_READ_CHUNK = 0, CHUNK_READ_SIZE_START, CHUNK_READ_SIZE, @@ -104,6 +103,14 @@ struct ChunkedHandler static int const DEFAULT_MAX_CHUNK_SIZE = 4096; + enum Action { + ACTION_DOCHUNK = 0, + ACTION_DECHUNK, + ACTION_PASSTHRU, + }; + + Action action; + IOBufferReader *chunked_reader; MIOBuffer *dechunked_buffer; int64_t dechunked_size; @@ -137,7 +144,9 @@ struct ChunkedHandler //@} ChunkedHandler(); - void init(IOBufferReader * buffer_in, HttpTunnelProducer * p); + void init(IOBufferReader *buffer_in, HttpTunnelProducer *p); + void init_by_action(IOBufferReader *buffer_in, Action action); + void clear(); /// Set the max chunk @a size. /// If @a size is zero it is set to @c DEFAULT_MAX_CHUNK_SIZE.
