Repository: trafficserver Updated Branches: refs/heads/master 2158d61dc -> 8141ceae2
TS-2362: Cache backwards compatibility to 3.2.0. Project: http://git-wip-us.apache.org/repos/asf/trafficserver/repo Commit: http://git-wip-us.apache.org/repos/asf/trafficserver/commit/d6bcd2d9 Tree: http://git-wip-us.apache.org/repos/asf/trafficserver/tree/d6bcd2d9 Diff: http://git-wip-us.apache.org/repos/asf/trafficserver/diff/d6bcd2d9 Branch: refs/heads/master Commit: d6bcd2d90e0bd7494a14375a45a413974158676f Parents: 2158d61 Author: Alan M. Carroll <[email protected]> Authored: Sun Nov 17 15:10:00 2013 -0600 Committer: Alan M. Carroll <[email protected]> Committed: Sat Aug 9 10:58:19 2014 -0500 ---------------------------------------------------------------------- CHANGES | 2 + iocore/cache/Cache.cc | 244 ++++++++++++++++++++++++++++++++++++++-- iocore/cache/CacheDisk.cc | 23 +++- iocore/cache/CachePages.cc | 2 +- iocore/cache/CacheRead.cc | 5 +- iocore/cache/CacheVol.cc | 2 +- iocore/cache/CacheWrite.cc | 7 +- iocore/cache/I_Cache.h | 8 +- iocore/cache/I_CacheDefs.h | 9 +- iocore/cache/P_CacheBC.h | 131 +++++++++++++++++++++ iocore/cache/P_CacheVol.h | 24 ++-- lib/ts/I_Version.h | 9 ++ proxy/Main.cc | 12 ++ proxy/hdrs/HdrHeap.cc | 2 +- proxy/hdrs/HdrHeap.h | 9 ++ proxy/hdrs/URL.cc | 1 - 16 files changed, 451 insertions(+), 39 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/trafficserver/blob/d6bcd2d9/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index f3c0696..0bc4568 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,8 @@ -*- coding: utf-8 -*- Changes with Apache Traffic Server 5.1.0 + *) [TS-2362] Make cache backwards compatible to 3.2.0. + *) [TS-2357] Add option to cache POST requests *) [TS-2996] Add consistent hash method to parent selection. http://git-wip-us.apache.org/repos/asf/trafficserver/blob/d6bcd2d9/iocore/cache/Cache.cc ---------------------------------------------------------------------- diff --git a/iocore/cache/Cache.cc b/iocore/cache/Cache.cc index 7d4a8f1..52d5797 100644 --- a/iocore/cache/Cache.cc +++ b/iocore/cache/Cache.cc @@ -35,6 +35,7 @@ #include "HttpSM.h" #include "HttpCacheSM.h" #include "InkAPIInternal.h" +#include "P_CacheBC.h" #endif // Compilation Options @@ -43,6 +44,8 @@ static size_t DEFAULT_RAM_CACHE_MULTIPLIER = 10; // I.e. 10x 1MB per 1GB of disk. +// This is the oldest version number that is still usable. +static short int const CACHE_DB_MAJOR_VERSION_COMPATIBLE = 21; #define DOCACHE_CLEAR_DYN_STAT(x) \ do { \ @@ -935,6 +938,19 @@ CacheProcessor::cacheInitialized() } } + // Update stripe version data. + if (gnvol) // start with whatever the first stripe is. + cacheProcessor.min_stripe_version = cacheProcessor.max_stripe_version = gvol[0]->header->version; + // scan the rest of the stripes. + for (i = 1; i < gnvol; i++) { + Vol* v = gvol[i]; + if (v->header->version < cacheProcessor.min_stripe_version) + cacheProcessor.min_stripe_version = v->header->version; + if (cacheProcessor.max_stripe_version < v->header->version) + cacheProcessor.max_stripe_version = v->header->version; + } + + if (caches_ready) { Debug("cache_init", "CacheProcessor::cacheInitialized - caches_ready=0x%0X, gnvol=%d", (unsigned int) caches_ready, gnvol); @@ -1363,7 +1379,9 @@ Vol::handle_dir_read(int event, void *data) } } - if (header->magic != VOL_MAGIC || header->version.ink_major != CACHE_DB_MAJOR_VERSION || footer->magic != VOL_MAGIC) { + if (!(header->magic == VOL_MAGIC && footer->magic == VOL_MAGIC && + CACHE_DB_MAJOR_VERSION_COMPATIBLE <= header->version.ink_major && header->version.ink_major <= CACHE_DB_MAJOR_VERSION + )) { Warning("bad footer in cache directory for '%s', clearing", hash_text); Note("clearing cache directory '%s'", hash_text); clear_dir(); @@ -2343,8 +2361,89 @@ static void unmarshal_helper(Doc *doc, Ptr<IOBufferData> &buf, int &okay) { tmp += r; } } + +/** Upgrade a marshalled fragment buffer to the current version. + + @internal I looked at doing this in place (rather than a copy & modify) but + - The in place logic would be even worse than this mess + - It wouldn't save you that much, since you end up doing inserts early in the buffer. + Without extreme care in the logic it could end up doing more copying thatn + the simpler copy & modify. + + @internal This logic presumes the existence of some slack at the end of the buffer, which + is usually the case (because lots of rounding is done). If there isn't enough slack then + this fails and we have a cache miss. The assumption that this is sufficiently rare that + code simplicity takes precedence should be checked at some point. + */ +static bool upgrade_doc_version(Ptr<IOBufferData>& buf) { + // Type definition is close enough to use for initial checking. + cache_bc::Doc_v23* doc = reinterpret_cast<cache_bc::Doc_v23*>(buf->data()); + bool zret = true; + + if (DOC_MAGIC == doc->magic) { + if (0 == doc->hlen) { + Debug("cache_bc", "Doc %p without header, no upgrade needed.", doc); + } else if (CACHE_FRAG_TYPE_HTTP_V23 == doc->doc_type) { + cache_bc::HTTPCacheAlt_v21* alt = reinterpret_cast<cache_bc::HTTPCacheAlt_v21*>(doc->hdr()); + if (alt && alt->is_unmarshalled_format()) { + Ptr<IOBufferData> d_buf(ioDataAllocator.alloc()); + Doc* d_doc; + char* src; + char* dst; + char* hdr_limit = doc->data(); + HTTPInfo::FragOffset* frags = reinterpret_cast<HTTPInfo::FragOffset*>(static_cast<char*>(buf->data()) + cache_bc::sizeofDoc_v23); + int frag_count = doc->_flen / sizeof(HTTPInfo::FragOffset); + size_t n = 0; + size_t content_size = doc->data_len(); + + Debug("cache_bc", "Doc %p is 3.2", doc); + + // Use the same buffer size, fail if no fit. + d_buf->alloc(buf->_size_index, buf->_mem_type); // Duplicate. + d_doc = reinterpret_cast<Doc*>(d_buf->data()); + n = d_buf->block_size(); + + src = buf->data(); + dst = d_buf->data(); + memcpy(dst, src, sizeofDoc); + src += sizeofDoc + doc->_flen; dst += sizeofDoc; n -= sizeofDoc; + + // We copy the fragment table iff there is a fragment table and there is only one alternate. + if (frag_count > 0 && cache_bc::HTTPInfo_v21::marshalled_length(src) > doc->hlen) + frag_count = 0; // inhibit fragment table insertion. + + while (zret && src < hdr_limit) { + zret = cache_bc::HTTPInfo_v21::copy_and_upgrade_unmarshalled_to_v23(dst, src, n, frag_count, frags); + } + if (zret && content_size <= n) { + memcpy(dst, src, content_size); // content + // Must update new Doc::len and Doc::hlen + // dst points at the first byte of the content, or one past the last byte of the alt header. + d_doc->len = (dst - reinterpret_cast<char*>(d_doc)) + content_size; + d_doc->hlen = (dst - reinterpret_cast<char*>(d_doc)) - sizeofDoc; + buf = d_buf; // replace original buffer with new buffer. + } else { + zret = false; + } + } + Doc* n_doc = reinterpret_cast<Doc*>(buf->data()); // access as current version. + // For now the base header size is the same. If that changes we'll need to handle the v22/23 case here + // as with the v21 and shift the content down to accomodate the bigger header. + ink_assert(sizeof(*n_doc) == sizeof(*doc)); + + n_doc->doc_type = CACHE_FRAG_TYPE_HTTP; // We converted so adjust doc_type. + // Set these to zero for debugging - they'll be updated to the current values if/when this is + // put in the aggregation buffer. + n_doc->v_major = 0; + n_doc->v_minor = 0; + n_doc->unused = 0; // force to zero to make future use easier. + } + } + return zret; +} #endif +// [amc] I think this is where all disk reads from cache funnel through here. int CacheVC::handleReadDone(int event, Event *e) { @@ -2371,20 +2470,46 @@ CacheVC::handleReadDone(int event, Event *e) goto Ldone; } + doc = reinterpret_cast<Doc*>(buf->data()); ink_assert(vol->mutex->nthread_holding < 1000); - ink_assert(((Doc *) buf->data())->magic == DOC_MAGIC); + ink_assert(doc->magic == DOC_MAGIC); + + /* We've read the raw data from disk, time to deserialize it. We have to account for a variety of formats that + may be present. + + As of cache version 24 we changed the @a doc_type to indicate a format change in the header which includes + version data inside the header. Prior to that we must use heuristics to deduce the actual format. For this reason + we send that header type off for special processing. Otherwise we can use the in object version to determine + the format. + + All of this processing must produce a serialized header that is compliant with the current version. This includes + updating the doc_type. + */ + + if (doc->doc_type == CACHE_FRAG_TYPE_HTTP_V23) { + if (upgrade_doc_version(buf)) { + doc = reinterpret_cast<Doc*>(buf->data()); // buf may be a new copy + } else { + Debug("cache_bc", "Upgrade of fragment failed - disk %s - doc id = %" PRIx64 ":%" PRIx64 "\n" + , vol->hash_text, read_key->slice64(0), read_key->slice64(1)); + doc->magic = DOC_CORRUPT; + // Should really trash the directory entry for this, as it's never going to work in the future. + // Or does that happen later anyway? + goto Ldone; + } + } // else if (doc->doc_type == CACHE_FRAG_TYPE_HTTP) // handle any version updates based on the object version in the header. + #ifdef VERIFY_JTEST_DATA char xx[500]; - if (read_key && *read_key == ((Doc *) buf->data())->key && request.valid() && !dir_head(&dir) && !vio.ndone) { + if (read_key && *read_key == doc->key && request.valid() && !dir_head(&dir) && !vio.ndone) { int ib = 0, xd = 0; request.url_get()->print(xx, 500, &ib, &xd); char *x = xx; for (int q = 0; q < 3; q++) x = strchr(x + 1, '/'); - ink_assert(!memcmp(((Doc *) buf->data())->data(), x, ib - (x - xx))); + ink_assert(!memcmp(doc->data(), x, ib - (x - xx))); } #endif - doc = (Doc *) buf->data(); if (is_debug_tag_set("cache_read")) { char xt[33]; @@ -2458,10 +2583,10 @@ CacheVC::handleReadDone(int event, Event *e) bool http_copy_hdr = false; #ifdef HTTP_CACHE http_copy_hdr = cache_config_ram_cache_compress && !f.doc_from_ram_cache && - doc->ftype == CACHE_FRAG_TYPE_HTTP && doc->hlen; + doc->doc_type == CACHE_FRAG_TYPE_HTTP && doc->hlen; // If http doc we need to unmarshal the headers before putting in the ram cache // unless it could be compressed - if (!http_copy_hdr && doc->ftype == CACHE_FRAG_TYPE_HTTP && doc->hlen && okay) + if (!http_copy_hdr && doc->doc_type == CACHE_FRAG_TYPE_HTTP && doc->hlen && okay) unmarshal_helper(doc, buf, okay); #endif // Put the request in the ram cache only if its a open_read or lookup @@ -2507,7 +2632,7 @@ CacheVC::handleReadDone(int event, Event *e) } // end VIO::READ check #ifdef HTTP_CACHE // If it could be compressed, unmarshal after - if (http_copy_hdr && doc->ftype == CACHE_FRAG_TYPE_HTTP && doc->hlen && okay) + if (http_copy_hdr && doc->doc_type == CACHE_FRAG_TYPE_HTTP && doc->hlen && okay) unmarshal_helper(doc, buf, okay); #endif } // end io.ok() check @@ -2618,7 +2743,7 @@ LramHit: { f.doc_from_ram_cache = true; io.aio_result = io.aiocb.aio_nbytes; Doc *doc = (Doc*)buf->data(); - if (cache_config_ram_cache_compress && doc->ftype == CACHE_FRAG_TYPE_HTTP && doc->hlen) { + if (cache_config_ram_cache_compress && doc->doc_type == CACHE_FRAG_TYPE_HTTP && doc->hlen) { SET_HANDLER(&CacheVC::handleReadDone); return EVENT_RETURN; } @@ -3521,3 +3646,104 @@ CacheProcessor::find_by_path(char const* path, int len) return 0; } + +// ---------------------------- + +namespace cache_bc { + static size_t const HTTP_ALT_MARSHAL_SIZE = ROUND(sizeof(HTTPCacheAlt), HDR_PTR_SIZE); // current size. + size_t + HTTPInfo_v21::marshalled_length(void* data) + { + size_t zret = ROUND(sizeof(HTTPCacheAlt_v21), HDR_PTR_SIZE); + HTTPCacheAlt_v21* alt = static_cast<HTTPCacheAlt_v21*>(data); + HdrHeap* hdr; + + hdr = reinterpret_cast<HdrHeap*>(reinterpret_cast<char*>(alt) + reinterpret_cast<uintptr_t>(alt->m_request_hdr.m_heap)); + zret += ROUND(hdr->unmarshal_size(), HDR_PTR_SIZE); + hdr = reinterpret_cast<HdrHeap*>(reinterpret_cast<char*>(alt) + reinterpret_cast<uintptr_t>(alt->m_response_hdr.m_heap)); + zret += ROUND(hdr->unmarshal_size(), HDR_PTR_SIZE); + return zret; + } + + // Copy an unmarshalled instance from @a src to @a dst. + // @a src is presumed to be Cache version 21 and the result + // is Cache version 23. @a length is the buffer available in @a dst. + // @return @c false if something went wrong (e.g., data overrun). + bool + HTTPInfo_v21::copy_and_upgrade_unmarshalled_to_v23( + char*& dst, char*& src, size_t& length, int n_frags, FragOffset* frag_offsets + ) + { + // Offsets of the data after the new stuff. + static const size_t OLD_OFFSET = offsetof(HTTPCacheAlt_v21, m_ext_buffer); + static const size_t NEW_OFFSET = offsetof(HTTPCacheAlt_v23, m_ext_buffer); + + HTTPCacheAlt_v21* s_alt = reinterpret_cast<HTTPCacheAlt_v21*>(src); + HTTPCacheAlt_v23* d_alt = reinterpret_cast<HTTPCacheAlt_v23*>(dst); + HdrHeap_v23* s_hdr; + HdrHeap_v23* d_hdr; + size_t hdr_size; + + if (length < HTTP_ALT_MARSHAL_SIZE) return false; // Absolutely no hope in this case. + + memcpy(dst, src, OLD_OFFSET); // initially same data + // Now data that's now after extra + memcpy( static_cast<char*>(dst) + NEW_OFFSET + , static_cast<char*>(src) + OLD_OFFSET + , sizeof(HTTPCacheAlt_v21) - OLD_OFFSET + ); + dst += HTTP_ALT_MARSHAL_SIZE; // move past fixed data. + length -= HTTP_ALT_MARSHAL_SIZE; + + // Extra data is fragment table - set that if we have it. + if (n_frags) { + static size_t const IFT_SIZE = HTTPCacheAlt_v23::N_INTEGRAL_FRAG_OFFSETS * sizeof(FragOffset); + size_t ift_actual = min(n_frags, HTTPCacheAlt_v23::N_INTEGRAL_FRAG_OFFSETS) * sizeof(FragOffset); + + if (length < (HTTP_ALT_MARSHAL_SIZE + n_frags * sizeof(FragOffset) - IFT_SIZE)) + return false; // can't place fragment table. + + d_alt->m_frag_offset_count = n_frags; + d_alt->m_frag_offsets = reinterpret_cast<FragOffset*>(dst - reinterpret_cast<char*>(d_alt)); + + memcpy(d_alt->m_integral_frag_offsets, frag_offsets, ift_actual); + n_frags -= HTTPCacheAlt_v23::N_INTEGRAL_FRAG_OFFSETS; + if (n_frags > 0) { + size_t k = sizeof(FragOffset) * n_frags; + memcpy(dst, frag_offsets + IFT_SIZE, k); + dst += k; + length -= k; + } else if (n_frags < 0) { + memset(dst + ift_actual, 0, IFT_SIZE - ift_actual); + } + } else { + d_alt->m_frag_offset_count = 0; + d_alt->m_frag_offsets = 0; + ink_zero(d_alt->m_integral_frag_offsets); + } + + // Copy over the headers, tweaking the swizzled pointers. + s_hdr = reinterpret_cast<HdrHeap_v23*>(reinterpret_cast<char*>(s_alt) + reinterpret_cast<uintptr_t>(s_alt->m_request_hdr.m_heap)); + d_hdr = reinterpret_cast<HdrHeap_v23*>(dst); + hdr_size = ROUND(s_hdr->unmarshal_size(), HDR_PTR_SIZE); + if (hdr_size > length) return false; + memcpy(d_hdr, s_hdr, hdr_size); + d_alt->m_request_hdr.m_heap = reinterpret_cast<HdrHeap_v23*>(reinterpret_cast<char*>(d_hdr) - reinterpret_cast<char*>(d_alt)); + dst += hdr_size; + length -= hdr_size; + + s_hdr = reinterpret_cast<HdrHeap_v23*>(reinterpret_cast<char*>(s_alt) + reinterpret_cast<uintptr_t>(s_alt->m_response_hdr.m_heap)); + d_hdr = reinterpret_cast<HdrHeap_v23*>(dst); + hdr_size = ROUND(s_hdr->unmarshal_size(), HDR_PTR_SIZE); + if (hdr_size > length) return false; + memcpy(d_hdr, s_hdr, hdr_size); + d_alt->m_response_hdr.m_heap = reinterpret_cast<HdrHeap_v23*>(reinterpret_cast<char*>(d_hdr) - reinterpret_cast<char*>(d_alt)); + dst += hdr_size; + length -= hdr_size; + + src = reinterpret_cast<char*>(s_hdr) + hdr_size; + + return true; + } + +} // cache_bc http://git-wip-us.apache.org/repos/asf/trafficserver/blob/d6bcd2d9/iocore/cache/CacheDisk.cc ---------------------------------------------------------------------- diff --git a/iocore/cache/CacheDisk.cc b/iocore/cache/CacheDisk.cc index 53fde0a..a4b4a48 100644 --- a/iocore/cache/CacheDisk.cc +++ b/iocore/cache/CacheDisk.cc @@ -133,11 +133,24 @@ CacheDisk::openStart(int event, void * /* data ATS_UNUSED */) return openDone(EVENT_IMMEDIATE, 0); } - if (header->magic != DISK_HEADER_MAGIC || header->num_blocks != (uint64_t)len) { - Warning("disk header different for disk %s: clearing the disk", path); - SET_HANDLER(&CacheDisk::clearDone); - clearDisk(); - return EVENT_DONE; + if (header->magic != DISK_HEADER_MAGIC || header->num_blocks != static_cast<uint64_t>(len)) { + uint64_t delta_3_2 = skip - (skip >> STORE_BLOCK_SHIFT); // block count change from 3.2 + if (static_cast<uint64_t>(len) == header->num_blocks + delta_3_2) { + header->num_blocks += delta_3_2; + // Only recover the space if there is a single stripe on this disk. The stripe space allocation logic can fail if + // there is any difference at all in splitting the disk into stripes. The problem is we can add only to the last + // stripe, because otherwise the stripe offsets are wrong. But if the stripes didn't split evenly and the last + // stripe isn't the short one, the split will be different this time. + // Further - the size is encoded in to the disk hash so if the size changes, the data is effectively lost anyway. + // So no space recovery. +// if (header->num_diskvol_blks == 1) +// header->vol_info[0].len += delta_3_2; + } else { + Warning("disk header different for disk %s: clearing the disk", path); + SET_HANDLER(&CacheDisk::clearDone); + clearDisk(); + return EVENT_DONE; + } } cleared = 0; http://git-wip-us.apache.org/repos/asf/trafficserver/blob/d6bcd2d9/iocore/cache/CachePages.cc ---------------------------------------------------------------------- diff --git a/iocore/cache/CachePages.cc b/iocore/cache/CachePages.cc index c35f4f2..cf0ec83 100644 --- a/iocore/cache/CachePages.cc +++ b/iocore/cache/CachePages.cc @@ -323,7 +323,7 @@ ShowCache::handleCacheEvent(int event, Event *e) { CHECK_SHOW(show("<tr><td>sync_serial</td><td>%lu</tr>\n", d->sync_serial)); CHECK_SHOW(show("<tr><td>write_serial</td><td>%lu</tr>\n", d->write_serial)); CHECK_SHOW(show("<tr><td>header length</td><td>%lu</tr>\n", d->hlen)); - CHECK_SHOW(show("<tr><td>fragment type</td><td>%lu</tr>\n", d->ftype)); + CHECK_SHOW(show("<tr><td>fragment type</td><td>%lu</tr>\n", d->doc_type)); CHECK_SHOW(show("<tr><td>No of Alternates</td><td>%d</td></tr>\n", alt_count)); CHECK_SHOW(show("<tr><td>Action</td>\n" http://git-wip-us.apache.org/repos/asf/trafficserver/blob/d6bcd2d9/iocore/cache/CacheRead.cc ---------------------------------------------------------------------- diff --git a/iocore/cache/CacheRead.cc b/iocore/cache/CacheRead.cc index 5b3d071..79bd542 100644 --- a/iocore/cache/CacheRead.cc +++ b/iocore/cache/CacheRead.cc @@ -1076,11 +1076,12 @@ CacheVC::openReadStartHead(int event, Event * e) if (info && info->m_alt) alt_length += info->m_alt->m_unmarshal_len; } Note("OpenReadHead failed for cachekey %X : vector inconsistency - " - "unmarshalled %d expecting %d in %d (base=%d, flen=%d) " + "unmarshalled %d expecting %d in %d (base=%d, ver=%d:%d) " "- vector n=%d size=%d" "first alt=%d[%s]" , key.slice32(0) - , uml, doc->hlen, doc->len, sizeofDoc, doc->_flen + , uml, doc->hlen, doc->len, sizeofDoc + , doc->v_major, doc->v_minor , vector.count(), alt_length , alt->m_magic , (CACHE_ALT_MAGIC_ALIVE == alt->m_magic ? "alive" http://git-wip-us.apache.org/repos/asf/trafficserver/blob/d6bcd2d9/iocore/cache/CacheVol.cc ---------------------------------------------------------------------- diff --git a/iocore/cache/CacheVol.cc b/iocore/cache/CacheVol.cc index 07d6199..72f8463 100644 --- a/iocore/cache/CacheVol.cc +++ b/iocore/cache/CacheVol.cc @@ -217,7 +217,7 @@ CacheVC::scanObject(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */) continue; } - if (doc->ftype != CACHE_FRAG_TYPE_HTTP || !doc->hlen) + if (doc->doc_type != CACHE_FRAG_TYPE_HTTP || !doc->hlen) goto Lskip; last_collision = NULL; http://git-wip-us.apache.org/repos/asf/trafficserver/blob/d6bcd2d9/iocore/cache/CacheWrite.cc ---------------------------------------------------------------------- diff --git a/iocore/cache/CacheWrite.cc b/iocore/cache/CacheWrite.cc index 7108fde..2487938 100644 --- a/iocore/cache/CacheWrite.cc +++ b/iocore/cache/CacheWrite.cc @@ -400,7 +400,7 @@ CacheVC::evacuateReadHead(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */ goto Lcollision; #ifdef HTTP_CACHE alternate_tmp = 0; - if (doc->ftype == CACHE_FRAG_TYPE_HTTP && doc->hlen) { + if (doc->doc_type == CACHE_FRAG_TYPE_HTTP && doc->hlen) { // its an http document if (vector.get_handles(doc->hdr(), doc->hlen) != doc->hlen) { Note("bad vector detected during evacuation"); @@ -751,8 +751,9 @@ agg_copy(char *p, CacheVC *vc) doc->magic = DOC_MAGIC; doc->len = len; doc->hlen = vc->header_len; - doc->ftype = vc->frag_type; - doc->_flen = 0; + doc->doc_type = vc->frag_type; + doc->v_major = CACHE_DB_MAJOR_VERSION; + doc->v_minor = CACHE_DB_MINOR_VERSION; doc->total_len = vc->total_len; doc->first_key = vc->first_key; doc->sync_serial = vol->header->sync_serial; http://git-wip-us.apache.org/repos/asf/trafficserver/blob/d6bcd2d9/iocore/cache/I_Cache.h ---------------------------------------------------------------------- diff --git a/iocore/cache/I_Cache.h b/iocore/cache/I_Cache.h index 5a84f0f..de30eb4 100644 --- a/iocore/cache/I_Cache.h +++ b/iocore/cache/I_Cache.h @@ -67,7 +67,9 @@ typedef HTTPInfo CacheHTTPInfo; struct CacheProcessor:public Processor { CacheProcessor() - : cb_after_init(0) + : min_stripe_version(CACHE_DB_MAJOR_VERSION, CACHE_DB_MINOR_VERSION) + , max_stripe_version(CACHE_DB_MAJOR_VERSION, CACHE_DB_MINOR_VERSION) + , cb_after_init(0) {} virtual int start(int n_cache_threads = 0, size_t stacksize = DEFAULT_STACKSIZE); @@ -184,6 +186,10 @@ struct CacheProcessor:public Processor static int fix; static int start_internal_flags; static int auto_clear_flag; + + VersionNumber min_stripe_version; + VersionNumber max_stripe_version; + CALLBACK_FUNC cb_after_init; }; http://git-wip-us.apache.org/repos/asf/trafficserver/blob/d6bcd2d9/iocore/cache/I_CacheDefs.h ---------------------------------------------------------------------- diff --git a/iocore/cache/I_CacheDefs.h b/iocore/cache/I_CacheDefs.h index 1bf3997..bd5bf46 100644 --- a/iocore/cache/I_CacheDefs.h +++ b/iocore/cache/I_CacheDefs.h @@ -33,8 +33,8 @@ #define CACHE_ALT_INDEX_DEFAULT -1 #define CACHE_ALT_REMOVED -2 -#define CACHE_DB_MAJOR_VERSION 23 -#define CACHE_DB_MINOR_VERSION 2 +#define CACHE_DB_MAJOR_VERSION 24 +#define CACHE_DB_MINOR_VERSION 0 #define CACHE_DIR_MAJOR_VERSION 18 #define CACHE_DIR_MINOR_VERSION 0 @@ -115,12 +115,13 @@ enum CacheDataType enum CacheFragType { CACHE_FRAG_TYPE_NONE, + CACHE_FRAG_TYPE_HTTP_V23, ///< DB version 23 or prior. + CACHE_FRAG_TYPE_RTSP, ///< Should be removed once Cache Toolkit is implemented. CACHE_FRAG_TYPE_HTTP, - CACHE_FRAG_TYPE_RTSP, NUM_CACHE_FRAG_TYPES }; -#define CacheKey INK_MD5 +typedef CryptoHash CacheKey; #define CACHE_ALLOW_MULTIPLE_WRITES 1 #define CACHE_EXPECTED_SIZE 32768 http://git-wip-us.apache.org/repos/asf/trafficserver/blob/d6bcd2d9/iocore/cache/P_CacheBC.h ---------------------------------------------------------------------- diff --git a/iocore/cache/P_CacheBC.h b/iocore/cache/P_CacheBC.h new file mode 100644 index 0000000..cc1c400 --- /dev/null +++ b/iocore/cache/P_CacheBC.h @@ -0,0 +1,131 @@ +/** @file + + Backwards compatibility support for the cache. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + + +#ifndef _P_CACHE_BC_H__ +#define _P_CACHE_BC_H__ + +namespace cache_bc { + + /* This looks kind of dumb, but I think it's useful. We import external structure + dependencies in to this namespace so we can at least (1) notice them and + (2) change them if the current structure changes. + */ + + typedef HTTPHdr HTTPHdr_v21; + typedef HdrHeap HdrHeap_v23; + typedef CryptoHash CryptoHash_v23; + typedef HTTPCacheAlt HTTPCacheAlt_v23; + + /** Cache backwards compatibility structure - the fragment table. + This is copied from @c HTTPCacheAlt in @c HTTP.h. + */ + struct HTTPCacheFragmentTable { + /// # of fragment offsets in this alternate. + /// @note This is one less than the number of fragments. + int m_frag_offset_count; + /// Type of offset for a fragment. + typedef uint64_t FragOffset; + /// Table of fragment offsets. + /// @note The offsets are forward looking so that frag[0] is the + /// first byte past the end of fragment 0 which is also the first + /// byte of fragment 1. For this reason there is no fragment offset + /// for the last fragment. + FragOffset *m_frag_offsets; + /// # of fragment offsets built in to object. + static int const N_INTEGRAL_FRAG_OFFSETS = 4; + /// Integral fragment offset table. + FragOffset m_integral_frag_offsets[N_INTEGRAL_FRAG_OFFSETS]; + }; + + // From before moving the fragment table to the alternate. + struct HTTPCacheAlt_v21 + { + uint32_t m_magic; + + int32_t m_writeable; + int32_t m_unmarshal_len; + + int32_t m_id; + int32_t m_rid; + + int32_t m_object_key[4]; + int32_t m_object_size[2]; + + HTTPHdr_v21 m_request_hdr; + HTTPHdr_v21 m_response_hdr; + + time_t m_request_sent_time; + time_t m_response_received_time; + + RefCountObj *m_ext_buffer; + + // The following methods were added for BC support. + // Checks itself to verify that it is unmarshalled and v21 format. + bool is_unmarshalled_format() const { + return CACHE_ALT_MAGIC_MARSHALED == m_magic && reinterpret_cast<intptr_t>(m_request_hdr.m_heap) == sizeof(*this); + } + }; + + /// Really just a namespace, doesn't depend on any of the members. + struct HTTPInfo_v21 { + typedef uint64_t FragOffset; + /// Version upgrade methods + /// @a src , @a dst , and @a n are updated upon return. + /// @a n is the space in @a dst remaining. + /// @return @c false if something went wrong. + static bool copy_and_upgrade_unmarshalled_to_v23(char*& dst, char*& src, size_t& length, int n_frags, FragOffset* frag_offsets); + /// The size of the marshalled data of a marshalled alternate header. + static size_t marshalled_length(void* data); + }; + + /// Pre version 24. + struct Doc_v23 + { + uint32_t magic; // DOC_MAGIC + uint32_t len; // length of this segment (including hlen, flen & sizeof(Doc), unrounded) + uint64_t total_len; // total length of document + CryptoHash_v23 first_key; ///< first key in object. + CryptoHash_v23 key; ///< Key for this doc. + uint32_t hlen; ///< Length of this header. + uint32_t doc_type:8; ///< Doc type - indicates the format of this structure and its content. + uint32_t _flen:24; ///< Fragment table length. + uint32_t sync_serial; + uint32_t write_serial; + uint32_t pinned; // pinned until + uint32_t checksum; + + char* hdr(); + char* data(); + size_t data_len(); + }; + + static size_t const sizeofDoc_v23 = sizeof(Doc_v23); + char* Doc_v23::data() { return reinterpret_cast<char*>(this) + sizeofDoc_v23 + _flen + hlen; } + size_t Doc_v23::data_len() { return len - sizeofDoc_v23 - hlen; } + char* Doc_v23::hdr() { return reinterpret_cast<char*>(this) + sizeofDoc_v23; } + + +} // namespace cache_bc + +#endif /* _P_CACHE_BC_H__ */ http://git-wip-us.apache.org/repos/asf/trafficserver/blob/d6bcd2d9/iocore/cache/P_CacheVol.h ---------------------------------------------------------------------- diff --git a/iocore/cache/P_CacheVol.h b/iocore/cache/P_CacheVol.h index 7c8ea0e..c14c200 100644 --- a/iocore/cache/P_CacheVol.h +++ b/iocore/cache/P_CacheVol.h @@ -623,13 +623,15 @@ struct CacheVol struct Doc { uint32_t magic; // DOC_MAGIC - uint32_t len; // length of this segment (including hlen, flen & sizeof(Doc), unrounded) + uint32_t len; // length of this fragment (including hlen & sizeof(Doc), unrounded) uint64_t total_len; // total length of document - CryptoHash first_key; // first key in document (http: vector) - CryptoHash key; - uint32_t hlen; // header length - uint32_t ftype:8; // fragment type CACHE_FRAG_TYPE_XX - uint32_t _flen:24; // fragment table length [amc] NOT USED + CryptoHash first_key; ///< first key in object. + CryptoHash key; ///< Key for this doc. + uint32_t hlen; ///< Length of this header. + uint32_t doc_type:8; ///< Doc type - indicates the format of this structure and its content. + uint32_t v_major:8; ///< Major version number. + uint32_t v_minor:8; ///< Minor version number. + uint32_t unused:8; ///< Unused, forced to zero. uint32_t sync_serial; uint32_t write_serial; uint32_t pinned; // pinned until @@ -774,31 +776,31 @@ vol_relative_length(Vol *v, off_t start_offset) TS_INLINE uint32_t Doc::prefix_len() { - return sizeofDoc + hlen + _flen; + return sizeofDoc + hlen; } TS_INLINE uint32_t Doc::data_len() { - return len - sizeofDoc - hlen - _flen; + return len - sizeofDoc - hlen; } TS_INLINE int Doc::single_fragment() { - return (data_len() == total_len); + return data_len() == total_len; } TS_INLINE char * Doc::hdr() { - return ((char *) this) + sizeofDoc + _flen; + return reinterpret_cast<char*>(this) + sizeofDoc; } TS_INLINE char * Doc::data() { - return ((char *) this) + sizeofDoc + _flen + hlen; + return this->hdr() + hlen; } int vol_dir_clear(Vol *d); http://git-wip-us.apache.org/repos/asf/trafficserver/blob/d6bcd2d9/lib/ts/I_Version.h ---------------------------------------------------------------------- diff --git a/lib/ts/I_Version.h b/lib/ts/I_Version.h index adfdd57..668c2e7 100644 --- a/lib/ts/I_Version.h +++ b/lib/ts/I_Version.h @@ -35,8 +35,17 @@ struct VersionNumber { short int ink_major; // incompatible change short int ink_minor; // minor change, not incompatible + + VersionNumber() {} + VersionNumber(short int major, short int minor) : ink_major(major), ink_minor(minor) {} }; +inline bool operator < (VersionNumber const& lhs, VersionNumber const& rhs) { + return lhs.ink_major < rhs.ink_major || + (lhs.ink_major == rhs.ink_major && lhs.ink_minor < rhs.ink_minor) + ; +} + struct Version { VersionNumber cacheDB; http://git-wip-us.apache.org/repos/asf/trafficserver/blob/d6bcd2d9/proxy/Main.cc ---------------------------------------------------------------------- diff --git a/proxy/Main.cc b/proxy/Main.cc index c1f97d7..222e374 100644 --- a/proxy/Main.cc +++ b/proxy/Main.cc @@ -396,6 +396,18 @@ CB_After_Cache_Init() int start; start = ink_atomic_swap(&delay_listen_for_cache_p, -1); + + // Check for cache BC after the cache is initialized and before listen, if possible. + if (cacheProcessor.min_stripe_version.ink_major < CACHE_DB_MAJOR_VERSION) { + // Versions before 23 need the MMH hash. + if (cacheProcessor.min_stripe_version.ink_major < 23) { + Debug("cache_bc", "Pre 4.0 stripe (cache version %d.%d) found, forcing MMH hash for cache URLs" + , cacheProcessor.min_stripe_version.ink_major, cacheProcessor.min_stripe_version.ink_minor + ); + URLHashContext::Setting = URLHashContext::MMH; + } + } + if (1 == start) { Debug("http_listen", "Delayed listen enable, cache initialization finished"); start_HttpProxyServer(); http://git-wip-us.apache.org/repos/asf/trafficserver/blob/d6bcd2d9/proxy/hdrs/HdrHeap.cc ---------------------------------------------------------------------- diff --git a/proxy/hdrs/HdrHeap.cc b/proxy/hdrs/HdrHeap.cc index cd762e6..790ff90 100644 --- a/proxy/hdrs/HdrHeap.cc +++ b/proxy/hdrs/HdrHeap.cc @@ -879,7 +879,7 @@ HdrHeap::unmarshal(int buf_length, int obj_type, HdrHeapObjImpl ** found_obj, Re return -1; } - int unmarshal_size = m_size + m_ronly_heap[0].m_heap_len; + int unmarshal_size = this->unmarshal_size(); if (unmarshal_size > buf_length) { ink_assert(!"HdrHeap::unmarshal truncated header"); return -1; http://git-wip-us.apache.org/repos/asf/trafficserver/blob/d6bcd2d9/proxy/hdrs/HdrHeap.h ---------------------------------------------------------------------- diff --git a/proxy/hdrs/HdrHeap.h b/proxy/hdrs/HdrHeap.h index 07d681b..dfe01ce 100644 --- a/proxy/hdrs/HdrHeap.h +++ b/proxy/hdrs/HdrHeap.h @@ -198,6 +198,10 @@ public: inkcoreapi int marshal_length(); inkcoreapi int marshal(char *buf, int length); int unmarshal(int buf_length, int obj_type, HdrHeapObjImpl ** found_obj, RefCountObj * block_ref); + /// Computes the valid data size of an unmarshalled instance. + /// Callers should round up to HDR_PTR_SIZE to get the actual footprint. + int unmarshal_size() const; // TBD - change this name, it's confusing. + // One option - overload marshal_length to return this value if @a magic is HDR_BUF_MAGIC_MARSHALED. void inherit_string_heaps(const HdrHeap * inherit_from); int attach_block(IOBufferBlock * b, const char *use_start); @@ -307,6 +311,11 @@ HdrHeap::free_string(const char *s, int len) } } +inline int +HdrHeap::unmarshal_size() const { + return m_size + m_ronly_heap[0].m_heap_len; +} + // struct MarshalXlate http://git-wip-us.apache.org/repos/asf/trafficserver/blob/d6bcd2d9/proxy/hdrs/URL.cc ---------------------------------------------------------------------- diff --git a/proxy/hdrs/URL.cc b/proxy/hdrs/URL.cc index 9a7bfec..06f8e53 100644 --- a/proxy/hdrs/URL.cc +++ b/proxy/hdrs/URL.cc @@ -1703,7 +1703,6 @@ url_MD5_get_general(URLImpl * url, CryptoContext& ctx, CryptoHash& hash) ctx.update(&port, sizeof(port)); ctx.finalize(hash); - Debug("amc", "hash 0x%" PRIx64 ":%" PRIx64, hash[0], hash[1]); } void
