Repository: trafficserver Updated Branches: refs/heads/master 3710a276c -> 83248169b
TS-3000: Add seed string for cache storage. Project: http://git-wip-us.apache.org/repos/asf/trafficserver/repo Commit: http://git-wip-us.apache.org/repos/asf/trafficserver/commit/83248169 Tree: http://git-wip-us.apache.org/repos/asf/trafficserver/tree/83248169 Diff: http://git-wip-us.apache.org/repos/asf/trafficserver/diff/83248169 Branch: refs/heads/master Commit: 83248169bbdaab383583fbb0b25f0916b18594c4 Parents: 3710a27 Author: Alan M. Carroll <[email protected]> Authored: Mon Aug 11 19:06:53 2014 -0500 Committer: Alan M. Carroll <[email protected]> Committed: Mon Aug 11 20:42:17 2014 -0500 ---------------------------------------------------------------------- .../configuration/storage.config.en.rst | 56 +++++-- iocore/cache/Cache.cc | 55 ++++--- iocore/cache/CacheDir.cc | 20 +-- iocore/cache/CachePagesInternal.cc | 2 +- iocore/cache/CacheWrite.cc | 4 +- iocore/cache/I_Store.h | 52 +++++-- iocore/cache/P_CacheDisk.h | 7 +- iocore/cache/P_CacheVol.h | 2 +- iocore/cache/Store.cc | 145 +++++++++---------- lib/ts/ink_memory.h | 6 + 10 files changed, 207 insertions(+), 142 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/trafficserver/blob/83248169/doc/reference/configuration/storage.config.en.rst ---------------------------------------------------------------------- diff --git a/doc/reference/configuration/storage.config.en.rst b/doc/reference/configuration/storage.config.en.rst index 9236344..af019b1 100644 --- a/doc/reference/configuration/storage.config.en.rst +++ b/doc/reference/configuration/storage.config.en.rst @@ -21,23 +21,31 @@ storage.config .. configfile:: storage.config -The :file:`storage.config` file (by default, located in +The :file:`storage.config` file (by default, located in ``/opt/trafficserver/etc/trafficserver/``) lists all the files, directories, and/or hard disk partitions that make up the Traffic Server cache. After you -modify the :file:`storage.config` file, you must restart Traffic Server. +modify the :file:`storage.config` file the new settings will not be effective until Traffic Server is restarted. Format ====== -The format of the :file:`storage.config` file is:: +The format of the :file:`storage.config` file is a series of lines of the form - pathname size volume=volume_number + *pathname* *size* [ ``volume=``\ *number* ] [ ``seed=``\ *string* ] -where :arg:`pathname` is the name of a partition, directory or file, :arg:`size` -is the size of the named partition, directory or file (in bytes), and -:arg:`volume` is the volume number that is used in :file:`volume.config` -and :file:`hosting.config`. You must specify a size for directories or -files; size is optional for raw partitions. :arg:`volume` is optional. +where :arg:`pathname` is the name of a partition, directory or file, :arg:`size` is the size of the named partition, +directory or file (in bytes), and :arg:`volume` is the volume number used in the files :file:`volume.config` and +:file:`hosting.config`. :arg:`seed` is used for seeding the :ref:`assignment-table`. You must specify a size for +directories or files; size is optional for raw partitions. :arg:`volume` is optional and :arg:`seed` are optional. + +.. note:: + + The :arg:`volume` option is independent of the :arg:`seed` option and either can be used with or without the other, + and their ordering on the line is irrelevant. + +.. note:: + + If the :arg:`seed` option is used every use must have a unique value for :arg:`string`. You can use any partition of any size. For best performance: @@ -67,6 +75,22 @@ supported. They include - ``G`` Gigabytes (1024^3 or 1,073,741,824 bytes) - ``T`` Terabytes (1024^4 or 1,099,511,627,776 bytes) +.. _assignment-table: + +Assignment Table +---------------- + +Each storage element defined in :file:`storage.config` is divided in to :term:`stripes`. The assignment table maps from +an object URL to a specific stripe. The table is initialized based on a pseudo-random process which is seeded by hashing +a string for each stripe. This string is composed of a seed string, an offset (the start of the stripe on the storage +element) and the length of the stripe. By default the path for the storage is used as the seed string. This ensures that +each stripe has a unique string for the assignment hash. This does make the assignment table very sensitive to the path +for the storage elements and changing even one can have a cascading effect which will effectively clear most of the cache. +This can be problem when drives fail and a system reboot causes the path names to change. + +The :arg:`seed` option can be used to create a fixed string that an administrator can use to keep the assignment table +consistent even if a device has a changed path. This value of the option is used instead of the path as the seed string +for the assignment table hash. Examples ======== @@ -90,7 +114,7 @@ cache file with:: .. note:: When using on-filesystem cache disk storage, you can only have one such directory specified. This will be address in a future version. - + Solaris Example --------------- @@ -124,6 +148,17 @@ In order to apply these settings, trigger a reload with :manpage:`udevadm(8)`::: udevadm trigger --subsystem-match=block +As an implementation note, modern Linux supports `alternative symlinked names for disk devices +<https://wiki.archlinux.org/index.php/persistent_block_device_naming>`_ in the ``/dev/disk`` directory structure. As +noted for the :ref:`assignment-table` the path used for the disk can effect the cache if it changes. This can be +ameloriated in some cases by using one of the alternate paths in via ``/dev/disk``. Note that if the ``by-id`` style is +used, replacing a failed drive will cause that path to change because the new drive will have a different physical ID. + +If this is not sufficient then the :arg:`seed` argument should be used to create a more permanent assignment table. An +example would be:: + + /dev/sde seed=cache.disk.0 + /dev/sdg seed=cache.disk.1 FreeBSD Example --------------- @@ -143,4 +178,3 @@ following rules are stored in :manpage:`devfs.conf(5)`:: # Assign /dev/ada1 and /dev/ada2 to the tserver user own ada[12] tserver:tserver - http://git-wip-us.apache.org/repos/asf/trafficserver/blob/83248169/iocore/cache/Cache.cc ---------------------------------------------------------------------- diff --git a/iocore/cache/Cache.cc b/iocore/cache/Cache.cc index be3674f..3913b71 100644 --- a/iocore/cache/Cache.cc +++ b/iocore/cache/Cache.cc @@ -739,10 +739,14 @@ CacheProcessor::start_internal(int flags) } } if (diskok) { + int sector_size = sd->hw_sector_size; + gdisks[gndisks] = new CacheDisk(); - gdisks[gndisks]->forced_volume_num = sd->vol_num; + gdisks[gndisks]->forced_volume_num = sd->forced_volume_num; + if (sd->hash_seed_string) + gdisks[gndisks]->hash_seed_string = ats_strdup(sd->hash_seed_string); + Debug("cache_hosting", "Disk: %d, blocks: %d", gndisks, blocks); - int sector_size = sd->hw_sector_size; if (sector_size < cache_config_force_sector_size) sector_size = cache_config_force_sector_size; @@ -1133,7 +1137,7 @@ int Vol::db_check(bool /* fix ATS_UNUSED */ ) { char tt[256]; - printf(" Data for [%s]\n", hash_text); + printf(" Data for [%s]\n", hash_text.get()); printf(" Length: %" PRIu64 "\n", (uint64_t)len); printf(" Write Position: %" PRIu64 "\n", (uint64_t) (header->write_pos - skip)); printf(" Phase: %d\n", (int)!!header->phase); @@ -1231,7 +1235,7 @@ vol_dir_clear(Vol *d) vol_clear_init(d); if (pwrite(d->fd, d->raw_dir, dir_len, d->skip) < 0) { - Warning("unable to clear cache directory '%s'", d->hash_text); + Warning("unable to clear cache directory '%s'", d->hash_text.get()); return -1; } return 0; @@ -1259,15 +1263,18 @@ Vol::clear_dir() int Vol::init(char *s, off_t blocks, off_t dir_skip, bool clear) { - dir_skip = ROUND_TO_STORE_BLOCK((dir_skip < START_POS ? START_POS : dir_skip)); - path = ats_strdup(s); - const size_t hash_text_size = strlen(s) + 32; - hash_text = (char *)ats_malloc(hash_text_size); - ink_strlcpy(hash_text, s, hash_text_size); - const size_t s_size = strlen(s); - snprintf(hash_text + s_size, (hash_text_size - s_size), " %" PRIu64 ":%" PRIu64 "", + char* seed_str = disk->hash_seed_string ? disk->hash_seed_string : s; + const size_t hash_seed_size = strlen(seed_str); + const size_t hash_text_size = hash_seed_size + 32; + + hash_text = static_cast<char *>(ats_malloc(hash_text_size)); + ink_strlcpy(hash_text, seed_str, hash_text_size); + snprintf(hash_text + hash_seed_size, (hash_text_size - hash_seed_size), " %" PRIu64 ":%" PRIu64 "", (uint64_t)dir_skip, (uint64_t)blocks); MD5Context().hash_immediate(hash_id, hash_text, strlen(hash_text)); + + dir_skip = ROUND_TO_STORE_BLOCK((dir_skip < START_POS ? START_POS : dir_skip)); + path = ats_strdup(s); len = blocks * STORE_BLOCK_SIZE; ink_assert(len <= MAX_VOL_SIZE); skip = dir_skip; @@ -1305,7 +1312,7 @@ Vol::init(char *s, off_t blocks, off_t dir_skip, bool clear) #endif if (clear) { - Note("clearing cache directory '%s'", hash_text); + Note("clearing cache directory '%s'", hash_text.get()); return clear_dir(); } @@ -1315,7 +1322,7 @@ Vol::init(char *s, off_t blocks, off_t dir_skip, bool clear) // try A off_t as = skip; if (is_debug_tag_set("cache_init")) - Note("reading directory '%s'", hash_text); + Note("reading directory '%s'", hash_text.get()); SET_HANDLER(&Vol::handle_header_read); init_info->vol_aio[0].aiocb.aio_offset = as; init_info->vol_aio[1].aiocb.aio_offset = as + footer_offset; @@ -1349,7 +1356,7 @@ Vol::handle_dir_clear(int event, void *data) if (event == AIO_EVENT_DONE) { op = (AIOCallback *) data; if ((size_t) op->aio_result != (size_t) op->aiocb.aio_nbytes) { - Warning("unable to clear cache directory '%s'", hash_text); + Warning("unable to clear cache directory '%s'", hash_text.get()); fd = -1; } @@ -1385,8 +1392,8 @@ Vol::handle_dir_read(int event, void *data) if (!(header->magic == VOL_MAGIC && footer->magic == VOL_MAGIC && CACHE_DB_MAJOR_VERSION_COMPATIBLE <= header->version.ink_major && header->version.ink_major <= CACHE_DB_MAJOR_VERSION )) { - Warning("bad footer in cache directory for '%s', clearing", hash_text); - Note("clearing cache directory '%s'", hash_text); + Warning("bad footer in cache directory for '%s', clearing", hash_text.get()); + Note("clearing cache directory '%s'", hash_text.get()); clear_dir(); return EVENT_DONE; } @@ -1482,7 +1489,7 @@ Vol::handle_recover_from_data(int event, void * /* data ATS_UNUSED */ ) io.aiocb.aio_nbytes = (skip + len) - recover_pos; } else if (event == AIO_EVENT_DONE) { if ((size_t) io.aiocb.aio_nbytes != (size_t) io.aio_result) { - Warning("disk read error on recover '%s', clearing", hash_text); + Warning("disk read error on recover '%s', clearing", hash_text.get()); goto Lclear; } if (io.aiocb.aio_offset == header->last_write_pos) { @@ -1499,7 +1506,7 @@ Vol::handle_recover_from_data(int event, void * /* data ATS_UNUSED */ ) while (done < to_check) { Doc *doc = (Doc *) (s + done); if (doc->magic != DOC_MAGIC || doc->write_serial > header->write_serial) { - Warning("no valid directory found while recovering '%s', clearing", hash_text); + Warning("no valid directory found while recovering '%s', clearing", hash_text.get()); goto Lclear; } done += round_to_approx_size(doc->len); @@ -1642,7 +1649,7 @@ Ldone:{ recover_pos += EVACUATION_SIZE; // safely cover the max write size if (recover_pos < header->write_pos && (recover_pos + EVACUATION_SIZE >= header->write_pos)) { Debug("cache_init", "Head Pos: %" PRIu64 ", Rec Pos: %" PRIu64 ", Wrapped:%d", header->write_pos, recover_pos, recover_wrapped); - Warning("no valid directory found while recovering '%s', clearing", hash_text); + Warning("no valid directory found while recovering '%s', clearing", hash_text.get()); goto Lclear; } @@ -1749,7 +1756,7 @@ Vol::handle_header_read(int event, void *data) (hf[0]->sync_serial >= hf[2]->sync_serial || hf[2]->sync_serial != hf[3]->sync_serial)) { SET_HANDLER(&Vol::handle_dir_read); if (is_debug_tag_set("cache_init")) - Note("using directory A for '%s'", hash_text); + Note("using directory A for '%s'", hash_text.get()); io.aiocb.aio_offset = skip; ink_assert(ink_aio_read(&io)); } @@ -1758,11 +1765,11 @@ Vol::handle_header_read(int event, void *data) SET_HANDLER(&Vol::handle_dir_read); if (is_debug_tag_set("cache_init")) - Note("using directory B for '%s'", hash_text); + Note("using directory B for '%s'", hash_text.get()); io.aiocb.aio_offset = skip + vol_dirlen(this); ink_assert(ink_aio_read(&io)); } else { - Note("no good directory, clearing '%s'", hash_text); + Note("no good directory, clearing '%s'", hash_text.get()); clear_dir(); delete init_info; init_info = 0; @@ -2467,7 +2474,7 @@ CacheVC::handleReadDone(int event, Event *e) if (!io.ok()) { Debug("cache_disk_error", "Read error on disk %s\n \ read range : [%" PRIu64 " - %" PRIu64 " bytes] [%" PRIu64 " - %" PRIu64 " blocks] \n", - vol->hash_text, (uint64_t)io.aiocb.aio_offset, (uint64_t)io.aiocb.aio_offset + io.aiocb.aio_nbytes, + vol->hash_text.get(), (uint64_t)io.aiocb.aio_offset, (uint64_t)io.aiocb.aio_offset + io.aiocb.aio_nbytes, (uint64_t)io.aiocb.aio_offset / 512, (uint64_t)(io.aiocb.aio_offset + io.aiocb.aio_nbytes) / 512); } goto Ldone; @@ -2494,7 +2501,7 @@ CacheVC::handleReadDone(int event, Event *e) doc = reinterpret_cast<Doc*>(buf->data()); // buf may be a new copy } else { Debug("cache_bc", "Upgrade of fragment failed - disk %s - doc id = %" PRIx64 ":%" PRIx64 "\n" - , vol->hash_text, read_key->slice64(0), read_key->slice64(1)); + , vol->hash_text.get(), read_key->slice64(0), read_key->slice64(1)); doc->magic = DOC_CORRUPT; // Should really trash the directory entry for this, as it's never going to work in the future. // Or does that happen later anyway? http://git-wip-us.apache.org/repos/asf/trafficserver/blob/83248169/iocore/cache/CacheDir.cc ---------------------------------------------------------------------- diff --git a/iocore/cache/CacheDir.cc b/iocore/cache/CacheDir.cc index 18b3be6..bd81325 100644 --- a/iocore/cache/CacheDir.cc +++ b/iocore/cache/CacheDir.cc @@ -1015,13 +1015,13 @@ sync_cache_dir_on_shutdown(void) Vol *d = gvol[i]; if (DISK_BAD(d->disk)) { - Debug("cache_dir_sync", "Dir %s: ignoring -- bad disk", d->hash_text); + Debug("cache_dir_sync", "Dir %s: ignoring -- bad disk", d->hash_text.get()); continue; } size_t dirlen = vol_dirlen(d); ink_assert(dirlen > 0); // make clang happy - if not > 0 the vol is seriously messed up if (!d->header->dirty && !d->dir_sync_in_progress) { - Debug("cache_dir_sync", "Dir %s: ignoring -- not dirty", d->hash_text); + Debug("cache_dir_sync", "Dir %s: ignoring -- not dirty", d->hash_text.get()); continue; } // recompute hit_evacuate_window @@ -1032,7 +1032,7 @@ sync_cache_dir_on_shutdown(void) // dont worry about the cachevc s in the agg queue // directories have not been inserted for these writes if (d->agg_buf_pos) { - Debug("cache_dir_sync", "Dir %s: flushing agg buffer first", d->hash_text); + Debug("cache_dir_sync", "Dir %s: flushing agg buffer first", d->hash_text.get()); // set write limit d->header->agg_pos = d->header->write_pos + d->agg_buf_pos; @@ -1054,7 +1054,7 @@ sync_cache_dir_on_shutdown(void) for (int i = 0; i < d->num_interim_vols; i++) { InterimCacheVol *sv = &(d->interim_vols[i]); if (sv->agg_buf_pos) { - Debug("cache_dir_sync", "Dir %s: flushing agg buffer first to interim", d->hash_text); + Debug("cache_dir_sync", "Dir %s: flushing agg buffer first to interim", d->hash_text.get()); sv->header->agg_pos = sv->header->write_pos + sv->agg_buf_pos; int r = pwrite(sv->fd, sv->agg_buffer, sv->agg_buf_pos, sv->header->write_pos); @@ -1096,7 +1096,7 @@ sync_cache_dir_on_shutdown(void) off_t start = d->skip + (B ? dirlen : 0); B = pwrite(d->fd, buf, dirlen, start); ink_assert(B == dirlen); - Debug("cache_dir_sync", "done syncing dir for vol %s", d->hash_text); + Debug("cache_dir_sync", "done syncing dir for vol %s", d->hash_text.get()); } Debug("cache_dir_sync", "sync done"); if (buf) @@ -1131,7 +1131,7 @@ Lrestart: if (event == AIO_EVENT_DONE) { // AIO Thread if (io.aio_result != (int64_t)io.aiocb.aio_nbytes) { - Warning("vol write error during directory sync '%s'", gvol[vol]->hash_text); + Warning("vol write error during directory sync '%s'", gvol[vol]->hash_text.get()); event = EVENT_NONE; goto Ldone; } @@ -1164,11 +1164,11 @@ Lrestart: The dirty bit it set in dir_insert, dir_overwrite and dir_delete_entry */ if (!d->header->dirty) { - Debug("cache_dir_sync", "Dir %s not dirty", d->hash_text); + Debug("cache_dir_sync", "Dir %s not dirty", d->hash_text.get()); goto Ldone; } if (d->is_io_in_progress() || d->agg_buf_pos) { - Debug("cache_dir_sync", "Dir %s: waiting for agg buffer", d->hash_text); + Debug("cache_dir_sync", "Dir %s: waiting for agg buffer", d->hash_text.get()); d->dir_sync_waiting = 1; if (!d->is_io_in_progress()) d->aggWrite(EVENT_IMMEDIATE, 0); @@ -1182,7 +1182,7 @@ Lrestart: #endif return EVENT_CONT; } - Debug("cache_dir_sync", "pos: %" PRIu64 " Dir %s dirty...syncing to disk", d->header->write_pos, d->hash_text); + Debug("cache_dir_sync", "pos: %" PRIu64 " Dir %s dirty...syncing to disk", d->header->write_pos, d->hash_text.get()); d->header->dirty = 0; if (buflen < dirlen) { if (buf) @@ -1276,7 +1276,7 @@ Vol::dir_check(bool /* fix ATS_UNUSED */) // TODO: we should eliminate this para free += dir_freelist_length(this, s); } int total = buckets * segments * DIR_DEPTH; - printf(" Directory for [%s]\n", hash_text); + printf(" Directory for [%s]\n", hash_text.get()); printf(" Bytes: %d\n", total * SIZEOF_DIR); printf(" Segments: %" PRIu64 "\n", (uint64_t)segments); printf(" Buckets: %" PRIu64 "\n", (uint64_t)buckets); http://git-wip-us.apache.org/repos/asf/trafficserver/blob/83248169/iocore/cache/CachePagesInternal.cc ---------------------------------------------------------------------- diff --git a/iocore/cache/CachePagesInternal.cc b/iocore/cache/CachePagesInternal.cc index 6221191..3fd8e9a 100644 --- a/iocore/cache/CachePagesInternal.cc +++ b/iocore/cache/CachePagesInternal.cc @@ -283,7 +283,7 @@ ShowCacheInternal::showVolVolumes(int event, Event * e) "<td>%u</td>" // sync serial "<td>%u</td>" // write serial "</tr>\n", - p->hash_text, + p->hash_text.get(), (uint64_t)((p->len - (p->start - p->skip)) / CACHE_BLOCK_SIZE), (uint64_t)(p->buckets * DIR_DEPTH * p->segments), (uint64_t)((p->header->write_pos - p->start) / CACHE_BLOCK_SIZE), http://git-wip-us.apache.org/repos/asf/trafficserver/blob/83248169/iocore/cache/CacheWrite.cc ---------------------------------------------------------------------- diff --git a/iocore/cache/CacheWrite.cc b/iocore/cache/CacheWrite.cc index a40f825..4050c88 100644 --- a/iocore/cache/CacheWrite.cc +++ b/iocore/cache/CacheWrite.cc @@ -319,7 +319,7 @@ Vol::aggWriteDone(int event, Event *e) header->write_pos += io.aiocb.aio_nbytes; ink_assert(header->write_pos >= start); DDebug("cache_agg", "Dir %s, Write: %" PRIu64 ", last Write: %" PRIu64 "\n", - hash_text, header->write_pos, header->last_write_pos); + hash_text.get(), header->write_pos, header->last_write_pos); ink_assert(header->write_pos == header->agg_pos); if (header->write_pos + EVACUATION_SIZE > scan_pos) periodic_scan(); @@ -330,7 +330,7 @@ Vol::aggWriteDone(int event, Event *e) // for fragments is this aggregation buffer Debug("cache_disk_error", "Write error on disk %s\n \ write range : [%" PRIu64 " - %" PRIu64 " bytes] [%" PRIu64 " - %" PRIu64 " blocks] \n", - hash_text, (uint64_t)io.aiocb.aio_offset, + hash_text.get(), (uint64_t)io.aiocb.aio_offset, (uint64_t)io.aiocb.aio_offset + io.aiocb.aio_nbytes, (uint64_t)io.aiocb.aio_offset / CACHE_BLOCK_SIZE, (uint64_t)(io.aiocb.aio_offset + io.aiocb.aio_nbytes) / CACHE_BLOCK_SIZE); http://git-wip-us.apache.org/repos/asf/trafficserver/blob/83248169/iocore/cache/I_Store.h ---------------------------------------------------------------------- diff --git a/iocore/cache/I_Store.h b/iocore/cache/I_Store.h index ba55657..303f880 100644 --- a/iocore/cache/I_Store.h +++ b/iocore/cache/I_Store.h @@ -43,20 +43,23 @@ // struct Span { - char *pathname; int64_t blocks; - int hw_sector_size; - bool file_pathname; // the pathname is a file - bool isRaw; int64_t offset; // used only if (file == true) + int hw_sector_size; int alignment; int disk_id; - int vol_num; - LINK(Span, link); - + int forced_volume_num; ///< Force span in to specific volume. private: bool is_mmapable_internal; public: + bool file_pathname; // the pathname is a file + bool isRaw; + // v- used as a magic location for copy constructor. + // we memcpy everything before this member and do explicit assignment for the rest. + ats_scoped_str pathname; + ats_scoped_str hash_seed_string; ///< Used to seed the stripe assignment hash. + SLINK(Span, link); + bool is_mmapable() { return is_mmapable_internal; } void set_mmapable(bool s) { is_mmapable_internal = s; } int64_t size() { return blocks * STORE_BLOCK_SIZE; } @@ -84,6 +87,7 @@ public: int write(int fd); int read(int fd); + /// Duplicate this span and all chained spans. Span *dup(); int64_t end() { return offset + blocks; } @@ -94,10 +98,33 @@ public: int64_t * offset, // for file, start offset (unsupported) char *buf, int buflen); // where to store the path + /// Set the hash seed string. + void hash_seed_string_set(char const* s); + /// Set the volume number. + void volume_number_set(int n); + Span() - : pathname(NULL), blocks(0), hw_sector_size(DEFAULT_HW_SECTOR_SIZE), file_pathname(false), - isRaw(true), offset(0), alignment(0), disk_id(0), is_mmapable_internal(false) + : blocks(0) + , offset(0) + , hw_sector_size(DEFAULT_HW_SECTOR_SIZE) + , alignment(0) + , disk_id(0) + , forced_volume_num(-1) + , is_mmapable_internal(false) + , file_pathname(false) + , isRaw(true) { } + + /// Copy constructor. + /// @internal Prior to this implementation handling the char* pointers was done manual + /// at every call site. We also need this because we have ats_scoped_str members. + Span(Span const& that) { + memcpy(this, &that, reinterpret_cast<intptr_t>(&(static_cast<Span*>(0)->pathname))); + if (that.pathname) pathname = ats_strdup(that.pathname); + if (that.hash_seed_string) hash_seed_string = ats_strdup(that.hash_seed_string); + link.next = NULL; + } + ~Span(); }; @@ -180,9 +207,10 @@ struct Store // const char *read_config(int fd = -1); int write_config_data(int fd); -private: - char const * const vol_str; - int getVolume(char* line); + + /// Additional configuration key values. + static char const VOLUME_KEY[]; + static char const HASH_SEED_KEY[]; }; extern Store theStore; http://git-wip-us.apache.org/repos/asf/trafficserver/blob/83248169/iocore/cache/P_CacheDisk.h ---------------------------------------------------------------------- diff --git a/iocore/cache/P_CacheDisk.h b/iocore/cache/P_CacheDisk.h index c704bdc..76c11ef 100644 --- a/iocore/cache/P_CacheDisk.h +++ b/iocore/cache/P_CacheDisk.h @@ -103,14 +103,17 @@ struct CacheDisk: public Continuation DiskVol *free_blocks; int num_errors; int cleared; - int forced_volume_num; // assuming zero is not a valid volume number + // Extra configuration values + int forced_volume_num; ///< Volume number for this disk. + ats_scoped_str hash_seed_string; ///< Base string for hash seed. + CacheDisk() : Continuation(new_ProxyMutex()), header(NULL), path(NULL), header_len(0), len(0), start(0), skip(0), num_usable_blocks(0), fd(-1), free_space(0), wasted_space(0), disk_vols(NULL), free_blocks(NULL), num_errors(0), cleared(0), - forced_volume_num(0) + forced_volume_num(-1) { } ~CacheDisk(); http://git-wip-us.apache.org/repos/asf/trafficserver/blob/83248169/iocore/cache/P_CacheVol.h ---------------------------------------------------------------------- diff --git a/iocore/cache/P_CacheVol.h b/iocore/cache/P_CacheVol.h index c14c200..d8748a3 100644 --- a/iocore/cache/P_CacheVol.h +++ b/iocore/cache/P_CacheVol.h @@ -414,7 +414,7 @@ void dir_clean_interimvol(InterimCacheVol *d); struct Vol: public Continuation { char *path; - char *hash_text; + ats_scoped_str hash_text; CryptoHash hash_id; int fd; http://git-wip-us.apache.org/repos/asf/trafficserver/blob/83248169/iocore/cache/Store.cc ---------------------------------------------------------------------- diff --git a/iocore/cache/Store.cc b/iocore/cache/Store.cc index f86c16b..be4ffc9 100644 --- a/iocore/cache/Store.cc +++ b/iocore/cache/Store.cc @@ -40,52 +40,18 @@ // Global Store theStore; -int -Store::getVolume(char* line) -{ - if (!line) { - return 0; - } - - int v = 0; - char* str = strstr(line, vol_str); - char* vol_start = str; - - if (!str) { - return 0; - } - - while (*str && !ParseRules::is_digit(*str)) - str++; - v = ink_atoi(str); - - while (*str && ParseRules::is_digit(*str)) - str++; - while(*str) { - *vol_start = *str; - vol_start++; - str++; - } - *vol_start = 0; - Debug("cache_init", "returning %d and '%s'", v, line); - - if (v < 0) { - return 0; - } - - return v; -} - - // // Store // + +char const Store::VOLUME_KEY[] = "volume"; +char const Store::HASH_SEED_KEY[] = "seed"; + Ptr<ProxyMutex> tmp_p; -Store::Store():n_disks(0), disk(NULL), +Store::Store():n_disks(0), disk(NULL) #if TS_USE_INTERIM_CACHE == 1 - n_interim_disks(0), interim_disk(NULL), + ,n_interim_disks(0), interim_disk(NULL) #endif - vol_str("volume=") { } @@ -234,6 +200,18 @@ Span::path(char *filename, int64_t * aoffset, char *buf, int buflen) } void +Span::hash_seed_string_set(char const* s) +{ + hash_seed_string = s ? ats_strdup(s) : NULL; +} + +void +Span::volume_number_set(int n) +{ + forced_volume_num = n; +} + +void Store::delete_all() { for (unsigned i = 0; i < n_disks; i++) { @@ -252,7 +230,6 @@ Store::~Store() Span::~Span() { - ats_free(pathname); if (link.next) delete link.next; } @@ -320,46 +297,57 @@ Store::read_config(int fd) } // For each line - char line[256]; - while (ink_file_fd_readline(fd, sizeof(line) - 1, line) > 0) { + char line[1024]; + int len; + while ((len = ink_file_fd_readline(fd, sizeof(line), line)) > 0) { + char const* path; + char const* seed = 0; // update lines - line[sizeof(line) - 1] = 0; - ln++; + ++ln; - // skip comments and blank lines + // Because the SimpleTokenizer is a bit too simple, we have to normalize whitespace. + for ( char *spot = line, *limit = line+len ; spot < limit ; ++spot ) + if (ParseRules::is_space(*spot)) *spot = ' '; // force whitespace to literal space. - if (*line == '#') - continue; - char *n = line; - n += strspn(n, " \t\n"); - if (!*n) - continue; + SimpleTokenizer tokens(line, ' ', SimpleTokenizer::OVERWRITE_INPUT_STRING); - int volume_id = getVolume(n); + // skip comments and blank lines + path = tokens.getNext(); + if (0 == path || '#' == path[0]) + continue; // parse - Debug("cache_init", "Store::read_config: \"%s\"", n); + Debug("cache_init", "Store::read_config: \"%s\"", path); - char *e = strpbrk(n, " \t\n"); - int len = e ? e - n : strlen(n); - (void) len; int64_t size = -1; - while (e && *e && !ParseRules::is_digit(*e)) - e++; - if (e && *e) { - if ((size = ink_atoi64(e)) <= 0) { - err = "error parsing size"; - goto Lfail; + int volume_num = -1; + char const* e; + while (0 != (e = tokens.getNext())) { + if (ParseRules::is_digit(*e)) { + if ((size = ink_atoi64(e)) <= 0) { + err = "error parsing size"; + goto Lfail; + } + } else if (0 == strncasecmp(HASH_SEED_KEY, e, sizeof(HASH_SEED_KEY)-1)) { + e += sizeof(HASH_SEED_KEY) - 1; + if ('=' == *e) ++e; + if (*e && !ParseRules::is_space(*e)) + seed = e; + } else if (0 == strncasecmp(VOLUME_KEY, e, sizeof(VOLUME_KEY)-1)) { + e += sizeof(VOLUME_KEY) - 1; + if ('=' == *e) ++e; + if (!*e || !ParseRules::is_digit(*e) || 0 >= (volume_num = ink_atoi(e))) { + err = "error parsing volume number"; + goto Lfail; + } } } - n[len] = 0; - char *pp = Layout::get()->relative(n); + char *pp = Layout::get()->relative(path); ns = new Span; - ns->vol_num = volume_id; - Debug("cache_init", "Store::read_config - ns = new Span; ns->init(\"%s\",%" PRId64 "), ns->vol_num=%d", - pp, size, ns->vol_num); + Debug("cache_init", "Store::read_config - ns = new Span; ns->init(\"%s\",%" PRId64 "), forced volume=%d%s%s", + pp, size, volume_num, seed ? " seed=" : "", seed ? seed : ""); if ((err = ns->init(pp, size))) { RecSignalWarning(REC_SIGNAL_SYSTEM_ERROR, "could not initialize storage \"%s\" [%s]", pp, err); Debug("cache_init", "Store::read_config - could not initialize storage \"%s\" [%s]", pp, err); @@ -370,6 +358,10 @@ Store::read_config(int fd) ats_free(pp); n_dsstore++; + // Set side values if present. + if (seed) ns->hash_seed_string_set(seed); + if (volume_num > 0) ns->volume_number_set(volume_num); + // new Span { Span *prev = cur; @@ -454,7 +446,7 @@ Store::write_config_data(int fd) for (unsigned i = 0; i < n_disks; i++) for (Span * sd = disk[i]; sd; sd = sd->link.next) { char buf[PATH_NAME_MAX + 64]; - snprintf(buf, sizeof(buf), "%s %" PRId64 "\n", sd->pathname, (int64_t) sd->blocks * (int64_t) STORE_BLOCK_SIZE); + snprintf(buf, sizeof(buf), "%s %" PRId64 "\n", sd->pathname.get(), (int64_t) sd->blocks * (int64_t) STORE_BLOCK_SIZE); if (ink_file_fd_writestring(fd, buf) == -1) return (-1); } @@ -574,7 +566,7 @@ Span::init(char *an, int64_t size) offset = 1; } - Debug("cache_init", "Span::init - %s hw_sector_size = %d size = %" PRId64 ", blocks = %" PRId64 ", disk_id = %d, file_pathname = %d", pathname, hw_sector_size, size, blocks, disk_id, file_pathname); + Debug("cache_init", "Span::init - %s hw_sector_size = %d size = %" PRId64 ", blocks = %" PRId64 ", disk_id = %d, file_pathname = %d", pathname.get(), hw_sector_size, size, blocks, disk_id, file_pathname); Lfail: return err; @@ -822,7 +814,7 @@ Span::init(char *filename, int64_t size) if (!file_pathname) if (size <= 0) return "When using directories for cache storage, you must specify a size\n"; - Debug("cache_init", "Span::init - mapped file \"%s\", %" PRId64 "", pathname, size); + Debug("cache_init", "Span::init - mapped file \"%s\", %" PRId64 "", pathname.get(), size); } blocks = size / STORE_BLOCK_SIZE; } @@ -860,10 +852,7 @@ try_alloc(Store & target, Span * source, unsigned int start_blocks, bool one_onl a = blocks; Span *d = new Span(*source); - d->pathname = ats_strdup(source->pathname); d->blocks = a; - d->file_pathname = source->file_pathname; - d->offset = source->offset; d->link.next = ds; if (d->file_pathname) @@ -935,7 +924,6 @@ Store::try_realloc(Store & s, Store & diff) goto Lfound; } else { Span *x = new Span(*d); - x->pathname = ats_strdup(x->pathname); // d will be the first vol d->blocks = sd->offset - d->offset; d->link.next = x; @@ -988,7 +976,7 @@ Span::write(int fd) { char buf[32]; - if (ink_file_fd_writestring(fd, (char *) (pathname ? pathname : ")")) == -1) + if (ink_file_fd_writestring(fd, (pathname ? pathname.get() : ")")) == -1) return (-1); if (ink_file_fd_writestring(fd, "\n") == -1) return (-1); @@ -1156,9 +1144,8 @@ Span * Span::dup() { Span *ds = new Span(*this); - ds->pathname = ats_strdup(pathname); - if (ds->link.next) - ds->link.next = ds->link.next->dup(); + if (this->link.next) + ds->link.next = this->link.next->dup(); return ds; } http://git-wip-us.apache.org/repos/asf/trafficserver/blob/83248169/lib/ts/ink_memory.h ---------------------------------------------------------------------- diff --git a/lib/ts/ink_memory.h b/lib/ts/ink_memory.h index 0b1ea5f..5859ed3 100644 --- a/lib/ts/ink_memory.h +++ b/lib/ts/ink_memory.h @@ -257,6 +257,12 @@ public: operator value_type () const { return _r; } + /// Explicit conversion to resource type. + /// @note Syntactic sugar for @c static_cast<value_type>(instance). Required when passing to var arg function + /// as automatic conversion won't be done. + value_type get() const { + return _r; + } /** Release resource from this container. After this call, the resource will @b not cleaned up when this container is destructed.
