Author: adrian.chadd
Date: Mon Jul 6 09:41:51 2009
New Revision: 14142
Added:
playpen/LUSCA_HEAD_storework/app/ufs_rebuild/Makefile.am
playpen/LUSCA_HEAD_storework/app/ufs_rebuild/ufs_build_dir.c
playpen/LUSCA_HEAD_storework/app/ufs_rebuild/ufs_build_log.c
playpen/LUSCA_HEAD_storework/app/ufs_rebuild/ufs_rebuild.c
Log:
Start building a "generate log from storedir somehow" tool.
The idea here (and this obviously doesn't yet build!) is to have one tool
that generates a binary swaplog on stdout. This tool will eventually be able
to choose between the swapdir contents, or the swaplog, or be forced to
pick one.
(Sidenote - i'd eventually prefer that all of the logic in this tool which
is reading the dirs and the files be moved into a library and reused between
a variety of smaller tools too. I'll think about that later.)
Added: playpen/LUSCA_HEAD_storework/app/ufs_rebuild/Makefile.am
==============================================================================
--- (empty file)
+++ playpen/LUSCA_HEAD_storework/app/ufs_rebuild/Makefile.am Mon Jul 6
09:41:51 2009
@@ -0,0 +1,28 @@
+#
+# Makefile for the Squid Object Cache server
+#
+# $Id: Makefile.am 11393 2006-09-22 09:08:46Z hno $
+#
+# Uncomment and customize the following to suit your needs:
+#
+
+
+AUTOMAKE_OPTIONS = subdir-objects
+
+INCLUDES = -I. -I$(top_builddir)
+
+TESTS=$(check_PROGRAMS)
+check_PROGRAMS=
+
+SUBDIRS =
+
+libexec_PROGRAMS = ufs_rebuild
+
+ufs_rebuild_SOURCES = ufs_rebuild.c ufs_build_log.c ufs_build_dir.c
+
+LDADD = -L../../lib -L../../libsqdebug -L../../libcore -L../../libmem
-L../../libsqstore -lcb -lsqdebug -lcore -lm -lsqstore -lmiscutil
+
+$(OBJS): $(top_srcdir)/include/version.h ../include/autoconf.h
+
+DISTCLEANFILES =
+
Added: playpen/LUSCA_HEAD_storework/app/ufs_rebuild/ufs_build_dir.c
==============================================================================
--- (empty file)
+++ playpen/LUSCA_HEAD_storework/app/ufs_rebuild/ufs_build_dir.c Mon Jul
6
09:41:51 2009
@@ -0,0 +1,267 @@
+
+/*
+ * This program provides the "rebuild" logic for a UFS spool.
+ *
+ * It will scan a UFS style directory for valid looking swap files
+ * and spit out a new style swap log to STDOUT.
+ *
+ * Adrian Chadd <[email protected]>
+ */
+
+#include "config.h"
+
+#if HAVE_INTTYPES_H
+#include <inttypes.h>
+#endif
+#if HAVE_STDIO_H
+#include <stdio.h>
+#endif
+#if HAVE_STDLIB_H
+#include <stdlib.h>
+#endif
+#if HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#if HAVE_STRING_H
+#include <string.h>
+#endif
+#if HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#if HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+
+#include <sys/stat.h>
+#include <dirent.h>
+
+#include "../include/util.h"
+
+#include "../libcore/kb.h"
+#include "../libcore/varargs.h"
+#include "../libcore/mem.h"
+#include "../libcore/tools.h"
+
+#include "../libsqdebug/debug.h"
+
+#include "../libsqtlv/tlv.h"
+
+#define SQUID_MD5_DIGEST_LENGTH 16
+
+#include "../libsqstore/store_mgr.h"
+#include "../libsqstore/store_meta.h"
+#include "../libsqstore/store_log.h"
+#include "../libsqstore/store_file_ufs.h"
+
+#define BUFSIZE 1024
+
+/* normally in libiapp .. */
+int shutting_down = 0;
+
+struct _rebuild_entry {
+ storeMetaIndexNew mi;
+ char *md5_key;
+ char *url;
+ char *storeurl;
+ squid_file_sz file_size; /* swap file size - object size
+ metadata */
+ int hdr_size; /* metadata size */
+ int swap_filen;
+};
+typedef struct _rebuild_entry rebuild_entry_t;
+
+void
+rebuild_entry_done(rebuild_entry_t *re)
+{
+ safe_free(re->md5_key);
+ safe_free(re->url);
+ safe_free(re->storeurl);
+}
+
+void
+rebuild_entry_init(rebuild_entry_t *re)
+{
+ bzero(re, sizeof(*re));
+ re->hdr_size = -1;
+ re->file_size = -1;
+ re->swap_filen = -1;
+}
+
+static int
+parse_header(char *buf, int len, rebuild_entry_t *re)
+{
+ tlv *t, *tlv_list;
+ int bl = len;
+ int parsed = 0;
+
+ tlv_list = tlv_unpack(buf, &bl, STORE_META_END + 10);
+ if (tlv_list == NULL) {
+ return -1;
+ }
+
+ re->hdr_size = bl;
+
+ for (t = tlv_list; t; t = t->next) {
+ switch (t->type) {
+ case STORE_META_URL:
+ debug(47, 5) (" STORE_META_URL\n");
+ /* XXX Is this OK? Is the URL guaranteed to be \0 terminated? */
+ re->url = xstrdup( (char *) t->value );
+ parsed++;
+ break;
+ case STORE_META_KEY_MD5:
+ debug(47, 5) (" STORE_META_KEY_MD5\n");
+ /* XXX should double-check key length? */
+ re->md5_key = xmalloc(SQUID_MD5_DIGEST_LENGTH);
+ memcpy(re->md5_key, t->value, SQUID_MD5_DIGEST_LENGTH);
+ parsed++;
+ break;
+ case STORE_META_STD_LFS:
+ debug(47, 5) (" STORE_META_STD_LFS\n");
+ /* XXX should double-check lengths match? */
+ memcpy(&re->mi, t->value, sizeof(re->mi));
+ parsed++;
+ break;
+ case STORE_META_OBJSIZE:
+ debug(47, 5) (" STORE_META_OBJSIZE\n");
+ /* XXX is this typecast'ed to the right "size" on all platforms
? */
+ break;
+ default:
+ break;
+ }
+ }
+ assert(tlv_list != NULL);
+ tlv_free(tlv_list);
+ return (parsed > 1);
+}
+
+int
+read_file(const char *path, rebuild_entry_t *re)
+{
+ int fd;
+ char buf[BUFSIZE];
+ int len;
+ struct stat sb;
+
+ debug(47, 3) ("read_file: %s\n", path);
+ fd = open(path, O_RDONLY);
+ if (fd < 0) {
+ perror("open");
+ return -1;
+ }
+
+ /* We need the entire file size */
+ if (fstat(fd, &sb) < 0) {
+ perror("fstat");
+ return -1;
+ }
+
+ len = read(fd, buf, BUFSIZE);
+ debug(47, 3) ("read_file: FILE: %s\n", path);
+
+ if (! parse_header(buf, len, re)) {
+ close(fd);
+ return -1;
+ }
+ re->file_size = sb.st_size;
+ close(fd);
+ return 1;
+}
+
+int
+write_swaplog_entry(rebuild_entry_t *re)
+{
+ storeSwapLogData sd;
+
+ sd.op = SWAP_LOG_ADD;
+ sd.swap_filen = re->swap_filen;
+ sd.timestamp = re->mi.timestamp;
+ sd.lastref = re->mi.lastref;
+ sd.expires = re->mi.expires;
+ sd.lastmod = re->mi.lastmod;
+ sd.swap_file_sz = re->file_size;
+ sd.refcount = re->mi.refcount;
+ sd.flags = re->mi.flags;
+
+ memcpy(&sd.key, re->md5_key, sizeof(sd.key));
+ if (! write(1, &sd, sizeof(sd)))
+ return -1;
+
+ return 1;
+}
+
+void
+read_dir(store_ufs_dir_t *sd)
+{
+ DIR *d;
+ struct dirent *de;
+ char path[SQUID_MAXPATHLEN];
+ char dir[SQUID_MAXPATHLEN];
+ rebuild_entry_t re;
+ int fn;
+ int i, j;
+
+ getCurrentTime();
+ for (i = 0; i < store_ufs_l1(sd); i++) {
+ for (j = 0; j < store_ufs_l2(sd); j++) {
+ (void) store_ufs_createDir(sd, i, j, dir);
+ getCurrentTime();
+ debug(47, 1) ("read_dir: opening dir %s\n", dir);
+ d = opendir(dir);
+ if (! d) {
+ perror("opendir");
+ continue;
+ }
+
+ while ( (de = readdir(d)) != NULL) {
+ if (de->d_name[0] == '.')
+ continue;
+ getCurrentTime();
+
+ /* Verify that the given filename belongs in
the given directory */
+ if (sscanf(de->d_name, "%x", &fn) != 1) {
+ debug(47, 1) ("read_dir: invalid %s\n",
de->d_name);
+ continue;
+ }
+ if (! store_ufs_filenum_correct_dir(sd, fn, i,
j)) {
+ debug(47, 1) ("read_dir: %s does not
belong in %d/%d\n", de->d_name,
i, j);
+ continue;
+ }
+
+ snprintf(path, sizeof(path) - 1, "%s/%s", dir,
de->d_name);
+ debug(47, 3) ("read_dir: opening %s\n", path);
+
+ rebuild_entry_init(&re);
+ (void) read_file(path, &re);
+ re.swap_filen = fn;
+ (void) write_swaplog_entry(&re);
+ rebuild_entry_done(&re);
+
+ }
+ closedir(d);
+ }
+ }
+}
+
+int
+main(int argc, char *argv[])
+{
+ /* Setup the debugging library */
+ _db_init("ALL,1");
+ _db_set_stderr_debug(1);
+ store_ufs_dir_t store_ufs_info;
+
+ if (argc < 5) {
+ printf("Usage: %s <store path> <l1> <l2> <path to swapfile>\n",
argv[0]);
+ exit(1);
+ }
+
+ store_ufs_init(&store_ufs_info, argv[1], atoi(argv[2]), atoi(argv[3]),
argv[4]);
+
+ /* Output swap header to stdout */
+ (void) storeSwapLogPrintHeader(1);
+
+ read_dir(&store_ufs_info);
+ store_ufs_done(&store_ufs_info);
+
+ return 0;
+}
Added: playpen/LUSCA_HEAD_storework/app/ufs_rebuild/ufs_build_log.c
==============================================================================
--- (empty file)
+++ playpen/LUSCA_HEAD_storework/app/ufs_rebuild/ufs_build_log.c Mon Jul
6
09:41:51 2009
@@ -0,0 +1,172 @@
+#include "config.h"
+
+#if HAVE_INTTYPES_H
+#include <inttypes.h>
+#endif
+#if HAVE_STDIO_H
+#include <stdio.h>
+#endif
+#if HAVE_STDLIB_H
+#include <stdlib.h>
+#endif
+#if HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#if HAVE_STRING_H
+#include <string.h>
+#endif
+#if HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#if HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+
+#include <dirent.h>
+
+#include "../include/util.h"
+
+#include "../libcore/kb.h"
+#include "../libcore/varargs.h"
+#include "../libcore/mem.h"
+#include "../libcore/tools.h"
+
+#include "../libsqdebug/debug.h"
+
+#include "../libsqtlv/tlv.h"
+
+#define SQUID_MD5_DIGEST_LENGTH 16
+
+#include "../libsqstore/store_mgr.h"
+#include "../libsqstore/store_log.h"
+
+/* normally in libiapp .. */
+int shutting_down = 0;
+
+int num_objects = 0;
+int num_valid_objects = 0;
+int num_invalid_objects = 0;
+
+int
+read_entry(FILE *fp, int version)
+{
+ int r;
+ char buf[128];
+ storeSwapLogData sd;
+ size_t s = -1;
+
+ if (version == 1) {
+ s = sizeof(storeSwapLogData);
+ } else {
+ s = sizeof(storeSwapLogDataOld);
+ }
+
+ r = fread(buf, s, 1, fp);
+ if (r != 1) {
+ debug(1, 2) ("fread: returned %d (ferror %d)\n", r, ferror(fp));
+ return -1;
+ }
+ num_objects++;
+
+ /* Decode the entry */
+ if (version == 1) {
+ memcpy(&sd, buf, sizeof(sd));
+ } else {
+ (void) storeSwapLogUpgradeEntry(&sd, (storeSwapLogDataOld *)
buf);
+ }
+
+ /* is it an ADD/DEL? Good. If not - count error and continue */
+ if (sd.op == SWAP_LOG_ADD || sd.op == SWAP_LOG_DEL) {
+ num_valid_objects++;
+ write(1, &sd, sizeof(sd));
+ } else {
+ debug(1, 5) ("error! Got swaplog entry op %d?!\n", sd.op);
+ num_invalid_objects++;
+ }
+
+ return 1;
+}
+
+void
+read_file(const char *swapfile)
+{
+ FILE *fp;
+ storeSwapLogHeader hdr;
+ int r;
+ int version = -1; /* -1 = not set, 0 = old, 1 = new */
+
+ fp = fopen(swapfile, "r");
+ if (! fp) {
+ perror("fopen");
+ return;
+ }
+
+ /* Read an entry - see if its a swap header! */
+ r = fread(&hdr, sizeof(hdr), 1, fp);
+ if (r != 1) {
+ perror("fread");
+ fclose(fp);
+ return;
+ }
+
+ /*
+ * If hdr is a swap log version then we can determine whether
+ * if its old or new. If it isn't then we can't determine what
+ * type it is, so assume its the "default" size.
+ */
+ if (hdr.op == SWAP_LOG_VERSION) {
+ if (fseek(fp, hdr.record_size, SEEK_SET) != 0) {
+ perror("fseek");
+ fclose(fp);
+ return;
+ }
+ if (hdr.version == 1 && hdr.record_size ==
sizeof(storeSwapLogData)) {
+ version = 1;
+ } else if (hdr.version == 1 && hdr.record_size ==
sizeof(storeSwapLogDataOld)) {
+ version = 0;
+ } else {
+ debug(1, 1) ("Unsupported swap.state version %d size
%d\n",
hdr.version, hdr.record_size);
+ fclose(fp);
+ return;
+ }
+ } else {
+ /* Otherwise, default based on the compile time size comparison
*/
+ rewind(fp);
+#if SIZEOF_SQUID_FILE_SZ == SIZEOF_SIZE_T
+ version = 1;
+#else
+ version = 0;
+#endif
+ }
+
+ /* begin echo'ing the log info */
+ storeSwapLogPrintHeader(1); /* to stdout */
+
+ /* Now - loop over until eof or error */
+ while (! feof(fp)) {
+ if (! read_entry(fp, version)) {
+ break;
+ }
+ }
+
+ fclose(fp);
+}
+
+int
+main(int argc, char *argv[])
+{
+ /* Setup the debugging library */
+ _db_init("ALL,1");
+ _db_set_stderr_debug(1);
+
+ if (argc < 3) {
+ printf("Usage: %s -f <path to swaplog>\n", argv[0]);
+ exit(1);
+ }
+
+ read_file(argv[2]);
+
+ debug(1, 1) ("%s: Read %d objects\n", argv[2], num_objects);
+
+ return 0;
+}
Added: playpen/LUSCA_HEAD_storework/app/ufs_rebuild/ufs_rebuild.c
==============================================================================
--- (empty file)
+++ playpen/LUSCA_HEAD_storework/app/ufs_rebuild/ufs_rebuild.c Mon Jul 6
09:41:51 2009
@@ -0,0 +1,10 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+
+int
+main(int argc, char *argv[])
+{
+ exit(0);
+}
--~--~---------~--~----~------------~-------~--~----~
You received this message because you are subscribed to the Google Groups
"lusca-commit" group.
To post to this group, send email to [email protected]
To unsubscribe from this group, send email to
[email protected]
For more options, visit this group at
http://groups.google.com/group/lusca-commit?hl=en
-~----------~----~----~----~------~----~------~--~---
