Edenhill has uploaded a new change for review.
https://gerrit.wikimedia.org/r/79748
Change subject: When reading offline VSL files (-r ..) make a copy of each
matched tags data since the data is volatile.
......................................................................
When reading offline VSL files (-r ..) make a copy of each matched tags data
since the data is volatile.
This is controlled manually by the new 'log.data.copy = true|false' config
property
which defaults to true.
Increases the scratch pad per logline from 512 to 2K.
Change-Id: I8935d10eae2496a2096a44f9f10d89585a48ae8c
---
M config.c
M varnishkafka.c
M varnishkafka.conf.example
M varnishkafka.h
4 files changed, 38 insertions(+), 4 deletions(-)
git pull
ssh://gerrit.wikimedia.org:29418/operations/software/varnish/varnishkafka
refs/changes/48/79748/1
diff --git a/config.c b/config.c
index feb5479..9ad9648 100644
--- a/config.c
+++ b/config.c
@@ -169,8 +169,9 @@
"try \"stdout\" or \"kafka\"", val);
return -1;
}
-
- } else if (!strncmp(name, "varnish.arg.", strlen("varnish.arg."))) {
+ } else if (!strcmp(name, "log.data.copy"))
+ conf.datacopy = conf_tof(val);
+ else if (!strncmp(name, "varnish.arg.", strlen("varnish.arg."))) {
const char *t = name + strlen("varnish.arg.");
int r = 0;
if (*t == '-')
diff --git a/varnishkafka.c b/varnishkafka.c
index 368a361..1fafeae 100644
--- a/varnishkafka.c
+++ b/varnishkafka.c
@@ -306,6 +306,24 @@
return len;
}
+/**
+ * Same as scratch_write0() but calls match_assign0() directly, thus
+ * not supporting escaping.
+ */
+static inline int scratch_write0 (const struct tag *tag, struct logline *lp,
+ const char *src, int len) {
+ char *dst;
+
+ if (unlikely((dst = scratch_alloc(tag, lp, len)) == NULL))
+ return -1;
+
+ memcpy(dst, src, len);
+
+ match_assign0(tag, lp, dst, len);
+
+ return len;
+}
+
/**
* Writes 'src' of 'len' bytes to scratch buffer, escaping
@@ -444,7 +462,10 @@
scratch_write_escaped(tag, lp, ptr, len);
} else {
- match_assign0(tag, lp, ptr, len);
+ if (conf.datacopy) /* copy volatile data */
+ scratch_write0(tag, lp, ptr, len);
+ else /* point to persistent data */
+ match_assign0(tag, lp, ptr, len);
}
}
diff --git a/varnishkafka.conf.example b/varnishkafka.conf.example
index a28bdd0..c8171b0 100644
--- a/varnishkafka.conf.example
+++ b/varnishkafka.conf.example
@@ -86,6 +86,17 @@
output = kafka
+# EXPERIMENTAL
+# Indicates if the log tag data read from VSL files should be copied instantly
+# when read (true). If this is set to false the data is assumed to be
+# persistent (for the duration of collecting and formatting a single request)
+# and no copies will be made, thus improving performance.
+#
+# NOTE:
+# Must be set to true for offline files (-r file..) due to the way
+# libvarnishapi reads its data.
+log.data.copy = true
+
# Start for sequence number (%n)
# Either a number, or the string "time" which will set it to the current
diff --git a/varnishkafka.h b/varnishkafka.h
index f118b84..7a3ce13 100644
--- a/varnishkafka.h
+++ b/varnishkafka.h
@@ -68,7 +68,7 @@
/* Scratch pad */
int sof;
- char scratch[512]; /* Must be at end of struct */
+ char scratch[2048]; /* Must be at end of struct */
};
@@ -128,6 +128,7 @@
uint64_t sequence_number;
+ int datacopy;
enum {
VK_FORMAT_STRING,
VK_FORMAT_JSON,
--
To view, visit https://gerrit.wikimedia.org/r/79748
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I8935d10eae2496a2096a44f9f10d89585a48ae8c
Gerrit-PatchSet: 1
Gerrit-Project: operations/software/varnish/varnishkafka
Gerrit-Branch: master
Gerrit-Owner: Edenhill <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
