blaginin commented on code in PR #26021:
URL: https://github.com/apache/datafusion/pull/26021#discussion_r4618887311
##########
datafusion/functions/src/regex/mod.rs:
##########
@@ -22,6 +25,87 @@ pub mod regexplike;
pub mod regexpmatch;
pub mod regexpreplace;
+/// Patterns are addressed by index rather than by reference so that `last` can
+/// memoize the previous row's pattern without holding a borrow of `indices`
+/// across rows. Repeated patterns yield the same key on consecutive rows, so
+/// the memo avoids hashing in that case.
+///
+/// Criterion benchmarks for each function using this cache are in
+/// `datafusion/functions/benches/regex_expressions/cache.rs`.
+pub(crate) struct RegexCache<'a> {
+ function_name: &'static str,
+ compiled: Vec<Regex>,
+ indices: HashMap<RegexKey<'a>, usize>,
+ last: Option<(RegexKey<'a>, usize)>,
+}
+
+#[derive(Clone, Copy, PartialEq, Eq)]
+struct RegexKey<'a> {
+ regex: &'a str,
+ flags: Option<&'a str>,
+}
+
+impl Hash for RegexKey<'_> {
+ fn hash<H: Hasher>(&self, state: &mut H) {
+ self.regex.hash(state);
+ // String hashing includes a delimiter, so absent flags need no
+ // additional discriminator. Equality still distinguishes every key.
+ if let Some(flags) = self.flags {
+ flags.hash(state);
+ }
+ }
+}
+
+impl<'a> RegexCache<'a> {
+ fn new(function_name: &'static str) -> Self {
+ Self {
+ function_name,
+ compiled: Vec::new(),
+ indices: HashMap::new(),
+ last: None,
+ }
+ }
+
+ #[inline]
+ pub(crate) fn get_or_compile(
+ &mut self,
+ regex: &'a str,
+ flags: Option<&'a str>,
+ ) -> Result<&Regex> {
+ let key = RegexKey { regex, flags };
Review Comment:
this will probs slow down queries that used to work before through stripping
flags
```
SELECT SUM(LENGTH(regexp_replace(
'aaaaaaaaaaaaaaaaaaaa',
'\w{20}',
'X',
'i' || repeat('g', n)
)))
FROM generate_series(1, 256) AS t(n);
```
if you run this in latest release, it'll be much faster than on this branch
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]