This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/skywalking.git
The following commit(s) were added to refs/heads/master by this push:
new 641163cf1d BanyanDB: fix storage and query Tag autocomplete data
(#12432)
641163cf1d is described below
commit 641163cf1d02397c131d1f6c4139e520c1ed7bc6
Author: Wan Kai <[email protected]>
AuthorDate: Thu Jul 11 23:19:07 2024 +0800
BanyanDB: fix storage and query Tag autocomplete data (#12432)
---
docs/en/changes/changes.md | 1 +
.../oap/server/core/analysis/TimeBucket.java | 29 +++++++++++++++++
.../searchtag/TagAutocompleteDispatcher.java | 4 ++-
.../oap/server/core/query/DurationUtils.java | 30 ++++++++++++++++++
.../oap/server/core/query/input/Duration.java | 8 +++++
.../oap/server/core/analysis/TimeBucketTest.java | 11 +++++++
.../oap/server/core/query/DurationTest.java | 36 ++++++++++++++++++++++
.../measure/BanyanDBTagAutocompleteQueryDAO.java | 24 +++++++++------
.../common/dao/JDBCTagAutoCompleteQueryDAO.java | 13 ++++----
.../{autocomplete.yml => autocomplete-keys.yml} | 5 +--
.../{autocomplete.yml => autocomplete-value.yml} | 0
test/e2e-v2/cases/zipkin/zipkin-cases.yaml | 4 ++-
12 files changed, 143 insertions(+), 22 deletions(-)
diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md
index ccaf6f1023..f41d34f202 100644
--- a/docs/en/changes/changes.md
+++ b/docs/en/changes/changes.md
@@ -30,6 +30,7 @@
* Support tracing topology query for debugging.
* Fix expression of graph `Current QPS` in MySQL dashboard.
* Support tracing logs query for debugging.
+* BanyanDB: fix Tag autocomplete data storage and query.
#### UI
* Highlight search log keywords.
diff --git
a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/TimeBucket.java
b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/TimeBucket.java
index 8c32c5e29e..dae5271bac 100644
---
a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/TimeBucket.java
+++
b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/TimeBucket.java
@@ -163,4 +163,33 @@ public class TimeBucket {
throw new UnexpectedException("Unknown downsampling value.");
}
}
+
+ /**
+ * Retain the minute time bucket to day time bucket. The format of
timeBucket in minute format is "yyyyMMddHHmm", `HHmm` will be set to `0000`.
+ * minuteTimeBucket `202407112218` will be converted to `202407110000`.
+ * @param minuteTimeBucket minuteTimeBucket
+ * @return minute time bucket in day precision
+ */
+ public static long retainToDay4MinuteBucket(long minuteTimeBucket) {
+ if (isMinuteBucket(minuteTimeBucket)) {
+ return minuteTimeBucket / 10000 * 10000;
+ } else {
+ throw new UnexpectedException("Current time bucket is not a minute
time bucket");
+ }
+ }
+
+ /**
+ * Retain the minute time bucket to day time bucket. The format of
timeBucket in minute format is "yyyyMMddHHmm", `HHmm` will be set to `2359`.
+ * minuteTimeBucket `202407112218` will be converted to `202407112359`.
+ * @param minuteTimeBucket minuteTimeBucket
+ * @return minute time bucket in day precision
+ */
+ public static long retainToDayLastMin4MinuteBucket(long minuteTimeBucket) {
+ if (isMinuteBucket(minuteTimeBucket)) {
+ return minuteTimeBucket / 10000 * 10000 + 2359;
+ } else {
+ throw new UnexpectedException("Current time bucket is not a minute
time bucket");
+ }
+ }
}
+
diff --git
a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/searchtag/TagAutocompleteDispatcher.java
b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/searchtag/TagAutocompleteDispatcher.java
index 7b3e262619..3c73420f44 100644
---
a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/searchtag/TagAutocompleteDispatcher.java
+++
b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/searchtag/TagAutocompleteDispatcher.java
@@ -19,6 +19,7 @@
package org.apache.skywalking.oap.server.core.analysis.manual.searchtag;
import org.apache.skywalking.oap.server.core.analysis.SourceDispatcher;
+import org.apache.skywalking.oap.server.core.analysis.TimeBucket;
import
org.apache.skywalking.oap.server.core.analysis.worker.MetricsStreamProcessor;
import org.apache.skywalking.oap.server.core.source.TagAutocomplete;
@@ -30,7 +31,8 @@ public class TagAutocompleteDispatcher implements
SourceDispatcher<TagAutocomple
autocomplete.setTagKey(source.getTagKey());
autocomplete.setTagValue(source.getTagValue());
autocomplete.setTagType(source.getTagType().name());
- autocomplete.setTimeBucket(source.getTimeBucket());
+ // change the precision in Day for reduce the storage
+
autocomplete.setTimeBucket(TimeBucket.retainToDay4MinuteBucket(source.getTimeBucket()));
MetricsStreamProcessor.getInstance().in(autocomplete);
}
}
diff --git
a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/DurationUtils.java
b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/DurationUtils.java
index 02a6dda313..b65cb9a40c 100644
---
a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/DurationUtils.java
+++
b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/DurationUtils.java
@@ -85,6 +85,36 @@ public enum DurationUtils {
throw new UnexpectedException("Unsupported step " + step.name());
}
+ public long startTimeDurationToMinuteTimeBucket(Step step, String dateStr)
{
+ long minTimeBucket = convertToTimeBucket(step, dateStr);
+ switch (step) {
+ case DAY:
+ return minTimeBucket * 100 * 100;
+ case HOUR:
+ return minTimeBucket * 100;
+ case MINUTE:
+ return minTimeBucket;
+ case SECOND:
+ return minTimeBucket / 100;
+ }
+ throw new UnexpectedException("Unsupported step " + step.name());
+ }
+
+ public long endTimeDurationToMinuteTimeBucket(Step step, String dateStr) {
+ long minTimeBucket = convertToTimeBucket(step, dateStr);
+ switch (step) {
+ case DAY:
+ return (minTimeBucket * 100 + 23) * 100 + 59;
+ case HOUR:
+ return minTimeBucket * 100 + 59;
+ case MINUTE:
+ return minTimeBucket;
+ case SECOND:
+ return minTimeBucket / 100;
+ }
+ throw new UnexpectedException("Unsupported step " + step.name());
+ }
+
public List<PointOfTime> getDurationPoints(Step step, long
startTimeBucket, long endTimeBucket) {
DateTime dateTime = parseToDateTime(step, startTimeBucket);
diff --git
a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/Duration.java
b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/Duration.java
index eecbc3a596..9cda596c52 100644
---
a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/Duration.java
+++
b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/input/Duration.java
@@ -65,6 +65,14 @@ public class Duration {
return DurationUtils.INSTANCE.endTimeDurationToSecondTimeBucket(step,
end);
}
+ public long getStartTimeBucketInMin() {
+ return
DurationUtils.INSTANCE.startTimeDurationToMinuteTimeBucket(step, start);
+ }
+
+ public long getEndTimeBucketInMin() {
+ return DurationUtils.INSTANCE.endTimeDurationToMinuteTimeBucket(step,
end);
+ }
+
/**
* Assemble time point based on {@link #step} and {@link #start} / {@link
#end}
*/
diff --git
a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/TimeBucketTest.java
b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/TimeBucketTest.java
index 26115eac53..05ea84c218 100644
---
a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/TimeBucketTest.java
+++
b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/analysis/TimeBucketTest.java
@@ -19,6 +19,7 @@
package org.apache.skywalking.oap.server.core.analysis;
import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
@@ -64,4 +65,14 @@ public class TimeBucketTest {
}
Assertions.assertEquals(instance.getTimeInMillis(), timestamp);
}
+
+ @Test
+ public void testRetainToDay4MinuteBucket() {
+ Assertions.assertEquals(202407110000L,
TimeBucket.retainToDay4MinuteBucket(202407112218L));
+ }
+
+ @Test
+ public void testRetainToDayLastMin4MinuteBucket() {
+ Assertions.assertEquals(202407112359L,
TimeBucket.retainToDayLastMin4MinuteBucket(202407112218L));
+ }
}
diff --git
a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/query/DurationTest.java
b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/query/DurationTest.java
index a1fc53af0c..9b421c060c 100644
---
a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/query/DurationTest.java
+++
b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/query/DurationTest.java
@@ -91,4 +91,40 @@ public class DurationTest {
Assertions.assertTrue(Arrays.asList(20220910L, 20220911L, 20220912L)
.equals(pointOfTimes.stream().map(PointOfTime::getPoint).collect(Collectors.toList())));
}
+
+ @Test
+ public void testStartTimeDurationToMinuteTimeBucket() {
+ Assertions.assertEquals(
+ 202209080000L,
DurationUtils.INSTANCE.startTimeDurationToMinuteTimeBucket(Step.DAY,
"2022-09-08"));
+ Assertions.assertEquals(
+ 202209081000L,
DurationUtils.INSTANCE.startTimeDurationToMinuteTimeBucket(Step.HOUR,
"2022-09-08 10"));
+ Assertions.assertEquals(
+ 202209081010L,
DurationUtils.INSTANCE.startTimeDurationToMinuteTimeBucket(Step.MINUTE,
"2022-09-08 1010"));
+ Assertions.assertEquals(
+ 202209081010L,
DurationUtils.INSTANCE.startTimeDurationToMinuteTimeBucket(Step.SECOND,
"2022-09-08 101010"));
+ try {
+
DurationUtils.INSTANCE.startTimeDurationToMinuteTimeBucket(Step.HOUR,
"2022-09-08 30");
+ Assertions.fail("Should throw IllegalArgumentException");
+ } catch (IllegalArgumentException e) {
+ Assertions.assertTrue(true);
+ }
+ }
+
+ @Test
+ public void testEndTimeDurationToMinuteTimeBucket() {
+ Assertions.assertEquals(
+ 202209082359L,
DurationUtils.INSTANCE.endTimeDurationToMinuteTimeBucket(Step.DAY,
"2022-09-08"));
+ Assertions.assertEquals(
+ 202209081059L,
DurationUtils.INSTANCE.endTimeDurationToMinuteTimeBucket(Step.HOUR, "2022-09-08
10"));
+ Assertions.assertEquals(
+ 202209081010L,
DurationUtils.INSTANCE.endTimeDurationToMinuteTimeBucket(Step.MINUTE,
"2022-09-08 1010"));
+ Assertions.assertEquals(
+ 202209081010L,
DurationUtils.INSTANCE.endTimeDurationToMinuteTimeBucket(Step.SECOND,
"2022-09-08 101010"));
+ try {
+
DurationUtils.INSTANCE.endTimeDurationToMinuteTimeBucket(Step.HOUR, "2022-09-08
30");
+ Assertions.fail("Should throw IllegalArgumentException");
+ } catch (IllegalArgumentException e) {
+ Assertions.assertTrue(true);
+ }
+ }
}
diff --git
a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBTagAutocompleteQueryDAO.java
b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBTagAutocompleteQueryDAO.java
index 9b8e2fa0e7..026925f645 100644
---
a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBTagAutocompleteQueryDAO.java
+++
b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBTagAutocompleteQueryDAO.java
@@ -54,12 +54,16 @@ public class BanyanDBTagAutocompleteQueryDAO extends
AbstractBanyanDBDAO impleme
@Override
public Set<String> queryTagAutocompleteKeys(TagType tagType, int limit,
Duration duration) throws IOException {
MetadataRegistry.Schema schema =
MetadataRegistry.INSTANCE.findMetadata(TagAutocompleteData.INDEX_NAME,
DownSampling.Minute);
- long startTB = 0;
- long endTB = 0;
+ long startMinTB = 0;
+ long endMinTB = 0;
if (nonNull(duration)) {
- startTB =
TimeBucket.getMinuteTimeBucket(duration.getStartTimestamp());
- endTB = TimeBucket.getMinuteTimeBucket(duration.getEndTimestamp());
+ startMinTB = duration.getStartTimeBucketInMin();
+ endMinTB = duration.getEndTimeBucketInMin();
}
+
+ long startTB = TimeBucket.retainToDay4MinuteBucket(startMinTB);
+ long endTB = TimeBucket.retainToDayLastMin4MinuteBucket(endMinTB);
+
TimestampRange range = null;
if (startTB > 0 && endTB > 0) {
range = new TimestampRange(TimeBucket.getTimestamp(startTB),
TimeBucket.getTimestamp(endTB));
@@ -91,15 +95,15 @@ public class BanyanDBTagAutocompleteQueryDAO extends
AbstractBanyanDBDAO impleme
@Override
public Set<String> queryTagAutocompleteValues(TagType tagType, String
tagKey, int limit, Duration duration) throws IOException {
MetadataRegistry.Schema schema =
MetadataRegistry.INSTANCE.findMetadata(TagAutocompleteData.INDEX_NAME,
DownSampling.Minute);
- long startSecondTB = 0;
- long endSecondTB = 0;
+ long startMinTB = 0;
+ long endMinTB = 0;
if (nonNull(duration)) {
- startSecondTB = duration.getStartTimeBucketInSec();
- endSecondTB = duration.getEndTimeBucketInSec();
+ startMinTB = duration.getStartTimeBucketInMin();
+ endMinTB = duration.getEndTimeBucketInMin();
}
- long startTB = startSecondTB / 1000000 * 10000;
- long endTB = endSecondTB / 1000000 * 10000 + 2359;
+ long startTB = TimeBucket.retainToDay4MinuteBucket(startMinTB);
+ long endTB = TimeBucket.retainToDayLastMin4MinuteBucket(endMinTB);
TimestampRange range = null;
if (startTB > 0 && endTB > 0) {
diff --git
a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTagAutoCompleteQueryDAO.java
b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTagAutoCompleteQueryDAO.java
index 6564e26bdf..f4ddca69f8 100644
---
a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTagAutoCompleteQueryDAO.java
+++
b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTagAutoCompleteQueryDAO.java
@@ -20,6 +20,7 @@ package
org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
+import org.apache.skywalking.oap.server.core.analysis.TimeBucket;
import
org.apache.skywalking.oap.server.core.analysis.manual.searchtag.TagAutocompleteData;
import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.TagType;
import org.apache.skywalking.oap.server.core.query.input.Duration;
@@ -125,15 +126,15 @@ public class JDBCTagAutoCompleteQueryDAO implements
ITagAutoCompleteQueryDAO {
final Duration duration,
final StringBuilder sql,
final List<Object> condition) {
- long startSecondTB = 0;
- long endSecondTB = 0;
+ long startMinTB = 0;
+ long endMinTB = 0;
if (nonNull(duration)) {
- startSecondTB = duration.getStartTimeBucketInSec();
- endSecondTB = duration.getEndTimeBucketInSec();
+ startMinTB = duration.getStartTimeBucketInMin();
+ endMinTB = duration.getEndTimeBucketInMin();
}
- long startTB = startSecondTB / 1000000 * 10000;
- long endTB = endSecondTB / 1000000 * 10000 + 2359;
+ long startTB = TimeBucket.retainToDay4MinuteBucket(startMinTB);
+ long endTB = TimeBucket.retainToDayLastMin4MinuteBucket(endMinTB);
sql.append(" and ");
sql.append(TagAutocompleteData.TAG_TYPE).append(" = ?");
diff --git a/test/e2e-v2/cases/zipkin/expected/autocomplete.yml
b/test/e2e-v2/cases/zipkin/expected/autocomplete-keys.yml
similarity index 97%
copy from test/e2e-v2/cases/zipkin/expected/autocomplete.yml
copy to test/e2e-v2/cases/zipkin/expected/autocomplete-keys.yml
index 0b6ed23a7b..29ccb631b1 100644
--- a/test/e2e-v2/cases/zipkin/expected/autocomplete.yml
+++ b/test/e2e-v2/cases/zipkin/expected/autocomplete-keys.yml
@@ -13,7 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-[
- "GET",
- "POST"
-]
+["http.method"]
diff --git a/test/e2e-v2/cases/zipkin/expected/autocomplete.yml
b/test/e2e-v2/cases/zipkin/expected/autocomplete-value.yml
similarity index 100%
rename from test/e2e-v2/cases/zipkin/expected/autocomplete.yml
rename to test/e2e-v2/cases/zipkin/expected/autocomplete-value.yml
diff --git a/test/e2e-v2/cases/zipkin/zipkin-cases.yaml
b/test/e2e-v2/cases/zipkin/zipkin-cases.yaml
index 995c7e2b3a..2a2c263d8a 100644
--- a/test/e2e-v2/cases/zipkin/zipkin-cases.yaml
+++ b/test/e2e-v2/cases/zipkin/zipkin-cases.yaml
@@ -27,5 +27,7 @@ cases:
- query: curl
http://${oap_host}:${oap_9412}/zipkin/api/v2/traces?serviceName=frontend&remoteServiceName=backend&spanName=get&annotationQuery=wr&limit=1
expected: expected/traces.yml
# autocomplete
+ - query: curl http://${oap_host}:${oap_9412}/zipkin/api/v2/autocompleteKeys
+ expected: expected/autocomplete-keys.yml
- query: curl
http://${oap_host}:${oap_9412}/zipkin/api/v2/autocompleteValues?key=http.method
- expected: expected/autocomplete.yml
+ expected: expected/autocomplete-value.yml
