github-advanced-security[bot] commented on code in PR #15817:
URL: https://github.com/apache/druid/pull/15817#discussion_r1564813281
##########
indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskParallelRunTest.java:
##########
@@ -824,7 +854,9 @@
.granularitySpec(new
ClientCompactionTaskGranularitySpec(Granularities.MINUTE, null, null))
.build();
- final Set<DataSegment> compactedSegments = runTask(compactionTask);
+ final DataSegmentWithSchemas dataSegmentWithSchemas =
runTask(compactionTask);
+ verifySchema(dataSegmentWithSchemas);
+ final Set<DataSegment> compactedSegments =
dataSegmentWithSchemas.getSegments();
Review Comment:
## Unread local variable
Variable 'Set<DataSegment> compactedSegments' is never read.
[Show more
details](https://github.com/apache/druid/security/code-scanning/7260)
##########
server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentDataCacheConcurrencyTest.java:
##########
@@ -113,6 +119,24 @@
new HashMap<>()
);
+ segmentSchemaCache = new SegmentSchemaCache(new NoopServiceEmitter());
+ FingerprintGenerator fingerprintGenerator = new
FingerprintGenerator(mapper);
Review Comment:
## Unread local variable
Variable 'FingerprintGenerator fingerprintGenerator' is never read.
[Show more
details](https://github.com/apache/druid/security/code-scanning/7262)
##########
processing/src/main/java/org/apache/druid/segment/DataSegmentWithSchemas.java:
##########
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.segment;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.druid.timeline.DataSegment;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * Encapsulates segment metadata and corresponding schema.
+ */
+public class DataSegmentWithSchemas
+{
+ private final Set<DataSegment> segments;
+ private final MinimalSegmentSchemas minimalSegmentSchemas;
+
+ public DataSegmentWithSchemas(String schemaVersion)
+ {
+ this.segments = new HashSet<>();
+ this.minimalSegmentSchemas = new MinimalSegmentSchemas(schemaVersion);
+ }
+
+ @JsonCreator
+ public DataSegmentWithSchemas(
+ @JsonProperty("segments") Set<DataSegment> segments,
+ @JsonProperty("minimalSegmentSchemas") MinimalSegmentSchemas
minimalSegmentSchemas
+ )
+ {
+ this.segments = segments;
+ this.minimalSegmentSchemas = minimalSegmentSchemas;
+ }
+
+ @JsonProperty
+ public Set<DataSegment> getSegments()
Review Comment:
## Exposing internal representation
getSegments exposes the internal representation stored in field segments.
The value may be modified [after this call to getSegments](1).
[Show more
details](https://github.com/apache/druid/security/code-scanning/7259)
##########
server/src/test/java/org/apache/druid/segment/metadata/SegmentSchemaBackFillQueueTest.java:
##########
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.segment.metadata;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.druid.common.config.NullHandling;
+import org.apache.druid.java.util.common.Intervals;
+import org.apache.druid.java.util.common.Pair;
+import org.apache.druid.java.util.common.concurrent.ScheduledExecutors;
+import org.apache.druid.metadata.TestDerbyConnector;
+import org.apache.druid.query.aggregation.AggregatorFactory;
+import org.apache.druid.query.aggregation.first.LongFirstAggregatorFactory;
+import org.apache.druid.segment.SchemaPayload;
+import org.apache.druid.segment.TestHelper;
+import org.apache.druid.segment.column.ColumnType;
+import org.apache.druid.segment.column.RowSignature;
+import org.apache.druid.server.metrics.NoopServiceEmitter;
+import org.apache.druid.timeline.DataSegment;
+import org.apache.druid.timeline.partition.LinearShardSpec;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+
+public class SegmentSchemaBackFillQueueTest
+{
+ static {
+ NullHandling.initializeForTests();
+ }
+
+ @Rule
+ public final TestDerbyConnector.DerbyConnectorRule derbyConnectorRule = new
TestDerbyConnector.DerbyConnectorRule(getEnabledConfig());
+
+ private final ObjectMapper mapper = TestHelper.makeJsonMapper();
+
+ @Test
+ public void testPublishSchema() throws InterruptedException
+ {
+ TestDerbyConnector derbyConnector = derbyConnectorRule.getConnector();
+ derbyConnector.createSegmentSchemaTable();
+ derbyConnector.createSegmentTable();
+
+ SegmentSchemaManager segmentSchemaManager = new SegmentSchemaManager(
+ derbyConnectorRule.metadataTablesConfigSupplier().get(),
+ mapper,
+ derbyConnector,
+ new FingerprintGenerator(mapper)
+ );
+
+ SegmentSchemaTestUtils segmentSchemaTestUtils =
+ new SegmentSchemaTestUtils(derbyConnectorRule, derbyConnector, mapper);
+ SegmentSchemaCache segmentSchemaCache = new SegmentSchemaCache(new
NoopServiceEmitter());
+ FingerprintGenerator fingerprintGenerator = new
FingerprintGenerator(mapper);
Review Comment:
## Unread local variable
Variable 'FingerprintGenerator fingerprintGenerator' is never read.
[Show more
details](https://github.com/apache/druid/security/code-scanning/7263)
##########
indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskParallelRunTest.java:
##########
@@ -869,7 +901,9 @@
.granularitySpec(new
ClientCompactionTaskGranularitySpec(Granularities.MINUTE, null, null))
.build();
- final Set<DataSegment> compactedSegments = runTask(compactionTask);
+ final DataSegmentWithSchemas dataSegmentWithSchemas =
runTask(compactionTask);
+ verifySchema(dataSegmentWithSchemas);
+ final Set<DataSegment> compactedSegments =
dataSegmentWithSchemas.getSegments();
Review Comment:
## Unread local variable
Variable 'Set<DataSegment> compactedSegments' is never read.
[Show more
details](https://github.com/apache/druid/security/code-scanning/7261)
##########
server/src/test/java/org/apache/druid/metadata/IndexerSqlMetadataStorageCoordinatorTestBase.java:
##########
@@ -0,0 +1,614 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.metadata;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.hash.Hashing;
+import com.google.common.io.BaseEncoding;
+import org.apache.druid.java.util.common.DateTimes;
+import org.apache.druid.java.util.common.ISE;
+import org.apache.druid.java.util.common.Intervals;
+import org.apache.druid.java.util.common.Pair;
+import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.java.util.common.jackson.JacksonUtils;
+import org.apache.druid.java.util.common.parsers.CloseableIterator;
+import org.apache.druid.segment.MinimalSegmentSchemas;
+import org.apache.druid.segment.TestHelper;
+import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig;
+import org.apache.druid.segment.metadata.FingerprintGenerator;
+import org.apache.druid.segment.metadata.SegmentSchemaManager;
+import org.apache.druid.segment.metadata.SegmentSchemaTestUtils;
+import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec;
+import org.apache.druid.server.http.DataSegmentPlus;
+import org.apache.druid.timeline.DataSegment;
+import org.apache.druid.timeline.SegmentId;
+import org.apache.druid.timeline.partition.LinearShardSpec;
+import org.apache.druid.timeline.partition.NoneShardSpec;
+import org.apache.druid.timeline.partition.NumberedShardSpec;
+import org.apache.druid.timeline.partition.ShardSpec;
+import org.apache.druid.timeline.partition.TombstoneShardSpec;
+import org.joda.time.DateTime;
+import org.joda.time.Interval;
+import org.junit.Assert;
+import org.skife.jdbi.v2.PreparedBatch;
+import org.skife.jdbi.v2.ResultIterator;
+import org.skife.jdbi.v2.util.StringMapper;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+public class IndexerSqlMetadataStorageCoordinatorTestBase
+{
+ protected static final int MAX_SQL_MEATADATA_RETRY_FOR_TEST = 2;
+
+ protected final ObjectMapper mapper = TestHelper.makeJsonMapper();
+
+ protected final DataSegment defaultSegment = new DataSegment(
+ "fooDataSource",
+ Intervals.of("2015-01-01T00Z/2015-01-02T00Z"),
+ "version",
+ ImmutableMap.of(),
+ ImmutableList.of("dim1"),
+ ImmutableList.of("m1"),
+ new LinearShardSpec(0),
+ 9,
+ 100
+ );
+
+ protected final DataSegment eternitySegment = new DataSegment(
+ "fooDataSource",
+ Intervals.ETERNITY,
+ "version",
+ ImmutableMap.of(),
+ ImmutableList.of("dim1"),
+ ImmutableList.of("m1"),
+ new LinearShardSpec(0),
+ 9,
+ 100
+ );
+
+
+ protected final DataSegment firstHalfEternityRangeSegment = new DataSegment(
+ "fooDataSource",
+ new Interval(DateTimes.MIN, DateTimes.of("3000")),
+ "version",
+ ImmutableMap.of(),
+ ImmutableList.of("dim1"),
+ ImmutableList.of("m1"),
+ new LinearShardSpec(0),
+ 9,
+ 100
+ );
+
+ protected final DataSegment secondHalfEternityRangeSegment = new DataSegment(
+ "fooDataSource",
+ new Interval(DateTimes.of("1970"), DateTimes.MAX),
+ "version",
+ ImmutableMap.of(),
+ ImmutableList.of("dim1"),
+ ImmutableList.of("m1"),
+ new LinearShardSpec(0),
+ 9,
+ 100
+ );
+ protected final DataSegment defaultSegment2 = new DataSegment(
+ "fooDataSource",
+ Intervals.of("2015-01-01T00Z/2015-01-02T00Z"),
+ "version",
+ ImmutableMap.of(),
+ ImmutableList.of("dim1"),
+ ImmutableList.of("m1"),
+ new LinearShardSpec(1),
+ 9,
+ 100
+ );
+
+ protected final DataSegment defaultSegment2WithBiggerSize = new DataSegment(
+ "fooDataSource",
+ Intervals.of("2015-01-01T00Z/2015-01-02T00Z"),
+ "version",
+ ImmutableMap.of(),
+ ImmutableList.of("dim1"),
+ ImmutableList.of("m1"),
+ new LinearShardSpec(1),
+ 9,
+ 200
+ );
+
+ protected final DataSegment defaultSegment3 = new DataSegment(
+ "fooDataSource",
+ Intervals.of("2015-01-03T00Z/2015-01-04T00Z"),
+ "version",
+ ImmutableMap.of(),
+ ImmutableList.of("dim1"),
+ ImmutableList.of("m1"),
+ NoneShardSpec.instance(),
+ 9,
+ 100
+ );
+
+ // Overshadows defaultSegment, defaultSegment2
+ protected final DataSegment defaultSegment4 = new DataSegment(
+ "fooDataSource",
+ Intervals.of("2015-01-01T00Z/2015-01-02T00Z"),
+ "zversion",
+ ImmutableMap.of(),
+ ImmutableList.of("dim1"),
+ ImmutableList.of("m1"),
+ new LinearShardSpec(0),
+ 9,
+ 100
+ );
+
+ protected final DataSegment numberedSegment0of0 = new DataSegment(
+ "fooDataSource",
+ Intervals.of("2015-01-01T00Z/2015-01-02T00Z"),
+ "zversion",
+ ImmutableMap.of(),
+ ImmutableList.of("dim1"),
+ ImmutableList.of("m1"),
+ new NumberedShardSpec(0, 0),
+ 9,
+ 100
+ );
+
+ protected final DataSegment numberedSegment1of0 = new DataSegment(
+ "fooDataSource",
+ Intervals.of("2015-01-01T00Z/2015-01-02T00Z"),
+ "zversion",
+ ImmutableMap.of(),
+ ImmutableList.of("dim1"),
+ ImmutableList.of("m1"),
+ new NumberedShardSpec(1, 0),
+ 9,
+ 100
+ );
+
+ protected final DataSegment numberedSegment2of0 = new DataSegment(
+ "fooDataSource",
+ Intervals.of("2015-01-01T00Z/2015-01-02T00Z"),
+ "zversion",
+ ImmutableMap.of(),
+ ImmutableList.of("dim1"),
+ ImmutableList.of("m1"),
+ new NumberedShardSpec(2, 0),
+ 9,
+ 100
+ );
+
+ protected final DataSegment numberedSegment2of1 = new DataSegment(
+ "fooDataSource",
+ Intervals.of("2015-01-01T00Z/2015-01-02T00Z"),
+ "zversion",
+ ImmutableMap.of(),
+ ImmutableList.of("dim1"),
+ ImmutableList.of("m1"),
+ new NumberedShardSpec(2, 1),
+ 9,
+ 100
+ );
+
+ protected final DataSegment numberedSegment3of1 = new DataSegment(
+ "fooDataSource",
+ Intervals.of("2015-01-01T00Z/2015-01-02T00Z"),
+ "zversion",
+ ImmutableMap.of(),
+ ImmutableList.of("dim1"),
+ ImmutableList.of("m1"),
+ new NumberedShardSpec(3, 1),
+ 9,
+ 100
+ );
+
+ protected final DataSegment existingSegment1 = new DataSegment(
+ "fooDataSource",
+ Intervals.of("1994-01-01T00Z/1994-01-02T00Z"),
+ "zversion",
+ ImmutableMap.of(),
+ ImmutableList.of("dim1"),
+ ImmutableList.of("m1"),
+ new NumberedShardSpec(1, 1),
+ 9,
+ 100
+ );
+
+ protected final DataSegment existingSegment2 = new DataSegment(
+ "fooDataSource",
+ Intervals.of("1994-01-02T00Z/1994-01-03T00Z"),
+ "zversion",
+ ImmutableMap.of(),
+ ImmutableList.of("dim1"),
+ ImmutableList.of("m1"),
+ new NumberedShardSpec(1, 1),
+ 9,
+ 100
+ );
+
+ protected final DataSegment hugeTimeRangeSegment1 = new DataSegment(
+ "hugeTimeRangeDataSource",
+ Intervals.of("-9994-01-02T00Z/1994-01-03T00Z"),
+ "zversion",
+ ImmutableMap.of(),
+ ImmutableList.of("dim1"),
+ ImmutableList.of("m1"),
+ new NumberedShardSpec(0, 1),
+ 9,
+ 100
+ );
+
+ protected final DataSegment hugeTimeRangeSegment2 = new DataSegment(
+ "hugeTimeRangeDataSource",
+ Intervals.of("2994-01-02T00Z/2994-01-03T00Z"),
+ "zversion",
+ ImmutableMap.of(),
+ ImmutableList.of("dim1"),
+ ImmutableList.of("m1"),
+ new NumberedShardSpec(0, 1),
+ 9,
+ 100
+ );
+
+ protected final DataSegment hugeTimeRangeSegment3 = new DataSegment(
+ "hugeTimeRangeDataSource",
+ Intervals.of("29940-01-02T00Z/29940-01-03T00Z"),
+ "zversion",
+ ImmutableMap.of(),
+ ImmutableList.of("dim1"),
+ ImmutableList.of("m1"),
+ new NumberedShardSpec(0, 1),
+ 9,
+ 100
+ );
+
+ protected final DataSegment hugeTimeRangeSegment4 = new DataSegment(
+ "hugeTimeRangeDataSource",
+ Intervals.of("1990-01-01T00Z/19940-01-01T00Z"),
+ "zversion",
+ ImmutableMap.of(),
+ ImmutableList.of("dim1"),
+ ImmutableList.of("m1"),
+ new NumberedShardSpec(0, 1),
+ 9,
+ 100
+ );
+
+ protected final Set<DataSegment> SEGMENTS = ImmutableSet.of(defaultSegment,
defaultSegment2);
+ protected final AtomicLong metadataUpdateCounter = new AtomicLong();
+ protected final AtomicLong segmentTableDropUpdateCounter = new AtomicLong();
+
+ protected IndexerSQLMetadataStorageCoordinator coordinator;
+ protected TestDerbyConnector derbyConnector;
+ protected SegmentSchemaManager segmentSchemaManager;
+ protected FingerprintGenerator fingerprintGenerator;
+ protected SegmentSchemaTestUtils segmentSchemaTestUtils;
+
+ protected static class DS
+ {
+ static final String WIKI = "wiki";
+ }
+
+ protected DataSegment createSegment(Interval interval, String version,
ShardSpec shardSpec)
+ {
+ return DataSegment.builder()
+ .dataSource(DS.WIKI)
+ .interval(interval)
+ .version(version)
+ .shardSpec(shardSpec)
+ .size(100)
+ .build();
+ }
+
+ protected List<DataSegment> createAndGetUsedYearSegments(final int
startYear, final int endYear) throws IOException
+ {
+ final List<DataSegment> segments = new ArrayList<>();
+
+ for (int year = startYear; year < endYear; year++) {
+ segments.add(createSegment(
+ Intervals.of("%d/%d", year, year + 1),
+ "version",
+ new LinearShardSpec(0))
+ );
+ }
+ final Set<DataSegment> segmentsSet = new HashSet<>(segments);
+ final Set<DataSegment> committedSegments =
coordinator.commitSegments(segmentsSet, new MinimalSegmentSchemas(
+ CentralizedDatasourceSchemaConfig.SCHEMA_VERSION));
+ Assert.assertTrue(committedSegments.containsAll(segmentsSet));
+
+ return segments;
+ }
+
+ protected ImmutableList<DataSegment> retrieveUnusedSegments(
+ final List<Interval> intervals,
+ final Integer limit,
+ final String lastSegmentId,
+ final SortOrder sortOrder,
+ final DateTime maxUsedStatusLastUpdatedTime,
+ final MetadataStorageTablesConfig tablesConfig
+ )
+ {
+ return derbyConnector.inReadOnlyTransaction(
+ (handle, status) -> {
+ try (final CloseableIterator<DataSegment> iterator =
+ SqlSegmentsMetadataQuery.forHandle(
+ handle,
+ derbyConnector,
+ tablesConfig,
+ mapper
+ )
+ .retrieveUnusedSegments(DS.WIKI,
intervals, null, limit, lastSegmentId, sortOrder,
maxUsedStatusLastUpdatedTime)) {
+ return ImmutableList.copyOf(iterator);
+ }
+ }
+ );
+ }
+
+ protected ImmutableList<DataSegmentPlus> retrieveUnusedSegmentsPlus(
+ final List<Interval> intervals,
+ final Integer limit,
+ final String lastSegmentId,
+ final SortOrder sortOrder,
+ final DateTime maxUsedStatusLastUpdatedTime,
+ MetadataStorageTablesConfig tablesConfig
+ )
+ {
+ return derbyConnector.inReadOnlyTransaction(
+ (handle, status) -> {
+ try (final CloseableIterator<DataSegmentPlus> iterator =
+ SqlSegmentsMetadataQuery.forHandle(
+ handle,
+ derbyConnector,
+ tablesConfig,
+ mapper
+ )
+
.retrieveUnusedSegmentsPlus(DS.WIKI, intervals, null, limit, lastSegmentId,
sortOrder, maxUsedStatusLastUpdatedTime)) {
+ return ImmutableList.copyOf(iterator);
+ }
+ }
+ );
+ }
+
+ protected void verifyContainsAllSegmentsPlus(
+ List<DataSegment> expectedSegments,
+ List<DataSegmentPlus> actualUnusedSegmentsPlus,
+ DateTime usedStatusLastUpdatedTime)
+ {
+ Map<SegmentId, DataSegment> expectedIdToSegment =
expectedSegments.stream().collect(Collectors.toMap(DataSegment::getId,
Function.identity()));
+ Map<SegmentId, DataSegmentPlus> actualIdToSegmentPlus =
actualUnusedSegmentsPlus.stream()
+
.collect(Collectors.toMap(d -> d.getDataSegment().getId(),
Function.identity()));
+ Assert.assertTrue(expectedIdToSegment.entrySet().stream().allMatch(e -> {
+ DataSegmentPlus segmentPlus = actualIdToSegmentPlus.get(e.getKey());
+ return segmentPlus != null
+ &&
!segmentPlus.getCreatedDate().isAfter(usedStatusLastUpdatedTime)
+ && segmentPlus.getUsedStatusLastUpdatedDate() != null
+ &&
segmentPlus.getUsedStatusLastUpdatedDate().equals(usedStatusLastUpdatedTime);
+ }));
+ }
+
+ protected void verifyEqualsAllSegmentsPlus(
+ List<DataSegment> expectedSegments,
+ List<DataSegmentPlus> actualUnusedSegmentsPlus,
+ DateTime usedStatusLastUpdatedTime
+ )
+ {
+ Assert.assertEquals(expectedSegments.size(),
actualUnusedSegmentsPlus.size());
+ for (int i = 0; i < expectedSegments.size(); i++) {
+ DataSegment expectedSegment = expectedSegments.get(i);
+ DataSegmentPlus actualSegmentPlus = actualUnusedSegmentsPlus.get(i);
+ Assert.assertEquals(expectedSegment.getId(),
actualSegmentPlus.getDataSegment().getId());
+
Assert.assertTrue(!actualSegmentPlus.getCreatedDate().isAfter(usedStatusLastUpdatedTime)
+ && actualSegmentPlus.getUsedStatusLastUpdatedDate() !=
null
+ &&
actualSegmentPlus.getUsedStatusLastUpdatedDate().equals(usedStatusLastUpdatedTime));
+ }
+ }
+
+ /**
+ * This test-only shard type is to test the behavior of "old generation"
tombstones with 1 core partition.
+ */
+ protected static class TombstoneShardSpecWith1CorePartition extends
TombstoneShardSpec
+ {
+ @Override
+ @JsonProperty("partitions")
+ public int getNumCorePartitions()
+ {
+ return 1;
+ }
+ }
+
+
+ protected void markAllSegmentsUnused(MetadataStorageTablesConfig
tablesConfig)
+ {
+ markAllSegmentsUnused(SEGMENTS, DateTimes.nowUtc(), tablesConfig);
+ }
+
+ protected void markAllSegmentsUnused(
+ Set<DataSegment> segments,
+ DateTime usedStatusLastUpdatedTime,
+ MetadataStorageTablesConfig tablesConfig
+ )
+ {
+ for (final DataSegment segment : segments) {
+ Assert.assertEquals(
+ 1,
+ (int) derbyConnector.getDBI().<Integer>withHandle(
+ handle -> {
+ String request = StringUtils.format(
+ "UPDATE %s SET used = false, used_status_last_updated =
:used_status_last_updated WHERE id = :id",
+ tablesConfig.getSegmentsTable()
+ );
+ return handle.createStatement(request)
+ .bind("id", segment.getId().toString())
+ .bind("used_status_last_updated",
usedStatusLastUpdatedTime.toString()
+ ).execute();
+ }
+ )
+ );
+ }
+ }
+
+ protected List<String> retrievePendingSegmentIds(MetadataStorageTablesConfig
tablesConfig)
+ {
+ final String table = tablesConfig.getPendingSegmentsTable();
+ return derbyConnector.retryWithHandle(
+ handle -> handle.createQuery("SELECT id FROM " + table + " ORDER BY
id")
+ .map(StringMapper.FIRST)
+ .list()
+ );
+ }
+
+ protected List<String> retrieveUsedSegmentIds(MetadataStorageTablesConfig
tablesConfig)
+ {
+ final String table = tablesConfig.getSegmentsTable();
+ return derbyConnector.retryWithHandle(
+ handle -> handle.createQuery("SELECT id FROM " + table + " WHERE used
= true ORDER BY id")
+ .map(StringMapper.FIRST)
+ .list()
+ );
+ }
+
+ protected List<DataSegment> retrieveUsedSegments(MetadataStorageTablesConfig
tablesConfig)
+ {
+ final String table = tablesConfig.getSegmentsTable();
+ return derbyConnector.retryWithHandle(
+ handle -> handle.createQuery("SELECT payload FROM " + table + " WHERE
used = true ORDER BY id")
+ .map((index, result, context) ->
JacksonUtils.readValue(mapper, result.getBytes(1), DataSegment.class))
+ .list()
+ );
+ }
+
+ protected List<String> retrieveUnusedSegmentIds(MetadataStorageTablesConfig
tablesConfig)
+ {
+ final String table = tablesConfig.getSegmentsTable();
+ return derbyConnector.retryWithHandle(
+ handle -> handle.createQuery("SELECT id FROM " + table + " WHERE used
= false ORDER BY id")
+ .map(StringMapper.FIRST)
+ .list()
+ );
+ }
+
+ protected Boolean
insertPendingSegmentAndSequenceName(Pair<SegmentIdWithShardSpec, String>
pendingSegmentSequenceName, MetadataStorageTablesConfig tablesConfig)
+ {
+ final SegmentIdWithShardSpec pendingSegment =
pendingSegmentSequenceName.lhs;
+ final String sequenceName = pendingSegmentSequenceName.rhs;
+ final String table = tablesConfig.getPendingSegmentsTable();
+ return derbyConnector.retryWithHandle(
+ handle -> {
+ handle.createStatement(
+ StringUtils.format(
+ "INSERT INTO %1$s (id, dataSource, created_date,
start, %2$send%2$s, sequence_name, sequence_prev_id, "
+ + "sequence_name_prev_id_sha1, payload) "
+ + "VALUES (:id, :dataSource, :created_date, :start,
:end, :sequence_name, :sequence_prev_id, "
+ + ":sequence_name_prev_id_sha1, :payload)",
+ table,
+ derbyConnector.getQuoteString()
+ )
+ )
+ .bind("id", pendingSegment.toString())
+ .bind("dataSource", pendingSegment.getDataSource())
+ .bind("created_date", DateTimes.nowUtc().toString())
+ .bind("start",
pendingSegment.getInterval().getStart().toString())
+ .bind("end", pendingSegment.getInterval().getEnd().toString())
+ .bind("sequence_name", sequenceName)
+ .bind("sequence_prev_id", pendingSegment.toString())
+ .bind("sequence_name_prev_id_sha1",
BaseEncoding.base16().encode(
+ Hashing.sha1()
Review Comment:
## Deprecated method or constructor invocation
Invoking [Hashing.sha1](1) should be avoided because it has been deprecated.
[Show more
details](https://github.com/apache/druid/security/code-scanning/7264)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
