This is an automated email from the ASF dual-hosted git repository.
xiangfu0 pushed a change to branch
xiangfu0/codex1/lucene-index-rollback-recovery
in repository https://gitbox.apache.org/repos/asf/pinot.git
omit fbe82e7e25a Report failed Lucene recovery marker cleanup
omit a51e7268166 Rebuild incompatible Lucene indexes during segment loading
add 46da5d2f908 Fix bytes pk for dim tables (#19501)
add 8705c083394 Mask credential query parameters in the controller request
log (#19899)
add fbc34d7e7b2 Mask more credential keys in Obfuscator's defaults (#19884)
add faa74c30178 Fail fast on EXPR_MIN/EXPR_MAX queries that the servers
cannot execute (#19883)
add 65175a46323 Document candidate list sharing invariants and test them
for upsert tables (#19878)
add b543d4d6b89 Bump org.webjars:swagger-ui from 5.33.1 to 5.33.2 (#19904)
add cf462e0254b Bump com.azure:azure-sdk-bom from 1.3.7 to 1.3.9 (#19903)
add ba3ddac2546 Bump software.amazon.awssdk:bom from 2.55.14 to 2.55.15
(#19902)
add b38d073146a Fix duplicate upsert rows after restart when a snapshot is
missed with metadataTTL (#19832)
add 1ecbd5c99db Fail op chains of terminated MSE queries instead of
dropping them (#19854)
add a46d4e047b4 Redact credentials from captured audit payloads (#19555)
add c5eeb91e388 Let OBJECT codecs serialize payloads into caller buffers
(#19836)
add fe6511365d0 Simplify re-ingestion tracking and skip obsolete queued
jobs (#19893)
add 0d624f598f2 Clone IdealState without a serialization round trip
(#19738)
add 55afab92fe1 Use managed Guava and Jackson instead of embedded
vulnerable copies (#19890)
add ae34ff9c5a7 Cache parsed Lucene queries across segment readers (#19865)
add 431526070e4 Rebuild incompatible Lucene indexes during segment loading
add ca3538f4b40 Report failed Lucene recovery marker cleanup
add 0d97da6d57c Preserve new default columns during Lucene rollback
recovery
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (fbe82e7e25a)
\
N -- N -- N
refs/heads/xiangfu0/codex1/lucene-index-rollback-recovery (0d97da6d57c)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.../BaseSingleStageBrokerRequestHandler.java | 97 +++++
.../instanceselector/BaseInstanceSelector.java | 3 +
.../ReplicaGroupInstanceSelector.java | 3 +-
.../broker/requesthandler/QueryValidationTest.java | 57 +++
.../instanceselector/InstanceSelectorTest.java | 18 +-
.../apache/pinot/common/audit/AuditRedactor.java | 424 +++++++++++++++++++++
.../pinot/common/audit/AuditRequestProcessor.java | 70 +++-
.../org/apache/pinot/common/utils/DataSchema.java | 4 +-
.../pinot/common/utils/helix/HelixHelper.java | 35 +-
.../common/utils/helix/IdealStateSingleCommit.java | 9 +-
.../pinot/common/audit/AuditRedactorTest.java | 377 ++++++++++++++++++
.../common/audit/AuditRequestProcessorTest.java | 264 +++++++++++--
.../pinot/common/utils/helix/HelixHelperTest.java | 113 ++++++
.../api/resources/ControllerResponseFilter.java | 51 ++-
.../resources/ControllerResponseFilterTest.java | 71 ++++
.../apache/pinot/core/common/ObjectSerDeUtils.java | 213 +++++++++--
.../core/common/datablock/DataBlockBuilder.java | 7 +-
.../manager/offline/DimensionTableDataManager.java | 12 +-
.../offline/MemoryOptimizedDimensionTable.java | 25 +-
.../operator/filter/TextMatchFilterOperator.java | 42 +-
.../function/LookupTransformFunction.java | 4 +-
.../org/apache/pinot/core/plan/FilterPlanNode.java | 6 +-
.../core/query/request/context/QueryContext.java | 12 +-
.../core/transport/InstanceRequestHandler.java | 36 +-
.../pinot/core/common/ObjectSerDeUtilsTest.java | 163 ++++++++
.../DimensionTableDataBytesPrimaryKeyTest.java | 235 ++++++++++++
.../core/transport/InstanceRequestHandlerTest.java | 46 +++
...ransformFunctionBytesPrimaryKeyQueriesTest.java | 287 ++++++++++++++
pinot-core/src/test/resources/data/dimAsset.csv | 2 +
.../src/test/resources/data/dimAsset_config.json | 2 +-
.../src/test/resources/data/dimAsset_schema.json | 9 +
.../UpsertSnapshotRestartIntegrationTest.java | 350 +++++++++++++++++
pinot-plugins/pinot-file-system/pinot-s3/pom.xml | 4 +
.../pinot-stream-ingestion/pinot-pulsar/pom.xml | 4 +
.../apache/pinot/query/runtime/QueryRunner.java | 9 +-
.../runtime/executor/OpChainSchedulerService.java | 131 +++++--
.../pinot/query/runtime/plan/OpChainConverter.java | 6 +-
.../plan/pipeline/PipelineBreakerOperator.java | 12 +
.../pinot/query/service/server/QueryServer.java | 9 +-
.../apache/pinot/query/QueryServerEnclosure.java | 9 +-
.../HardLimitExecutorOpChainRegressionTest.java | 13 +-
.../executor/OpChainSchedulerServiceTest.java | 175 +++++++--
.../plan/OpChainConverterDispatcherTest.java | 5 +-
.../plan/pipeline/PipelineBreakerExecutorTest.java | 51 +++
.../queries/GatedExecutorServicePlugin.java | 142 +++++++
.../query/runtime/queries/QueryRunnerTestBase.java | 14 +-
.../QueryTerminatedBeforeStageStartsTest.java | 190 +++++++++
.../runtime/queries/ResourceBasedQueriesTest.java | 94 ++++-
.../src/test/resources/queries/LookupJoin.json | 154 +++++++-
.../segment/index/loader/SegmentPreProcessor.java | 74 +---
.../invertedindex/MultiColumnTextIndexHandler.java | 73 ++--
.../loader/invertedindex/VectorIndexHandler.java | 4 +-
.../index/readers/text/LuceneTextIndexReader.java | 104 ++++-
.../segment/readers/PinotSegmentColumnReader.java | 9 +
.../segment/readers/PinotSegmentRecordReader.java | 83 +++-
.../upsert/BasePartitionUpsertMetadataManager.java | 61 ++-
.../invertedindex/HnswIndexRollbackTest.java | 20 +-
.../TextIndexRollbackRecoveryTest.java | 85 ++++-
.../readers/text/LuceneTextIndexCountTest.java | 2 +-
.../text/LuceneTextIndexQueryCacheTest.java | 264 +++++++++++++
.../readers/PinotSegmentRecordReaderTest.java | 41 +-
.../local/segment/readers/PinotSegmentUtil.java | 4 +
.../local/upsert/UpsertSnapshotRestartTest.java | 345 +++++++++++++++++
.../utils/LuceneIndexCompatibilityUtilsTest.java | 15 +-
.../segment/local/utils/LuceneIndexTestUtils.java | 61 +++
.../segment/spi/index/reader/QueryScopedCache.java | 47 +++
.../segment/spi/index/reader/TextIndexReader.java | 19 +
.../server/api/resources/ReingestionResource.java | 71 ++--
.../ReingestionConsumptionTimeoutTest.java | 13 +-
.../api/resources/ReingestionResourceApiTest.java | 112 +++++-
.../apache/pinot/spi/query/QueryThreadContext.java | 10 +-
.../org/apache/pinot/spi/utils/Obfuscator.java | 21 +-
.../org/apache/pinot/spi/utils/ObfuscatorTest.java | 35 ++
pom.xml | 117 +++++-
74 files changed, 5249 insertions(+), 510 deletions(-)
create mode 100644
pinot-common/src/main/java/org/apache/pinot/common/audit/AuditRedactor.java
create mode 100644
pinot-common/src/test/java/org/apache/pinot/common/audit/AuditRedactorTest.java
create mode 100644
pinot-controller/src/test/java/org/apache/pinot/controller/api/resources/ControllerResponseFilterTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/data/manager/offline/DimensionTableDataBytesPrimaryKeyTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/queries/LookupTransformFunctionBytesPrimaryKeyQueriesTest.java
create mode 100644 pinot-core/src/test/resources/data/dimAsset.csv
copy pinot-integration-tests/src/test/resources/dimDayOfWeek_config.json =>
pinot-core/src/test/resources/data/dimAsset_config.json (89%)
create mode 100644 pinot-core/src/test/resources/data/dimAsset_schema.json
create mode 100644
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/UpsertSnapshotRestartIntegrationTest.java
create mode 100644
pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/queries/GatedExecutorServicePlugin.java
create mode 100644
pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/queries/QueryTerminatedBeforeStageStartsTest.java
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/readers/text/LuceneTextIndexQueryCacheTest.java
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/upsert/UpsertSnapshotRestartTest.java
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/LuceneIndexTestUtils.java
create mode 100644
pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/reader/QueryScopedCache.java
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
