This is an automated email from the ASF dual-hosted git repository.
spmallette pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
from d02442461b Merge branch '3.8-dev'
add fb0b2a274d Add pluggable storage layer to TinkerStorageGraph
add 7fd8747be9 Move TinkerStorageGraph storage changelog entries to 4.0.0
section
add f3e09b543c Make TinkerStorageGraph commits fsync-durable with a
configurable sync mode
add c37449203f Serialize concurrent commit writes to TinkerStorageGraph
storage
add d95220565e Lock TinkerStorageGraph storage directory to a single writer
add c6d5261edb Auto-compact TinkerStorageGraph log to bound its growth
add 36c6a102f6 Stream TinkerStorageGraph snapshots one element at a time
add 297872215d Add integrity checks to TinkerStorageGraph storage format
add df86d60296 Add crash-consistency and streaming-scale tests for
TinkerStorageGraph storage
add 71dec51076 Record TinkerStorageGraph format version once per store
add cd98d2e2c8 Document TinkerStorageGraph durability, compaction, and
locking
add 6a85511157 Extract AbstractLogStorage base from GraphBinaryStorage
add 63ec0e3b70 Encode TinkerStorageGraph elements as dictionary-referenced
components
add 29b7d802bc Add opt-in vertex-property id persistence to
TinkerStorageGraph storage
add 902b270af1 Document TinkerStorageGraph compact encoding and
vertex-property id option
add e8c8d0e952 Broaden TinkerStorage TCK: value types, ids, varints,
unicode round-trips
add b27a891adc Test TinkerStorage durability, lifecycle, and dictionary
crash window
add 3fe0eb39a1 Reject corrupt TinkerStorage frames with a clear error
add 298566af64 Document TinkerStorageGraph single-writer stance; enforce
meta-property conflict
add 3e4e2d0587 Document TinkerStorageGraph persistence and fix stale class
doc
add 7101528962 Rename graphLocation to storage.directory for
TinkerStorageGraph
add c3d8a4a1de Revised Upgrade docs around TinkerGraph storage
capabilities.
add 685823d032 Document that TinkerStorageGraph is in-memory without a
storage engine
add 108ad2e022 Bound decoder allocations and dictionary refs in
TinkerStorageGraph storage
add ec8fc29d82 Publish TinkerStorageGraph commits under the storage lock
add c3b129cb74 Rename ByteBufferBuffer to TinkerByteBuffer and add its
unit tests
add adc91e310c Restore TinkerStorageGraph index definitions on reopen
add b3fb189332 Document that persisted multi-properties require setting
default cardinality
add 6362d1c081 Snapshot compaction from committed state so close() cannot
persist uncommitted data
add 0d7c5f1bb2 Rebuild the write-side dictionary on replay to stop
unbounded reopen growth
add cd1990f991 Cut the storage log back to its last complete frame on
replay
add 40240b1799 Fix NullPointerException on rollback with a vertex index
but no edge index
add 64f54b5bba Roll back and stop accepting commits when the storage log
fails
add 975d8af820 Reject commits on a TinkerStorageGraph after its storage is
closed
add ba3dae6408 Trim the header comment in
tinkerstoragegraph-persistent.properties
add 73ccaceca3 Add a read-only recovery open for a damaged
TinkerStorageGraph store
add 9ffb79d78b Undo dictionary entries from a commit that fails to encode
in TinkerStorageGraph
add d4a116f542 Checksum the length of each TinkerStorageGraph log frame so
corruption is not cut back as a torn tail
add d72f090f83 Close TinkerStorageGraph storage and release its directory
lock even when compaction fails on close
add e117a6456d Back off a failed TinkerStorageGraph auto-compaction and
remove its partial snapshot
add 1d299015e4 Detect a missing TinkerStorageGraph snapshot, log or
VERSION marker instead of opening without it
add 3144b8e3a1 Document the value types TinkerStorageGraph can persist and
name the property and type it cannot
add 9f1a48aef2 Document how a persisted TinkerStorageGraph behaves under
concurrent writes
add 80d81aae19 Reject a blank storage directory and a negative compaction
threshold for TinkerStorageGraph
add 9e5afdf0d4 Update the threat model for TinkerStorageGraph persistence
add 151315ebaa Share TinkerStorageGraph file helpers and drop the
deprecated Class.newInstance() in storage selection
add f4bb58ecaf Record the settings a TinkerStorageGraph store was created
with and keep data-shaping settings fixed
add 3e748f5cf1 Stop TinkerStorageGraph commits when its storage directory
or log is removed while open
add ac1c574f33 Keep TinkerStorageGraph indexes unchanged when a commit
fails in storage
add 86ccdf058e Detect TinkerStorageGraph snapshot and log files copied in
from another store
add 4c8a721a99 Tighten TinkerStorageGraph configuration checks and leave
directories as found after failed or recovery opens
add e20cfeaa07 Stop TinkerStorageGraph writing index definitions after
close and serialize those writes
add 4e77a85a3c Fix stale and misplaced Javadoc in the TinkerStorageGraph
storage package
add 7487c1335a Document what TinkerStorageGraph persistence does not
preserve or detect
add 9acac0c722 Refuse explicit compaction of a closed or recovered
TinkerStorageGraph
add c9efeda55a Tell agents the CHANGELOG breaking marker is applied only
at release
add 4270ed7fee Skip TinkerStorageGraph storage writes for transactions
that changed nothing
add 5f8380509c Make TinkerStorageGraph clear() durable and refuse what a
closed or failed store cannot do
add a370a15aab Harden TinkerStorageGraph storage against torn writes,
corrupt records and replaced logs
add c92e8119f9 Refuse TinkerStorageGraph opens that would hide lost data
or damaged settings
add 10006dc74c Tighten TinkerStorageGraph storage test assertions
add 8745d18e14 Update TinkerStorageGraph persistence and upgrade docs
add 9e21f2df4e Move TinkerStorageGraph storage internals into a
tinkergraph-gremlin DESIGN.md
add 52099af346 Warn that a single-cardinality TinkerStorageGraph keeps
only the last multi-property value on reload
add 0d1d0d8bc1 Remove the breaking tag from a hand-written CHANGELOG entry
add 9a0ff251c2 Fix preserveVertexPropertyIds when a TinkerStorageGraph
store is created
add 27ecef800d Stop TinkerStorageGraph commits when its LOCK file is
removed or replaced
add f048ea422f Number TinkerStorageGraph log records so repeated or
reordered ones fail the open
add 2560f699a8 Skip an interrupted compaction's old log in a
TinkerStorageGraph recovery open
add 2965c680cf Report a TinkerStorageGraph commit that fails after its
storage write as durable
add ca4a2b0180 Skip TinkerStorageGraph compaction when nothing was
committed since the last one
add 3a629ee628 Fail a TinkerStorageGraph commit that was waiting on the
commit lock while the graph was cleared
add 50ed289057 Checksum TinkerStorageGraph file headers so a damaged one
is reported as damage
add 1ee20cd69c Bound every size a stored TinkerStorageGraph value
declares, including labels read by type
add 5bb3a5f219 Correct TinkerStorageGraph notes on why dictionary
numbering survives compaction
add d4e89cc245 Read TinkerStorageGraph settings independent of the default
locale and charset
add cc2d0bb0fd Fix stale and orphaned comments in the TinkerStorageGraph
storage code
add 523aab3750 Document the remaining TinkerStorageGraph persistence
caveats and point read-only opens to recovery
add 220fa8f0fc Document how GraphBinary writes a ByteBuffer value
add 3e52f51a45 Keep the TinkerStorageGraph directory lock held against
other processes
add 6f8a65447b Empty a TinkerStorageGraph store on clear() even when
nothing was logged since the last compaction
add f70b65091c Keep a TinkerStorageGraph vertex re-added after clear()
when an older transaction on its id ends
add f110b72001 Cut a damaged final TinkerStorageGraph log record only when
it ends in unwritten sectors
add e34659e8eb Keep a TinkerStorageGraph commit and its storage healthy
when automatic compaction throws an Error
add 40995792f1 Warn when TinkerMemoryGraph is configured with storage
settings it ignores
add 04aec31c85 Test that TinkerStorageGraph stops commits when a
compaction cannot install its empty log
add cebb027645 Name the TinkerStorageGraph crash-window tests for what
they verify
add d59f9ddbdb Pin the single-cardinality multi-property collapse of a
reopened TinkerStorageGraph in a test
add 591feb8f6f Bound the commit-lock wait in a TinkerStorageGraph clear()
test
add 645e75406b Checksum the TinkerStorageGraph VERSION marker and let a
recover open read past a damaged one
add e96e596cd9 Report an unusable value in a TinkerStorageGraph settings
record as a damaged settings file
add a5a71a50cd Link the TinkerGraph disk storage upgrade note to the
reference docs by URL
add 73053a8909 Close small gaps in the TinkerStorageGraph persistence docs
add 6698049b95 Keep the TinkerStorageGraph directory lock held when a
second open in the same JVM is refused
add e08237cf54 Keep an element added while TinkerStorageGraph clear() runs
consistent in memory and on disk
add 41969cdda8 Fail-stop TinkerStorageGraph storage when clear() cannot
empty it and require engines to implement clear()
add d530220b7b Say that a TinkerStorageGraph commit that fails after its
storage write may be partly visible until reopen
add d65ea88ec3 Fail a TinkerStorageGraph open that sets a storage
directory without a storage engine
add ae893c737e Close configured services and the storage engine when a
TinkerStorageGraph open fails
add 9536425ce9 Use the inherited directory delete in
TinkerStorageGraphProvider
add a69f1a105c Merge TinkerTransaction's vertex and edge storage mutation
conversion into one method
new a95d2e442a Merge pull request #3639 from apache/tinkergraph-storage
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.skills/tinker-dev/SKILL.md | 8 +-
CHANGELOG.asciidoc | 7 +-
THREAT_MODEL.md | 11 +-
.../reference/implementations-tinkergraph.asciidoc | 352 ++++-
docs/src/upgrade/release-4.x.x.asciidoc | 91 +-
...o.properties => tinkergraph-storage.properties} | 11 +-
.../io/binary/types/BinarySerializer.java | 8 +
.../conf/tinkergraph-credentials.properties | 4 +
.../conf/tinkerstoragegraph-persistent.properties | 39 +
.../gremlin/server/auth/SimpleAuthenticator.java | 54 +
tinkergraph-gremlin/DESIGN.md | 294 ++++
.../tinkergraph/structure/AbstractTinkerGraph.java | 196 ++-
.../structure/TinkerElementContainer.java | 10 +-
.../gremlin/tinkergraph/structure/TinkerGraph.java | 51 +-
.../tinkergraph/structure/TinkerMemoryGraph.java | 38 +-
.../tinkergraph/structure/TinkerStorageGraph.java | 336 ++++-
.../tinkergraph/structure/TinkerTransaction.java | 149 +-
.../structure/storage/AbstractLogStorage.java | 1562 ++++++++++++++++++++
.../structure/storage/DefaultStorage.java | 31 +-
.../structure/storage/DirectoryLock.java | 334 +++++
.../structure/storage/GraphBinaryStorage.java | 634 ++++++++
.../structure/storage/IndexDefinitions.java | 183 +++
.../structure/storage/StorageFiles.java | 88 ++
.../structure/storage/StorageSettings.java | 314 ++++
.../tinkergraph/structure/storage/SyncMode.java | 76 +
.../structure/storage/TinkerByteBuffer.java | 371 +++++
.../structure/storage/TinkerStorage.java | 155 ++
.../structure/storage/TinkerStorageMutation.java | 67 +
.../tinkergraph/TinkerMemoryGraphProvider.java | 20 -
.../tinkergraph/TinkerMemoryGraphUUIDProvider.java | 6 -
.../tinkergraph/TinkerStorageGraphProvider.java | 14 +-
.../structure/TinkerMemoryGraphTest.java | 211 +--
.../structure/TinkerStorageGraphTest.java | 455 +++++-
.../AbstractTinkerStorageConformanceTest.java | 715 +++++++++
.../structure/storage/ConcurrencyProbeStorage.java | 89 ++
.../structure/storage/DirectoryLockHolder.java | 23 +-
.../structure/storage/DirectoryLockTest.java | 187 +++
.../structure/storage/FaultInjectingStorage.java | 164 ++
.../structure/storage/GraphBinaryStorageTest.java | 1489 +++++++++++++++++++
.../storage/StorageCommitSerializationTest.java | 105 ++
.../storage/StorageCrashConsistencyTest.java | 231 +++
.../structure/storage/StorageFailureTest.java | 778 ++++++++++
.../structure/storage/StorageRecoveryTest.java | 895 +++++++++++
.../structure/storage/StorageSettingsTest.java | 489 ++++++
.../tinkergraph/structure/storage/StoreWriter.java | 50 +
.../structure/storage/SyncModeTest.java | 58 +
.../structure/storage/TinkerByteBufferTest.java | 265 ++++
47 files changed, 11312 insertions(+), 406 deletions(-)
rename gremlin-console/conf/{tinkergraph-gryo.properties =>
tinkergraph-storage.properties} (60%)
create mode 100644 gremlin-server/conf/tinkerstoragegraph-persistent.properties
create mode 100644 tinkergraph-gremlin/DESIGN.md
create mode 100644
tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/AbstractLogStorage.java
copy
gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/function/BulkSetSupplier.java
=>
tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/DefaultStorage.java
(55%)
create mode 100644
tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/DirectoryLock.java
create mode 100644
tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java
create mode 100644
tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/IndexDefinitions.java
create mode 100644
tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/StorageFiles.java
create mode 100644
tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/StorageSettings.java
create mode 100644
tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/SyncMode.java
create mode 100644
tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/TinkerByteBuffer.java
create mode 100644
tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/TinkerStorage.java
create mode 100644
tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/TinkerStorageMutation.java
create mode 100644
tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/AbstractTinkerStorageConformanceTest.java
create mode 100644
tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/ConcurrencyProbeStorage.java
copy
gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/server/util/ExceptionTestUtils.java
=>
tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/DirectoryLockHolder.java
(54%)
create mode 100644
tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/DirectoryLockTest.java
create mode 100644
tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/FaultInjectingStorage.java
create mode 100644
tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java
create mode 100644
tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/StorageCommitSerializationTest.java
create mode 100644
tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/StorageCrashConsistencyTest.java
create mode 100644
tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/StorageFailureTest.java
create mode 100644
tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/StorageRecoveryTest.java
create mode 100644
tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/StorageSettingsTest.java
create mode 100644
tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/StoreWriter.java
create mode 100644
tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/SyncModeTest.java
create mode 100644
tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/TinkerByteBufferTest.java
