smengcl commented on code in PR #4360:
URL: https://github.com/apache/ozone/pull/4360#discussion_r1136375305
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java:
##########
@@ -298,27 +376,144 @@ private void verifySnapshotInfoForSnapDiff(final
SnapshotInfo fromSnapshot,
}
}
- private ManagedRocksDB createDbForSnapshotDiff(OzoneConfiguration config) {
- final ManagedOptions managedOptions = new ManagedOptions();
- managedOptions.setCreateIfMissing(true);
+ private int getIndexFromToken(final String token) throws IOException {
+ if (isBlank(token)) {
+ return 0;
+ }
+
+ // Validate that token passed in the request is valid integer as of now.
+ // Later we can change it if we migrate to encrypted or cursor token.
+ try {
+ int index = Integer.parseInt(token);
+ if (index < 0) {
+ throw new IOException("Passed token is invalid. Resend the request " +
+ "with valid token returned in previous request.");
+ }
+ return index;
+ } catch (NumberFormatException exception) {
+ throw new IOException("Passed token is invalid. " +
+ "Resend the request with valid token returned in previous request.");
+ }
+ }
+
+ private ManagedRocksDB createRocksDbForSnapshotDiff(
+ final ManagedDBOptions dbOptions, String dbPath,
+ final List<ColumnFamilyDescriptor> familyDescriptors,
+ final List<ColumnFamilyHandle> familyHandles
+ ) {
+ try {
+ return ManagedRocksDB.open(dbOptions,
+ dbPath,
+ familyDescriptors,
+ familyHandles);
+ } catch (RocksDBException exception) {
+ // TODO: [SNAPSHOT] Fail gracefully.
+ throw new RuntimeException(exception);
+ }
+ }
+
+ private String getDbPath(final OzoneConfiguration config) {
+ File dbDirPath = ServerUtils.getDBPath(config,
+ OZONE_OM_SNAPSHOT_DIFF_DB_DIR);
+ return Paths.get(dbDirPath.toString(), OM_SNAPSHOT_DIFF_DB_NAME)
+ .toFile().getAbsolutePath();
+ }
- final File dbDirPath =
- ServerUtils.getDBPath(config, OZONE_OM_SNAPSHOT_DIFF_DB_DIR);
+ private List<ColumnFamilyDescriptor> getExitingColumnFamilyDescriptors(
+ final String path) {
+ try {
+ return RocksDatabase.listColumnFamiliesEmptyOptions(path)
+ .stream()
+ .map(columnFamilyName -> new ColumnFamilyDescriptor(
+ columnFamilyName, columnFamilyOptions))
+ .collect(Collectors.toList());
+ } catch (RocksDBException exception) {
+ // TODO: [SNAPSHOT] Fail gracefully.
+ throw new RuntimeException(exception);
+ }
+ }
- String dbPath = Paths.get(dbDirPath.toString(), OM_SNAPSHOT_DIFF_DB_NAME)
- .toFile()
- .getAbsolutePath();
+ /**
+ * Return the column family from column family list if it was existing
+ * column family, otherwise create new column family.
+ * This is for backward and forward compatibility.
+ * For backward compatibility, when column family doesn't exist. it will
+ * create new one and return that.
+ * For forward compatibility, it will return the existing one.
+ */
+ private ColumnFamilyHandle getOrCreateColumnFamily(
+ final String columnFamilyName,
+ final List<ColumnFamilyDescriptor> familyDescriptors,
+ final List<ColumnFamilyHandle> familyHandles) {
+
+ for (int i = 0; i < familyDescriptors.size(); i++) {
+ String cfName = StringUtils.bytes2String(familyDescriptors.get(i)
+ .getName());
+ if (columnFamilyName.equals(cfName)) {
+ return familyHandles.get(i);
+ }
+ }
try {
- return ManagedRocksDB.open(managedOptions, dbPath);
+ ColumnFamilyDescriptor columnFamilyDescriptor =
+ new
ColumnFamilyDescriptor(StringUtils.string2Bytes(columnFamilyName),
+ columnFamilyOptions);
+ ColumnFamilyHandle columnFamily = snapshotDiffDb.get()
+ .createColumnFamily(columnFamilyDescriptor);
+
+ // Add column family and descriptor so that they can be closed if needed.
+ familyHandles.add(columnFamily);
+ familyDescriptors.add(columnFamilyDescriptor);
+ return columnFamily;
} catch (RocksDBException exception) {
- // TODO: Fail gracefully.
+ // TODO: [SNAPSHOT] Fail gracefully.
throw new RuntimeException(exception);
}
}
+ private void closeRocksDbObjects(
+ final ManagedDBOptions managedDBOptions,
+ final ManagedColumnFamilyOptions managedColumnFamilyOptions,
+ final List<ColumnFamilyDescriptor> columnFamilyDescriptors,
+ final List<ColumnFamilyHandle> columnFamilyHandles,
+ final ManagedRocksDB managedRocksDB) {
+
+ if (managedDBOptions != null) {
+ managedDBOptions.close();
+ }
+ if (managedColumnFamilyOptions != null) {
+ closeColumnFamilyOptions(managedColumnFamilyOptions);
+ }
+ if (columnFamilyDescriptors != null) {
+ columnFamilyDescriptors.forEach(columnFamilyDescriptor ->
+ closeColumnFamilyOptions((ManagedColumnFamilyOptions)
+ columnFamilyDescriptor.getOptions()));
+ }
+ if (columnFamilyHandles != null) {
+ columnFamilyHandles.forEach(ColumnFamilyHandle::close);
+ }
+ if (managedRocksDB != null) {
+ managedRocksDB.close();
+ }
Review Comment:
Not sure if it matters but RDB (cpp) code always closes `Options` after
`DB`:
https://github.com/facebook/rocksdb/blob/c1ec0b28eb6e93bc2799a68066cdc8147f92ba59/java/jmh/src/main/java/org/rocksdb/jmh/MultiGetBenchmarks.java#L105-L110
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java:
##########
@@ -298,27 +376,144 @@ private void verifySnapshotInfoForSnapDiff(final
SnapshotInfo fromSnapshot,
}
}
- private ManagedRocksDB createDbForSnapshotDiff(OzoneConfiguration config) {
- final ManagedOptions managedOptions = new ManagedOptions();
- managedOptions.setCreateIfMissing(true);
+ private int getIndexFromToken(final String token) throws IOException {
+ if (isBlank(token)) {
+ return 0;
+ }
+
+ // Validate that token passed in the request is valid integer as of now.
+ // Later we can change it if we migrate to encrypted or cursor token.
+ try {
+ int index = Integer.parseInt(token);
+ if (index < 0) {
+ throw new IOException("Passed token is invalid. Resend the request " +
+ "with valid token returned in previous request.");
+ }
+ return index;
+ } catch (NumberFormatException exception) {
+ throw new IOException("Passed token is invalid. " +
+ "Resend the request with valid token returned in previous request.");
+ }
+ }
+
+ private ManagedRocksDB createRocksDbForSnapshotDiff(
+ final ManagedDBOptions dbOptions, String dbPath,
+ final List<ColumnFamilyDescriptor> familyDescriptors,
+ final List<ColumnFamilyHandle> familyHandles
+ ) {
+ try {
+ return ManagedRocksDB.open(dbOptions,
+ dbPath,
+ familyDescriptors,
+ familyHandles);
+ } catch (RocksDBException exception) {
+ // TODO: [SNAPSHOT] Fail gracefully.
+ throw new RuntimeException(exception);
+ }
+ }
+
+ private String getDbPath(final OzoneConfiguration config) {
+ File dbDirPath = ServerUtils.getDBPath(config,
+ OZONE_OM_SNAPSHOT_DIFF_DB_DIR);
+ return Paths.get(dbDirPath.toString(), OM_SNAPSHOT_DIFF_DB_NAME)
+ .toFile().getAbsolutePath();
+ }
- final File dbDirPath =
- ServerUtils.getDBPath(config, OZONE_OM_SNAPSHOT_DIFF_DB_DIR);
+ private List<ColumnFamilyDescriptor> getExitingColumnFamilyDescriptors(
+ final String path) {
+ try {
+ return RocksDatabase.listColumnFamiliesEmptyOptions(path)
+ .stream()
+ .map(columnFamilyName -> new ColumnFamilyDescriptor(
+ columnFamilyName, columnFamilyOptions))
+ .collect(Collectors.toList());
+ } catch (RocksDBException exception) {
+ // TODO: [SNAPSHOT] Fail gracefully.
+ throw new RuntimeException(exception);
+ }
+ }
- String dbPath = Paths.get(dbDirPath.toString(), OM_SNAPSHOT_DIFF_DB_NAME)
- .toFile()
- .getAbsolutePath();
+ /**
+ * Return the column family from column family list if it was existing
+ * column family, otherwise create new column family.
+ * This is for backward and forward compatibility.
+ * For backward compatibility, when column family doesn't exist. it will
+ * create new one and return that.
+ * For forward compatibility, it will return the existing one.
+ */
+ private ColumnFamilyHandle getOrCreateColumnFamily(
+ final String columnFamilyName,
+ final List<ColumnFamilyDescriptor> familyDescriptors,
+ final List<ColumnFamilyHandle> familyHandles) {
+
+ for (int i = 0; i < familyDescriptors.size(); i++) {
+ String cfName = StringUtils.bytes2String(familyDescriptors.get(i)
+ .getName());
+ if (columnFamilyName.equals(cfName)) {
+ return familyHandles.get(i);
+ }
+ }
try {
- return ManagedRocksDB.open(managedOptions, dbPath);
+ ColumnFamilyDescriptor columnFamilyDescriptor =
+ new
ColumnFamilyDescriptor(StringUtils.string2Bytes(columnFamilyName),
+ columnFamilyOptions);
+ ColumnFamilyHandle columnFamily = snapshotDiffDb.get()
+ .createColumnFamily(columnFamilyDescriptor);
+
+ // Add column family and descriptor so that they can be closed if needed.
+ familyHandles.add(columnFamily);
+ familyDescriptors.add(columnFamilyDescriptor);
+ return columnFamily;
} catch (RocksDBException exception) {
- // TODO: Fail gracefully.
+ // TODO: [SNAPSHOT] Fail gracefully.
throw new RuntimeException(exception);
}
}
+ private void closeRocksDbObjects(
+ final ManagedDBOptions managedDBOptions,
+ final ManagedColumnFamilyOptions managedColumnFamilyOptions,
+ final List<ColumnFamilyDescriptor> columnFamilyDescriptors,
+ final List<ColumnFamilyHandle> columnFamilyHandles,
+ final ManagedRocksDB managedRocksDB) {
+
+ if (managedDBOptions != null) {
+ managedDBOptions.close();
+ }
+ if (managedColumnFamilyOptions != null) {
+ closeColumnFamilyOptions(managedColumnFamilyOptions);
+ }
+ if (columnFamilyDescriptors != null) {
+ columnFamilyDescriptors.forEach(columnFamilyDescriptor ->
+ closeColumnFamilyOptions((ManagedColumnFamilyOptions)
+ columnFamilyDescriptor.getOptions()));
+ }
+ if (columnFamilyHandles != null) {
+ columnFamilyHandles.forEach(ColumnFamilyHandle::close);
+ }
+ if (managedRocksDB != null) {
+ managedRocksDB.close();
+ }
Review Comment:
Not sure if it matters but RDB example code always closes `Options` after
`DB`:
https://github.com/facebook/rocksdb/blob/c1ec0b28eb6e93bc2799a68066cdc8147f92ba59/java/jmh/src/main/java/org/rocksdb/jmh/MultiGetBenchmarks.java#L105-L110
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
