This is an automated email from the ASF dual-hosted git repository.
jinglun pushed a commit to branch HADOOP-19236-original
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit d0664d90078ea1c0d3e8dccf27b5c2042f5078c3
Author: lijinglun <lijing...@bytedance.com>
AuthorDate: Thu Aug 15 17:27:25 2024 +0800
Integration of TOS: Add ObjectUtils UUIDUtils and Range.
---
.../apache/hadoop/fs/tosfs/object/ObjectUtils.java | 91 ++++++++++++++-
.../org/apache/hadoop/fs/tosfs/util/Range.java | 122 ++++++++++++++++++++-
.../org/apache/hadoop/fs/tosfs/util/UUIDUtils.java | 31 +++++-
3 files changed, 241 insertions(+), 3 deletions(-)
diff --git
a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectUtils.java
b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectUtils.java
index 4b8613d8cef..eeebdeef658 100644
---
a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectUtils.java
+++
b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectUtils.java
@@ -1,2 +1,91 @@
-package org.apache.hadoop.fs.tosfs.util;public class ObjectUtils {
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.tosfs.util.Range;
+import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
+import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Lists;
+
+import java.util.List;
+
+public class ObjectUtils {
+ public static final String SLASH = "/";
+
+ private ObjectUtils() {
+ }
+
+ public static Path keyToPath(String key) {
+ return new Path(SLASH + key);
+ }
+
+ public static String path(String key) {
+ return key.startsWith(SLASH) ? key : SLASH + key;
+ }
+
+ public static String pathToKey(Path p) {
+ return pathToKey(p, false);
+ }
+
+ public static String pathToKey(Path p, Boolean isDir) {
+ Preconditions.checkArgument(p != null, "Null path");
+ if (p.toUri().getScheme() != null && p.toUri().getPath().isEmpty()) {
+ return "";
+ }
+ String key = p.toUri().getPath().substring(1);
+ if (isDir && !key.isEmpty()) {
+ return key.endsWith(SLASH) ? key : key + SLASH;
+ }
+ return key;
+ }
+
+ public static void deleteAllObjects(ObjectStorage storage,
Iterable<ObjectInfo> objects, int batchSize) {
+ List<String> keysToDelete = Lists.newArrayList();
+ for (ObjectInfo obj : objects) {
+ keysToDelete.add(obj.key());
+
+ if (keysToDelete.size() == batchSize) {
+ batchDelete(storage, keysToDelete);
+ keysToDelete.clear();
+ }
+ }
+
+ if (!keysToDelete.isEmpty()) {
+ batchDelete(storage, keysToDelete);
+ }
+ }
+
+ private static void batchDelete(ObjectStorage storage, List<String> keys) {
+ List<String> failedKeys = storage.batchDelete(keys);
+ if (!failedKeys.isEmpty()) {
+ throw new RuntimeException(String.format("Failed to delete %s objects,
detail: %s",
+ failedKeys.size(), Joiner.on(",").join(failedKeys)));
+ }
+ }
+
+ public static Range calculateRange(final long offset, final long limit,
final long objSize) {
+ Preconditions.checkArgument(offset >= 0, String.format("offset is a
negative number: %s", offset));
+ Preconditions.checkArgument(offset <= objSize,
+ String.format("offset: %s is bigger than object size: %s", offset,
objSize));
+ long len = limit < 0 ? objSize - offset : Math.min(objSize - offset,
limit);
+ return Range.of(offset, len);
+ }
}
+
diff --git
a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/util/Range.java
b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/util/Range.java
index db68a6e1e5c..933fa6596d3 100644
---
a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/util/Range.java
+++
b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/util/Range.java
@@ -1,2 +1,122 @@
-package org.apache.hadoop.fs.tosfs.util;public class Range {
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.util;
+
+import org.apache.hadoop.thirdparty.com.google.common.base.MoreObjects;
+import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Lists;
+
+import java.util.List;
+import java.util.Objects;
+
+public class Range {
+ private final long off;
+ private final long len;
+
+ private Range(long off, long len) {
+ this.off = off;
+ this.len = len;
+ }
+
+ public static Range of(long off, long len) {
+ return new Range(off, len);
+ }
+
+ public long off() {
+ return off;
+ }
+
+ public long len() {
+ return len;
+ }
+
+ public long end() {
+ return off + len;
+ }
+
+ public boolean include(long pos) {
+ return pos >= off && pos < off + len;
+ }
+
+ public boolean include(Range r) {
+ return off() <= r.off() && r.end() <= end();
+ }
+
+ public boolean overlap(Range r) {
+ return r.off() < end() && off() < r.end();
+ }
+
+ public boolean overlap(long off1, long len1) {
+ return off1 < end() && off() < (off1 + len1);
+ }
+
+ public Range alignTo(long fixedSize) {
+ // Get the aligned start.
+ long alignedStart = off - off % fixedSize;
+ // Get the aligned end.
+ long mod = end() % fixedSize;
+ long alignedEnd = mod == 0 ? end() : (end() - mod + fixedSize);
+ return Range.of(alignedStart, alignedEnd - alignedStart);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(off, len);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ } else if (!(o instanceof Range)) {
+ return false;
+ }
+
+ Range that = (Range) o;
+ return Objects.equals(off, that.off)
+ && Objects.equals(len, that.len);
+ }
+
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("offset", off)
+ .add("length", len)
+ .toString();
+ }
+
+ public static List<Range> split(long totalSize, long width) {
+ Preconditions.checkArgument(totalSize >= 0, "Size %s must be >= 0",
totalSize);
+ Preconditions.checkArgument(width > 0, "Width %s must be positive", width);
+
+ long remain = totalSize % width;
+ long rangeNum = totalSize / width;
+
+ List<Range> ranges = Lists.newArrayListWithCapacity((int) rangeNum +
(remain == 0 ? 0 : 1));
+ for (int i = 0; i < rangeNum; i++) {
+ ranges.add(Range.of(i * width, width));
+ }
+
+ if (remain > 0) {
+ ranges.add(Range.of(totalSize - remain, remain));
+ }
+
+ return ranges;
+ }
}
+
diff --git
a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/util/UUIDUtils.java
b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/util/UUIDUtils.java
index 5b864a10e11..a94a1a31619 100644
---
a/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/util/UUIDUtils.java
+++
b/hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/util/UUIDUtils.java
@@ -1,2 +1,31 @@
-package org.apache.hadoop.fs.tosfs.util;public class UUIDUtils {
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.util;
+
+import java.util.UUID;
+
+public class UUIDUtils {
+ private UUIDUtils() {
+ }
+
+ public static String random() {
+ UUID uuid = UUID.randomUUID();
+ return uuid.toString().replace("-", "");
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
