This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new a31d0fba6a HDDS-12211. Add TestHttpFSMetrics to test OpsCreate and
OpsAppend metrics (#7860)
a31d0fba6a is described below
commit a31d0fba6ab741c8cb0a360bf9eea7021500e8b9
Author: Peter Lee <[email protected]>
AuthorDate: Sat Feb 15 00:52:21 2025 +0800
HDDS-12211. Add TestHttpFSMetrics to test OpsCreate and OpsAppend metrics
(#7860)
---
hadoop-ozone/httpfsgateway/pom.xml | 2 -
.../http/server/metrics/HttpFSServerMetrics.java | 48 +++++++
.../service/hadoop/FileSystemAccessService.java | 2 +-
.../fs/http/server/metrics/TestHttpFSMetrics.java | 140 +++++++++++++++++++++
.../src/test/resources/log4j.properties | 19 +++
5 files changed, 208 insertions(+), 3 deletions(-)
diff --git a/hadoop-ozone/httpfsgateway/pom.xml
b/hadoop-ozone/httpfsgateway/pom.xml
index a1fe21aa10..328e0c3f53 100644
--- a/hadoop-ozone/httpfsgateway/pom.xml
+++ b/hadoop-ozone/httpfsgateway/pom.xml
@@ -34,8 +34,6 @@
<httpfs.source.repository>REPO NOT AVAIL</httpfs.source.repository>
<httpfs.source.revision>REVISION NOT AVAIL</httpfs.source.revision>
<maven.build.timestamp.format>yyyy-MM-dd'T'HH:mm:ssZ</maven.build.timestamp.format>
- <!-- no tests in this module so far -->
- <maven.test.skip>true</maven.test.skip>
</properties>
<dependencies>
diff --git
a/hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/fs/http/server/metrics/HttpFSServerMetrics.java
b/hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/fs/http/server/metrics/HttpFSServerMetrics.java
index 8c3c8c2de4..3d849f6b1c 100644
---
a/hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/fs/http/server/metrics/HttpFSServerMetrics.java
+++
b/hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/fs/http/server/metrics/HttpFSServerMetrics.java
@@ -149,4 +149,52 @@ public void incrOpsCheckAccess() {
public void shutdown() {
DefaultMetricsSystem.shutdown();
}
+
+ public long getBytesWritten() {
+ return bytesWritten.value();
+ }
+
+ public long getBytesRead() {
+ return bytesRead.value();
+ }
+
+ public long getOpsCreate() {
+ return opsCreate.value();
+ }
+
+ public long getOpsAppend() {
+ return opsAppend.value();
+ }
+
+ public long getOpsTruncate() {
+ return opsTruncate.value();
+ }
+
+ public long getOpsDelete() {
+ return opsDelete.value();
+ }
+
+ public long getOpsRename() {
+ return opsRename.value();
+ }
+
+ public long getOpsMkdir() {
+ return opsMkdir.value();
+ }
+
+ public long getOpsOpen() {
+ return opsOpen.value();
+ }
+
+ public long getOpsListing() {
+ return opsListing.value();
+ }
+
+ public long getOpsStat() {
+ return opsStat.value();
+ }
+
+ public long getOpsCheckAccess() {
+ return opsCheckAccess.value();
+ }
}
diff --git
a/hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/service/hadoop/FileSystemAccessService.java
b/hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/service/hadoop/FileSystemAccessService.java
index 986e79b7e9..2a9759bdad 100644
---
a/hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/service/hadoop/FileSystemAccessService.java
+++
b/hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/lib/service/hadoop/FileSystemAccessService.java
@@ -81,7 +81,7 @@ public class FileSystemAccessService extends BaseService
private static final String[] HADOOP_CONF_FILES
= {"core-site.xml", "hdfs-site.xml"};
- private static final String FILE_SYSTEM_SERVICE_CREATED
+ public static final String FILE_SYSTEM_SERVICE_CREATED
= "FileSystemAccessService.created";
private static class CachedFileSystem {
diff --git
a/hadoop-ozone/httpfsgateway/src/test/java/org/apache/ozone/fs/http/server/metrics/TestHttpFSMetrics.java
b/hadoop-ozone/httpfsgateway/src/test/java/org/apache/ozone/fs/http/server/metrics/TestHttpFSMetrics.java
new file mode 100644
index 0000000000..bfaf477ed3
--- /dev/null
+++
b/hadoop-ozone/httpfsgateway/src/test/java/org/apache/ozone/fs/http/server/metrics/TestHttpFSMetrics.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ozone.fs.http.server.metrics;
+
+import static
org.apache.ozone.lib.service.hadoop.FileSystemAccessService.FILE_SYSTEM_SERVICE_CREATED;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.isA;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.ozone.fs.http.server.FSOperations;
+import org.apache.ozone.fs.http.server.HttpFSServerWebApp;
+import org.apache.ozone.lib.service.FileSystemAccess;
+import org.apache.ozone.lib.service.hadoop.FileSystemAccessService;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+/**
+ * Test class for HttpFSServerMetrics.
+ */
+public class TestHttpFSMetrics {
+ private static final FileSystem mockFs = mock(FileSystem.class);
+ private static final FSDataOutputStream fsDataOutputStream =
mock(FSDataOutputStream.class);
+ public static class MockFileSystemAccessService extends
FileSystemAccessService {
+ @Override
+ protected FileSystem createFileSystem(Configuration namenodeConf) throws
IOException {
+ return mockFs;
+ }
+
+ @Override
+ protected void closeFileSystem(FileSystem fs) throws IOException {
+ // do nothing
+ }
+ }
+
+ private HttpFSServerWebApp webApp;
+ private HttpFSServerMetrics metrics;
+ private Configuration conf;
+ private UserGroupInformation ugi;
+ private FileSystemAccess fsAccess;
+
+ @BeforeAll
+ static void init(@TempDir File dir) throws Exception {
+ File tempDir = new File(dir, "temp");
+ tempDir.mkdirs();
+ File logDir = new File(dir, "log");
+ logDir.mkdirs();
+ File confDir = new File(dir, "conf");
+ confDir.mkdirs();
+ System.setProperty("httpfs.home.dir", dir.getAbsolutePath());
+ }
+
+ @BeforeEach
+ public void setUp() throws Exception {
+ conf = new Configuration();
+ conf.setBoolean(FILE_SYSTEM_SERVICE_CREATED, true);
+
+ webApp = new HttpFSServerWebApp();
+ webApp.init();
+ webApp.setService(MockFileSystemAccessService.class);
+
+ fsAccess = HttpFSServerWebApp.get().get(FileSystemAccess.class);
+ metrics = HttpFSServerWebApp.getMetrics();
+ ugi = UserGroupInformation.createUserForTesting("testuser", new String[] {
"testgroup" });
+ }
+
+ @AfterEach
+ public void tearDown() {
+ if (metrics != null) {
+ metrics.shutdown();
+ }
+ HttpFSServerWebApp.get().destroy();
+ }
+
+ @Test
+ public void testFsCreate() throws Exception {
+ long initialCreateOps = metrics.getOpsCreate();
+ long initialBytesWritten = metrics.getBytesWritten();
+
+ FSOperations.FSCreate createOp = new FSOperations.FSCreate(
+ new ByteArrayInputStream("test".getBytes()),
+ "/test.txt",
+ (short) 0644,
+ true,
+ (short) 3,
+ 1024L,
+ (short) 0644);
+
+ when(mockFs.create(isA(Path.class), isA(FsPermission.class),
isA(Boolean.class), isA(Integer.class),
+ isA(Short.class), isA(Long.class),
any())).thenReturn(fsDataOutputStream);
+ fsAccess.execute(ugi.getShortUserName(), conf, createOp);
+
+ assertEquals(initialCreateOps + 1, metrics.getOpsCreate());
+ assertEquals(initialBytesWritten + 4, metrics.getBytesWritten());
+ }
+
+ @Test
+ public void testFsAppend() throws Exception {
+ long initialAppendOps = metrics.getOpsAppend();
+ long initialBytesWritten = metrics.getBytesWritten();
+
+ FSOperations.FSAppend appendOp = new FSOperations.FSAppend(
+ new ByteArrayInputStream("test".getBytes()),
+ "/test.txt");
+
+ when(mockFs.append(isA(Path.class),
isA(Integer.class))).thenReturn(fsDataOutputStream);
+ fsAccess.execute(ugi.getShortUserName(), conf, appendOp);
+
+ assertEquals(initialAppendOps + 1, metrics.getOpsAppend());
+ assertEquals(initialBytesWritten + 4, metrics.getBytesWritten());
+ }
+}
diff --git a/hadoop-ozone/httpfsgateway/src/test/resources/log4j.properties
b/hadoop-ozone/httpfsgateway/src/test/resources/log4j.properties
new file mode 100644
index 0000000000..86cc62f67b
--- /dev/null
+++ b/hadoop-ozone/httpfsgateway/src/test/resources/log4j.properties
@@ -0,0 +1,19 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=info,stdout
+log4j.threshold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2}
(%F:%M(%L)) - %m%n
+
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
