keith-turner commented on PR #5040:
URL: https://github.com/apache/accumulo/pull/5040#issuecomment-2465200143
Wrote the following test to experiment with these changes from a performance
perspective.
```java
package org.apache.accumulo.test.zookeeper;
import static java.nio.charset.StandardCharsets.UTF_8;
import static
org.apache.accumulo.harness.AccumuloITBase.ZOOKEEPER_TESTING_SERVER;
import java.io.File;
import java.util.List;
import java.util.UUID;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.fate.zookeeper.ZooCache;
import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter;
import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
@Tag(ZOOKEEPER_TESTING_SERVER)
public class ZooCacheIT {
private static ZooKeeperTestingServer szk = null;
private static ZooReaderWriter zk = null;
private static final String ZK_ROOT = "/accumulo/" + UUID.randomUUID();
@TempDir
private static File tempDir;
@BeforeAll
public static void setup() throws Exception {
szk = new ZooKeeperTestingServer(tempDir);
zk = szk.getZooReaderWriter();
zk.mkdirs(ZK_ROOT + Constants.ZTABLES);
}
@AfterAll
public static void teardown() throws Exception {
szk.close();
}
@Test
public void testLotsOfNodes() throws Exception {
for (int i = 0; i < 30_000; i++) {
zk.putPersistentData(ZK_ROOT + Constants.ZTABLES + "/" + i,
"i".getBytes(UTF_8),
ZooUtil.NodeExistsPolicy.FAIL);
}
var zc = new ZooCache(zk, null);
List<String> children = zc.getChildren(ZK_ROOT + Constants.ZTABLES);
Assertions.assertEquals(30_000, children.size());
for (int i = 0; i < 20; i++) {
long t1 = System.currentTimeMillis();
for (var child : children) {
zc.get(ZK_ROOT + Constants.ZTABLES + "/" + child);
}
long t2 = System.currentTimeMillis();
System.out.println(i + " : " + (t2 - t1) + "ms");
}
}
}
```
When running against these changes saw the following. Assuming the first
read from ZK its taking 22s to do 30K RPC to ZK.
```
0 : 21593ms
1 : 14ms
2 : 13ms
3 : 14ms
4 : 15ms
5 : 6ms
6 : 6ms
7 : 7ms
8 : 13ms
9 : 14ms
10 : 6ms
11 : 9ms
12 : 6ms
13 : 6ms
14 : 6ms
15 : 5ms
16 : 5ms
17 : 6ms
18 : 6ms
19 : 6ms
```
When running these test against the code that existed before these changes
saw the following where it takes 57s to read the 30K entries the first time.
So the extra CPU overhead is causing it to take 2.6 times longer.
```
0 : 56883ms
1 : 11ms
2 : 11ms
3 : 11ms
4 : 11ms
5 : 9ms
6 : 9ms
7 : 8ms
8 : 6ms
9 : 5ms
10 : 6ms
11 : 5ms
12 : 9ms
13 : 6ms
14 : 5ms
15 : 5ms
16 : 5ms
17 : 5ms
18 : 5ms
19 : 6ms
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
