This is an automated email from the ASF dual-hosted git repository.
tanxinyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 389ae8002e2 Fix region migration IT & Add daily IT (#12804)
389ae8002e2 is described below
commit 389ae8002e213f58705076415f64d07e34536d5d
Author: Li Yu Heng <[email protected]>
AuthorDate: Wed Jun 26 19:15:54 2024 +0800
Fix region migration IT & Add daily IT (#12804)
---
.github/workflows/daily-it.yml | 50 ++++++++++++++++++++
integration-test/pom.xml | 14 ++++++
.../org/apache/iotdb/itbase/category/DailyIT.java} | 22 +--------
.../IoTDBRegionMigrateReliabilityITFramework.java | 55 +++++++++++++++++++---
.../pass/IoTDBRegionMigrateClusterCrashIT.java | 3 ++
.../pass/IoTDBRegionMigrateConfigNodeCrashIT.java | 3 ++
.../pass/IoTDBRegionMigrateNormalIT.java | 3 ++
...rateCoordinatorCrashWhenRemoveRemotePeerIT.java | 3 ++
.../IoTDBRegionMigrateDataNodeCrashIT.java | 6 +++
...nMigrateOriginalCrashWhenDeleteLocalPeerIT.java | 3 ++
...MigrateOriginalCrashWhenRemoveRemotePeerIT.java | 3 ++
.../procedure/env/RegionMaintainHandler.java | 17 +++++--
12 files changed, 152 insertions(+), 30 deletions(-)
diff --git a/.github/workflows/daily-it.yml b/.github/workflows/daily-it.yml
new file mode 100644
index 00000000000..77aaa87a4bc
--- /dev/null
+++ b/.github/workflows/daily-it.yml
@@ -0,0 +1,50 @@
+name: Daily IT
+
+on:
+ schedule:
+ # Run at UTC 19:00 every day (CST 03:00 AM)
+ - cron: '0 19 * * *'
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+env:
+ MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
-Dmaven.wagon.http.retryHandler.class=standard
-Dmaven.wagon.http.retryHandler.count=3
+ DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+
+jobs:
+ Simple:
+ strategy:
+ fail-fast: false
+ max-parallel: 20
+ matrix:
+ java: [ 8, 17 ]
+ runs-on: [self-hosted, iotdb]
+ # group: self-hosted
+ # labels: iotdb
+ steps:
+ - uses: actions/checkout@v4
+ - name: Set up JDK ${{ matrix.java }}
+ uses: actions/setup-java@v4
+ with:
+ distribution: liberica
+ java-version: ${{ matrix.java }}
+ - name: IT/UT Test
+ shell: bash
+ # we do not compile client-cpp for saving time, it is tested in
client.yml
+ # we can skip influxdb-protocol because it has been tested separately
in influxdb-protocol.yml
+ run: |
+ mvn clean verify \
+ -P with-integration-tests \
+ -DskipUTs \
+ -DintegrationTest.forkCount=6 -DConfigNodeMaxHeapSize=1024
-DDataNodeMaxHeapSize=1024 \
+ -pl integration-test \
+ -am -PDailyIT
+ - name: Upload Artifact
+ if: failure()
+ uses: actions/upload-artifact@v4
+ with:
+ name: cluster-log-java${{ matrix.java }}-${{ runner.os }}
+ path: integration-test/target/cluster-logs
+ retention-days: 3
diff --git a/integration-test/pom.xml b/integration-test/pom.xml
index 68806474fc1..896a9bfa940 100644
--- a/integration-test/pom.xml
+++ b/integration-test/pom.xml
@@ -506,5 +506,19 @@
<integrationTest.testEnv>Cluster1</integrationTest.testEnv>
</properties>
</profile>
+ <profile>
+ <id>DailyIT</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <properties>
+ <integrationTest.excludedGroups/>
+
<integrationTest.includedGroups>org.apache.iotdb.itbase.category.DailyIT</integrationTest.includedGroups>
+
<integrationTest.launchNodeInSameJVM>false</integrationTest.launchNodeInSameJVM>
+
<integrationTest.randomSelectWriteNode>true</integrationTest.randomSelectWriteNode>
+
<integrationTest.readAndVerifyWithMultiNode>true</integrationTest.readAndVerifyWithMultiNode>
+ <integrationTest.testEnv>Cluster1</integrationTest.testEnv>
+ </properties>
+ </profile>
</profiles>
</project>
diff --git
a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateNormalIT.java
b/integration-test/src/main/java/org/apache/iotdb/itbase/category/DailyIT.java
similarity index 52%
copy from
integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateNormalIT.java
copy to
integration-test/src/main/java/org/apache/iotdb/itbase/category/DailyIT.java
index f6e31d08304..d1061ce9c3a 100644
---
a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateNormalIT.java
+++
b/integration-test/src/main/java/org/apache/iotdb/itbase/category/DailyIT.java
@@ -17,24 +17,6 @@
* under the License.
*/
-package org.apache.iotdb.confignode.it.regionmigration.pass;
+package org.apache.iotdb.itbase.category;
-import org.apache.iotdb.commons.utils.KillPoint.KillNode;
-import
org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionMigrateReliabilityITFramework;
-import org.apache.iotdb.it.framework.IoTDBTestRunner;
-
-import org.junit.Test;
-import org.junit.runner.RunWith;
-
-@RunWith(IoTDBTestRunner.class)
-public class IoTDBRegionMigrateNormalIT extends
IoTDBRegionMigrateReliabilityITFramework {
- @Test
- public void normal1C2DTest() throws Exception {
- successTest(1, 1, 1, 2, noKillPoints(), noKillPoints(),
KillNode.ALL_NODES);
- }
-
- @Test
- public void normal3C3DTest() throws Exception {
- successTest(2, 3, 3, 3, noKillPoints(), noKillPoints(),
KillNode.ALL_NODES);
- }
-}
+public interface DailyIT {}
diff --git
a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/IoTDBRegionMigrateReliabilityITFramework.java
b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/IoTDBRegionMigrateReliabilityITFramework.java
index 15d396f8d46..d52251b4a1a 100644
---
a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/IoTDBRegionMigrateReliabilityITFramework.java
+++
b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/IoTDBRegionMigrateReliabilityITFramework.java
@@ -52,6 +52,9 @@ import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Proxy;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
@@ -205,11 +208,10 @@ public class IoTDBRegionMigrateReliabilityITFramework {
EnvFactory.getEnv().registerDataNodeKillPoints(new
ArrayList<>(dataNodeKeywords));
EnvFactory.getEnv().initClusterEnvironment(configNodeNum, dataNodeNum);
- try (final Connection connection = EnvFactory.getEnv().getConnection();
- final Statement statement = connection.createStatement();
+ try (final Connection connection =
closeQuietly(EnvFactory.getEnv().getConnection());
+ final Statement statement = closeQuietly(connection.createStatement());
SyncConfigNodeIServiceClient client =
(SyncConfigNodeIServiceClient)
EnvFactory.getEnv().getLeaderConfigNodeConnection()) {
-
statement.execute(INSERTION1);
ResultSet result = statement.executeQuery(SHOW_REGIONS);
@@ -297,10 +299,11 @@ public class IoTDBRegionMigrateReliabilityITFramework {
checkRegionFileExistIfNodeAlive(originalDataNode);
checkPeersClearIfNodeAlive(allDataNodeId, destDataNode,
selectedRegion);
}
+
+ LOGGER.info("test pass");
} catch (InconsistentDataException ignore) {
}
- LOGGER.info("test pass");
}
private void restartDataNodes(List<DataNodeWrapper> dataNodeWrappers) {
@@ -519,7 +522,7 @@ public class IoTDBRegionMigrateReliabilityITFramework {
AtomicReference<SyncConfigNodeIServiceClient> clientRef = new
AtomicReference<>(client);
try {
Awaitility.await()
- .atMost(1, TimeUnit.MINUTES)
+ .atMost(2, TimeUnit.MINUTES)
.pollDelay(2, TimeUnit.SECONDS)
.until(
() -> {
@@ -585,7 +588,15 @@ public class IoTDBRegionMigrateReliabilityITFramework {
private static void checkRegionFileClear(int dataNode) {
File originalRegionDir = new File(buildRegionDirPath(dataNode));
Assert.assertTrue(originalRegionDir.isDirectory());
- Assert.assertEquals(0,
Objects.requireNonNull(originalRegionDir.listFiles()).length);
+ try {
+ Assert.assertEquals(0,
Objects.requireNonNull(originalRegionDir.listFiles()).length);
+ } catch (AssertionError e) {
+ LOGGER.error(
+ "Original DataNode {} region file not clear, these files is still
remain: {}",
+ dataNode,
+ Arrays.toString(originalRegionDir.listFiles()));
+ throw e;
+ }
LOGGER.info("Original DataNode {} region file clear", dataNode);
}
@@ -649,9 +660,16 @@ public class IoTDBRegionMigrateReliabilityITFramework {
private void checkClusterStillWritable() {
try (Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {
- statement.execute(INSERTION2);
+ // check old data
ResultSet resultSet = statement.executeQuery(COUNT_TIMESERIES);
resultSet.next();
+ Assert.assertEquals(1, resultSet.getLong(1));
+ Assert.assertEquals(1, resultSet.getLong(2));
+ LOGGER.info("Old data is still remain");
+ // write new data
+ statement.execute(INSERTION2);
+ resultSet = statement.executeQuery(COUNT_TIMESERIES);
+ resultSet.next();
Assert.assertEquals(2, resultSet.getLong(1));
Assert.assertEquals(2, resultSet.getLong(2));
LOGGER.info("Region group is still writable");
@@ -708,4 +726,27 @@ public class IoTDBRegionMigrateReliabilityITFramework {
Arrays.stream(keywords).map(KillPoint::enumToString).collect(Collectors.toList()));
return result;
}
+
+ private static <T> T closeQuietly(T t) {
+ InvocationHandler handler =
+ (proxy, method, args) -> {
+ try {
+ if (method.getName().equals("close")) {
+ try {
+ method.invoke(t, args);
+ } catch (Throwable e) {
+ LOGGER.warn("Exception happens during close(): ", e);
+ }
+ return null;
+ } else {
+ return method.invoke(t, args);
+ }
+ } catch (InvocationTargetException e) {
+ throw e.getTargetException();
+ }
+ };
+ return (T)
+ Proxy.newProxyInstance(
+ t.getClass().getClassLoader(), t.getClass().getInterfaces(),
handler);
+ }
}
diff --git
a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateClusterCrashIT.java
b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateClusterCrashIT.java
index be18a7f54d7..da6bf2dfdc6 100644
---
a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateClusterCrashIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateClusterCrashIT.java
@@ -23,10 +23,13 @@ import
org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionMigrateReliabil
import org.apache.iotdb.confignode.procedure.state.AddRegionPeerState;
import org.apache.iotdb.confignode.procedure.state.RemoveRegionPeerState;
import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.DailyIT;
import org.junit.Test;
+import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
+@Category({DailyIT.class})
@RunWith(IoTDBTestRunner.class)
public class IoTDBRegionMigrateClusterCrashIT extends
IoTDBRegionMigrateReliabilityITFramework {
diff --git
a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateConfigNodeCrashIT.java
b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateConfigNodeCrashIT.java
index 9baeab50877..f6916eff402 100644
---
a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateConfigNodeCrashIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateConfigNodeCrashIT.java
@@ -26,15 +26,18 @@ import
org.apache.iotdb.confignode.procedure.state.AddRegionPeerState;
import org.apache.iotdb.confignode.procedure.state.RegionTransitionState;
import org.apache.iotdb.confignode.procedure.state.RemoveRegionPeerState;
import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.DailyIT;
import org.junit.Ignore;
import org.junit.Test;
+import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import java.util.Arrays;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
+@Category({DailyIT.class})
@RunWith(IoTDBTestRunner.class)
public class IoTDBRegionMigrateConfigNodeCrashIT extends
IoTDBRegionMigrateReliabilityITFramework {
@Test
diff --git
a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateNormalIT.java
b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateNormalIT.java
index f6e31d08304..f56296b9cfa 100644
---
a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateNormalIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/IoTDBRegionMigrateNormalIT.java
@@ -22,10 +22,13 @@ package org.apache.iotdb.confignode.it.regionmigration.pass;
import org.apache.iotdb.commons.utils.KillPoint.KillNode;
import
org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionMigrateReliabilityITFramework;
import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.ClusterIT;
import org.junit.Test;
+import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
+@Category({ClusterIT.class})
@RunWith(IoTDBTestRunner.class)
public class IoTDBRegionMigrateNormalIT extends
IoTDBRegionMigrateReliabilityITFramework {
@Test
diff --git
a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/datanodecrash/IoTDBRegionMigrateCoordinatorCrashWhenRemoveRemotePeerIT.java
b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/datanodecrash/IoTDBRegionMigrateCoordinatorCrashWhenRemoveRemotePeerIT.java
index 5a63e76f979..11fd4552fec 100644
---
a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/datanodecrash/IoTDBRegionMigrateCoordinatorCrashWhenRemoveRemotePeerIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/datanodecrash/IoTDBRegionMigrateCoordinatorCrashWhenRemoveRemotePeerIT.java
@@ -22,10 +22,13 @@ package
org.apache.iotdb.confignode.it.regionmigration.pass.datanodecrash;
import
org.apache.iotdb.commons.utils.KillPoint.IoTConsensusRemovePeerCoordinatorKillPoints;
import
org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionMigrateDataNodeCrashITFramework;
import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.DailyIT;
import org.junit.Test;
+import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
+@Category({DailyIT.class})
@RunWith(IoTDBTestRunner.class)
public class IoTDBRegionMigrateCoordinatorCrashWhenRemoveRemotePeerIT
extends IoTDBRegionMigrateDataNodeCrashITFramework {
diff --git
a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/datanodecrash/IoTDBRegionMigrateDataNodeCrashIT.java
b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/datanodecrash/IoTDBRegionMigrateDataNodeCrashIT.java
index a5c619ec704..6819af9f87b 100644
---
a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/datanodecrash/IoTDBRegionMigrateDataNodeCrashIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/datanodecrash/IoTDBRegionMigrateDataNodeCrashIT.java
@@ -22,9 +22,15 @@ package
org.apache.iotdb.confignode.it.regionmigration.pass.datanodecrash;
import org.apache.iotdb.commons.utils.KillPoint.DataNodeKillPoints;
import org.apache.iotdb.commons.utils.KillPoint.KillNode;
import
org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionMigrateReliabilityITFramework;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.DailyIT;
import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+@Category({DailyIT.class})
+@RunWith(IoTDBTestRunner.class)
public class IoTDBRegionMigrateDataNodeCrashIT extends
IoTDBRegionMigrateReliabilityITFramework {
// region Coordinator DataNode crash tests
diff --git
a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/datanodecrash/IoTDBRegionMigrateOriginalCrashWhenDeleteLocalPeerIT.java
b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/datanodecrash/IoTDBRegionMigrateOriginalCrashWhenDeleteLocalPeerIT.java
index a21884b5da2..047037c7d14 100644
---
a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/datanodecrash/IoTDBRegionMigrateOriginalCrashWhenDeleteLocalPeerIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/datanodecrash/IoTDBRegionMigrateOriginalCrashWhenDeleteLocalPeerIT.java
@@ -22,10 +22,13 @@ package
org.apache.iotdb.confignode.it.regionmigration.pass.datanodecrash;
import
org.apache.iotdb.commons.utils.KillPoint.IoTConsensusDeleteLocalPeerKillPoints;
import
org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionMigrateDataNodeCrashITFramework;
import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.DailyIT;
import org.junit.Test;
+import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
+@Category({DailyIT.class})
@RunWith(IoTDBTestRunner.class)
public class IoTDBRegionMigrateOriginalCrashWhenDeleteLocalPeerIT
extends IoTDBRegionMigrateDataNodeCrashITFramework {
diff --git
a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/datanodecrash/IoTDBRegionMigrateOriginalCrashWhenRemoveRemotePeerIT.java
b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/datanodecrash/IoTDBRegionMigrateOriginalCrashWhenRemoveRemotePeerIT.java
index cf6e0eb19e6..69ea8c27088 100644
---
a/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/datanodecrash/IoTDBRegionMigrateOriginalCrashWhenRemoveRemotePeerIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/confignode/it/regionmigration/pass/datanodecrash/IoTDBRegionMigrateOriginalCrashWhenRemoveRemotePeerIT.java
@@ -22,10 +22,13 @@ package
org.apache.iotdb.confignode.it.regionmigration.pass.datanodecrash;
import
org.apache.iotdb.commons.utils.KillPoint.IoTConsensusInactivatePeerKillPoints;
import
org.apache.iotdb.confignode.it.regionmigration.IoTDBRegionMigrateDataNodeCrashITFramework;
import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.DailyIT;
import org.junit.Test;
+import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
+@Category({DailyIT.class})
@RunWith(IoTDBTestRunner.class)
public class IoTDBRegionMigrateOriginalCrashWhenRemoveRemotePeerIT
extends IoTDBRegionMigrateDataNodeCrashITFramework {
diff --git
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandler.java
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandler.java
index 9bb5b860eb9..7ef330f460f 100644
---
a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandler.java
+++
b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandler.java
@@ -667,9 +667,20 @@ public class RegionMaintainHandler {
*/
public void transferRegionLeader(TConsensusGroupId regionId,
TDataNodeLocation originalDataNode)
throws ProcedureException {
- Optional<TDataNodeLocation> newLeaderNode =
- filterDataNodeWithOtherRegionReplica(regionId, originalDataNode);
- newLeaderNode.orElseThrow(() -> new ProcedureException("Cannot find the
new leader"));
+ // find new leader
+ final int findNewLeaderTimeLimitSecond = 10;
+ long startTime = System.nanoTime();
+ Optional<TDataNodeLocation> newLeaderNode = Optional.empty();
+ while (System.nanoTime() - startTime <
TimeUnit.SECONDS.toNanos(findNewLeaderTimeLimitSecond)) {
+ newLeaderNode = filterDataNodeWithOtherRegionReplica(regionId,
originalDataNode);
+ if (newLeaderNode.isPresent()) {
+ break;
+ }
+ }
+ newLeaderNode.orElseThrow(
+ () ->
+ new ProcedureException(
+ "Cannot find the new leader after " +
findNewLeaderTimeLimitSecond + " seconds"));
// ratis needs DataNode to do election by itself
long timestamp = System.nanoTime();
