This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch rel/0.13
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/0.13 by this push:
new 4acc508d27 [To rel/0.13] [IOTDB-3662] add audit log. (#7717)
4acc508d27 is described below
commit 4acc508d27fb13ee141d3c5f7fdfde898f36e5f0
Author: Zhijia Cao <[email protected]>
AuthorDate: Fri Oct 28 18:35:04 2022 +0800
[To rel/0.13] [IOTDB-3662] add audit log. (#7717)
Co-authored-by: Haonan <[email protected]>
---
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 1 +
.../server/basic/ClusterServiceProvider.java | 5 ++
.../resources/conf/iotdb-engine.properties | 10 ++-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 23 ++++++
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 7 ++
.../iotdb/db/service/basic/ServiceProvider.java | 3 +
.../service/basic/StandaloneServiceProvider.java | 6 +-
.../db/service/thrift/impl/TSServiceImpl.java | 3 +-
.../org/apache/iotdb/db/utils/AuditLogUtils.java | 83 ++++++++++++++++++++++
9 files changed, 138 insertions(+), 3 deletions(-)
diff --git a/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
b/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
index 7996345033..e4905bb8fd 100644
--- a/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
+++ b/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
@@ -745,6 +745,7 @@ nodeName
| wildcard? INTEGER_LITERAL wildcard?
| QUTOED_ID_IN_NODE_NAME
| STRING_LITERAL
+ | SYSTEM
;
nodeNameWithoutWildcard
diff --git
a/cluster/src/main/java/org/apache/iotdb/cluster/server/basic/ClusterServiceProvider.java
b/cluster/src/main/java/org/apache/iotdb/cluster/server/basic/ClusterServiceProvider.java
index be515775f4..37d579e21b 100644
---
a/cluster/src/main/java/org/apache/iotdb/cluster/server/basic/ClusterServiceProvider.java
+++
b/cluster/src/main/java/org/apache/iotdb/cluster/server/basic/ClusterServiceProvider.java
@@ -32,6 +32,7 @@ import org.apache.iotdb.db.qp.physical.sys.FlushPlan;
import org.apache.iotdb.db.qp.physical.sys.SetSystemModePlan;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.service.basic.ServiceProvider;
+import org.apache.iotdb.db.utils.AuditLogUtils;
import org.apache.iotdb.rpc.RpcUtils;
import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.service.rpc.thrift.TSStatus;
@@ -89,6 +90,10 @@ public class ClusterServiceProvider extends ServiceProvider {
@Override
public boolean executeNonQuery(PhysicalPlan plan) {
+ AuditLogUtils.writeAuditLog(
+ plan.getOperatorName(),
+ String.format(
+ "measurements size:%s", plan.getPaths() == null ? 0 :
plan.getPaths().size()));
TSStatus tsStatus = executeNonQueryPlan(plan);
return tsStatus.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode();
}
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties
b/server/src/assembly/resources/conf/iotdb-engine.properties
index f8267b9c38..6e44b96b59 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -949,4 +949,12 @@ timestamp_precision=ms
### Group By Fill Configuration
####################
# Datatype: float
-# group_by_fill_cache_size_in_mb=1.0
\ No newline at end of file
+# group_by_fill_cache_size_in_mb=1.0
+
+# determines whether audit logs are written to log files or IoTDB or not
write.choice:LOGGER/IOTDB/NONE
+# Datatype: String
+# audit_log_storage=NONE
+
+# determines whether audit logs record IoTDB write operation
+# Datatype: boolean
+# enable_auditLog_write=false
\ No newline at end of file
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index c305c49cf4..50d9486a3e 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -27,6 +27,7 @@ import
org.apache.iotdb.db.exception.LoadConfigurationException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.service.thrift.impl.InfluxDBServiceImpl;
import org.apache.iotdb.db.service.thrift.impl.TSServiceImpl;
+import org.apache.iotdb.db.utils.AuditLogUtils;
import org.apache.iotdb.rpc.RpcTransportFactory;
import org.apache.iotdb.rpc.RpcUtils;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
@@ -873,6 +874,12 @@ public class IoTDBConfig {
/** number of threads given to archiving tasks */
private int archivingThreadNum = 2;
+ // determines whether audit logs are written to log files or IoTDB
+ private String auditLogStorage = AuditLogUtils.LOG_LEVEL_NONE;
+
+ // determines whether audit logs record IoTDB write operation
+ private boolean enableAuditLogWrite = false;
+
// customizedProperties, this should be empty by default.
private Properties customizedProperties = new Properties();
@@ -2806,4 +2813,20 @@ public class IoTDBConfig {
public void setPatternMatchingThreshold(int patternMatchingThreshold) {
this.patternMatchingThreshold = patternMatchingThreshold;
}
+
+ public String getAuditLogStorage() {
+ return auditLogStorage;
+ }
+
+ public void setAuditLogStorage(String auditLogStorage) {
+ this.auditLogStorage = auditLogStorage;
+ }
+
+ public boolean isEnableAuditLogWrite() {
+ return enableAuditLogWrite;
+ }
+
+ public void setEnableAuditLogWrite(boolean enableAuditLogWrite) {
+ this.enableAuditLogWrite = enableAuditLogWrite;
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 3ed1f5e726..d12c871780 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -891,6 +891,13 @@ public class IoTDBDescriptor {
properties.getProperty(
"schema_query_fetch_size",
String.valueOf(conf.getSchemaQueryFetchSize()))));
+ conf.setAuditLogStorage(properties.getProperty("audit_log_storage",
conf.getAuditLogStorage()));
+
+ conf.setEnableAuditLogWrite(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "enable_auditLog_write",
String.valueOf(conf.isEnableAuditLogWrite()))));
+
// At the same time, set TSFileConfig
TSFileDescriptor.getInstance()
.getConfig()
diff --git
a/server/src/main/java/org/apache/iotdb/db/service/basic/ServiceProvider.java
b/server/src/main/java/org/apache/iotdb/db/service/basic/ServiceProvider.java
index 3b37af1ae7..630e8d3dad 100644
---
a/server/src/main/java/org/apache/iotdb/db/service/basic/ServiceProvider.java
+++
b/server/src/main/java/org/apache/iotdb/db/service/basic/ServiceProvider.java
@@ -43,6 +43,7 @@ import org.apache.iotdb.db.query.control.SessionManager;
import org.apache.iotdb.db.query.control.SessionTimeoutManager;
import org.apache.iotdb.db.query.control.clientsession.IClientSession;
import org.apache.iotdb.db.query.control.tracing.TracingManager;
+import org.apache.iotdb.db.utils.AuditLogUtils;
import org.apache.iotdb.rpc.RpcUtils;
import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.service.rpc.thrift.TSProtocolVersion;
@@ -219,6 +220,7 @@ public abstract class ServiceProvider {
// TODO we should close this connection ASAP, otherwise there will be
DDoS.
}
SessionTimeoutManager.getInstance().register(session);
+ AuditLogUtils.writeAuditLog(AuditLogUtils.TYPE_LOGIN, "user login");
return openSessionResp.sessionId(session == null ? -1 : session.getId());
}
@@ -235,6 +237,7 @@ public abstract class ServiceProvider {
public boolean closeSession(IClientSession session) {
AUDIT_LOGGER.info("Session-{} is closing", session);
+ AuditLogUtils.writeAuditLog(AuditLogUtils.TYPE_LOGOUT, "user logout");
return SessionTimeoutManager.getInstance().unregister(session);
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/service/basic/StandaloneServiceProvider.java
b/server/src/main/java/org/apache/iotdb/db/service/basic/StandaloneServiceProvider.java
index a931ffe27b..8ad1f16e60 100644
---
a/server/src/main/java/org/apache/iotdb/db/service/basic/StandaloneServiceProvider.java
+++
b/server/src/main/java/org/apache/iotdb/db/service/basic/StandaloneServiceProvider.java
@@ -28,6 +28,7 @@ import org.apache.iotdb.db.qp.physical.PhysicalPlan;
import org.apache.iotdb.db.qp.physical.sys.FlushPlan;
import org.apache.iotdb.db.qp.physical.sys.SetSystemModePlan;
import org.apache.iotdb.db.query.context.QueryContext;
+import org.apache.iotdb.db.utils.AuditLogUtils;
public class StandaloneServiceProvider extends ServiceProvider {
@@ -50,7 +51,10 @@ public class StandaloneServiceProvider extends
ServiceProvider {
&& IoTDBDescriptor.getInstance().getConfig().isReadOnly()) {
throw new StorageEngineReadonlyException();
}
-
+ AuditLogUtils.writeAuditLog(
+ plan.getOperatorName(),
+ String.format(
+ "measurements size:%s", plan.getPaths() == null ? 0 :
plan.getPaths().size()));
return executor.processNonQuery(plan);
}
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/TSServiceImpl.java
b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/TSServiceImpl.java
index eefc352bc3..3f98f7a06d 100644
---
a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/TSServiceImpl.java
+++
b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/TSServiceImpl.java
@@ -78,6 +78,7 @@ import org.apache.iotdb.db.service.metrics.MetricService;
import org.apache.iotdb.db.service.metrics.enums.Operation;
import org.apache.iotdb.db.tools.watermark.GroupedLSBWatermarkEncoder;
import org.apache.iotdb.db.tools.watermark.WatermarkEncoder;
+import org.apache.iotdb.db.utils.AuditLogUtils;
import org.apache.iotdb.db.utils.QueryDataSetUtils;
import org.apache.iotdb.metrics.config.MetricConfigDescriptor;
import org.apache.iotdb.metrics.utils.MetricLevel;
@@ -779,7 +780,7 @@ public class TSServiceImpl implements TSIService.Iface {
if (status != null) {
return new TSExecuteStatementResp(status);
}
-
+ AuditLogUtils.writeAuditLog(AuditLogUtils.TYPE_QUERY, req.getStatement());
QueryTask queryTask =
new QueryTask(
physicalPlan,
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/AuditLogUtils.java
b/server/src/main/java/org/apache/iotdb/db/utils/AuditLogUtils.java
new file mode 100644
index 0000000000..41786e9303
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/utils/AuditLogUtils.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.utils;
+
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBConstant;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.exception.metadata.IllegalPathException;
+import org.apache.iotdb.db.exception.query.QueryProcessException;
+import org.apache.iotdb.db.metadata.path.PartialPath;
+import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan;
+import org.apache.iotdb.db.qp.utils.DateTimeUtils;
+import org.apache.iotdb.db.query.control.SessionManager;
+import org.apache.iotdb.db.query.control.clientsession.IClientSession;
+import org.apache.iotdb.db.service.IoTDB;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class AuditLogUtils {
+ private static final Logger logger =
LoggerFactory.getLogger(AuditLogUtils.class);
+ private static final Logger AUDIT_LOGGER =
+ LoggerFactory.getLogger(IoTDBConstant.AUDIT_LOGGER_NAME);
+
+ public static final String LOG = "log";
+ public static final String TYPE = "type";
+ public static final String AUDIT_LOG_DEVICE = "root.system.audit.'%s'";
+ public static final String TYPE_QUERY = "QUERY";
+ public static final String TYPE_LOGIN = "LOGIN";
+ public static final String TYPE_LOGOUT = "LOGOUT";
+ public static final String TYPE_INSERT = "INSERT";
+ public static final String TYPE_DELETE = "DELETE";
+ public static final String LOG_LEVEL_IOTDB = "IOTDB";
+ public static final String LOG_LEVEL_LOGGER = "LOGGER";
+ public static final String LOG_LEVEL_NONE = "NONE";
+
+ public static void writeAuditLog(String type, String log) {
+ IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
+ String auditLogStorage = config.getAuditLogStorage();
+ IClientSession currSession = SessionManager.getInstance().getCurrSession();
+ if (currSession == null) {
+ return;
+ }
+ String username = currSession.getUsername();
+ if (LOG_LEVEL_IOTDB.equals(auditLogStorage)) {
+ try {
+ InsertRowPlan insertRowPlan =
+ new InsertRowPlan(
+ new PartialPath(String.format(AUDIT_LOG_DEVICE, username)),
+ DateTimeUtils.currentTime(),
+ new String[] {TYPE, LOG},
+ new String[] {type, log});
+ IoTDB.serviceProvider.getExecutor().insert(insertRowPlan);
+ } catch (IllegalPathException | QueryProcessException e) {
+ logger.error("write audit log series error,", e);
+ }
+ } else if (LOG_LEVEL_LOGGER.equals(auditLogStorage)) {
+ if (type.contains(TYPE_INSERT)) {
+ if (config.isEnableAuditLogWrite()) {
+ AUDIT_LOGGER.debug("user:{},type:{},action:{}", username, type, log);
+ }
+ } else {
+ AUDIT_LOGGER.info("user:{},type:{},action:{}", username, type, log);
+ }
+ }
+ }
+}
