wu-sheng commented on a change in pull request #7766: URL: https://github.com/apache/skywalking/pull/7766#discussion_r753743522
########## File path: oap-server/server-storage-plugin/storage-iotdb-plugin/pom.xml ########## @@ -0,0 +1,63 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + ~ + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <parent> + <artifactId>server-storage-plugin</artifactId> + <groupId>org.apache.skywalking</groupId> + <version>8.9.0-SNAPSHOT</version> + </parent> + <modelVersion>4.0.0</modelVersion> + + <artifactId>storage-iotdb-plugin</artifactId> + <packaging>jar</packaging> + + <properties> + </properties> + + <dependencies> + <dependency> + <groupId>org.apache.skywalking</groupId> + <artifactId>server-core</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.skywalking</groupId> + <artifactId>library-client</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>iotdb-session</artifactId> + <exclusions> + <exclusion> + <groupId>net.jpountz.lz4</groupId> + <artifactId>lz4</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.lz4</groupId> + <artifactId>lz4-java</artifactId> + <version>1.6.0</version> + </dependency> Review comment: Check the previous exclusive, the tricky thing here is, this repo changed the artifact ID. This is manual override dependency, and keep consistent with our existing dependency. But the version should be managed in the bom. ########## File path: oap-server/server-storage-plugin/storage-iotdb-plugin/pom.xml ########## @@ -0,0 +1,63 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + ~ + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <parent> + <artifactId>server-storage-plugin</artifactId> + <groupId>org.apache.skywalking</groupId> + <version>8.9.0-SNAPSHOT</version> + </parent> + <modelVersion>4.0.0</modelVersion> + + <artifactId>storage-iotdb-plugin</artifactId> + <packaging>jar</packaging> + + <properties> + </properties> + + <dependencies> + <dependency> + <groupId>org.apache.skywalking</groupId> + <artifactId>server-core</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.skywalking</groupId> + <artifactId>library-client</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>iotdb-session</artifactId> + <exclusions> + <exclusion> + <groupId>net.jpountz.lz4</groupId> + <artifactId>lz4</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.lz4</groupId> + <artifactId>lz4-java</artifactId> + <version>1.6.0</version> + </dependency> Review comment: OK, @LIU-WEI-git please recheck, I used to notice this jar in previous dependency check report. ########## File path: oap-server/server-storage-plugin/storage-iotdb-plugin/pom.xml ########## @@ -0,0 +1,63 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + ~ + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <parent> + <artifactId>server-storage-plugin</artifactId> + <groupId>org.apache.skywalking</groupId> + <version>8.9.0-SNAPSHOT</version> + </parent> + <modelVersion>4.0.0</modelVersion> + + <artifactId>storage-iotdb-plugin</artifactId> + <packaging>jar</packaging> + + <properties> + </properties> + + <dependencies> + <dependency> + <groupId>org.apache.skywalking</groupId> + <artifactId>server-core</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.skywalking</groupId> + <artifactId>library-client</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>iotdb-session</artifactId> + <exclusions> + <exclusion> + <groupId>net.jpountz.lz4</groupId> + <artifactId>lz4</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.lz4</groupId> + <artifactId>lz4-java</artifactId> + <version>1.6.0</version> + </dependency> Review comment: @kezhenxu94 I can see this now.  ########## File path: oap-server/server-storage-plugin/storage-iotdb-plugin/pom.xml ########## @@ -44,6 +44,16 @@ <dependency> <groupId>org.apache.iotdb</groupId> <artifactId>iotdb-session</artifactId> + <exclusions> + <exclusion> + <groupId>net.jpountz.lz4</groupId> + <artifactId>lz4</artifactId> + </exclusion> Review comment: @jixuan1989 We manually replaced this with `1.6.0` to avoid 2 versions existing together in the final binary. ########## File path: docs/en/setup/backend/backend-storage.md ########## @@ -280,6 +280,24 @@ storage: All connection-related settings, including URL link, username, and password are found in `application.yml`. Only part of the settings are listed here. Please follow [HikariCP](https://github.com/brettwooldridge/HikariCP) connection pool document for full settings. +## IoTDB Review comment: I think you missed the option list on the top of file. ########## File path: oap-server/server-storage-plugin/storage-iotdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/iotdb/IoTDBTableInstaller.java ########## @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.iotdb; + +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.core.storage.model.ModelInstaller; +import org.apache.skywalking.oap.server.library.client.Client; +import org.apache.skywalking.oap.server.library.module.ModuleManager; + +public class IoTDBTableInstaller extends ModelInstaller { Review comment: Could you consider to add a long comment here about what are typically tables look like? Such as including 5 types 1. Metadata, use `service_traffic` for example. 2. Metrics, use `service_cpm` 3. Trace segment 4. Log 5. Profiling snapshots With all these, when we face bugs one day, or with IoTDB improvement, we will know the expected table structure. ########## File path: oap-server/server-storage-plugin/storage-iotdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/iotdb/base/IoTDBHistoryDeleteDAO.java ########## @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.iotdb.base; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.storage.IHistoryDeleteDAO; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.storage.plugin.iotdb.IoTDBClient; +import org.joda.time.DateTime; + +import java.io.IOException; + +@Slf4j +@RequiredArgsConstructor +public class IoTDBHistoryDeleteDAO implements IHistoryDeleteDAO { + private final IoTDBClient client; + + @Override + public void deleteHistory(Model model, String timeBucketColumnName, int ttl) throws IOException { + if (log.isDebugEnabled()) { + log.debug("TTL execution log, model: {}, TTL: {}", model.getName(), ttl); + } Review comment: Do we really need logs like this? Even we need, I think we should move this to general DataTTLKeeperTimer, rather than implementation. Feel free to raise another PR to add this if needed. ########## File path: oap-server/server-storage-plugin/storage-iotdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/iotdb/base/IoTDBInsertRequest.java ########## @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.iotdb.base; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.skywalking.oap.server.core.storage.StorageData; +import org.apache.skywalking.oap.server.core.storage.StorageHashMapBuilder; +import org.apache.skywalking.oap.server.core.storage.type.StorageDataComplexObject; +import org.apache.skywalking.oap.server.library.client.request.InsertRequest; +import org.apache.skywalking.oap.server.library.client.request.UpdateRequest; +import org.apache.skywalking.oap.server.storage.plugin.iotdb.IoTDBClient; +import org.apache.skywalking.oap.server.storage.plugin.iotdb.IoTDBTableMetaInfo; + +@Getter +@Setter +@ToString +@Slf4j +public class IoTDBInsertRequest implements InsertRequest, UpdateRequest { + private String modelName; + private long time; + private List<String> indexes; + private List<String> indexValues; + private List<String> measurements; + private List<TSDataType> measurementTypes; + private List<Object> measurementValues; + + public <T extends StorageData> IoTDBInsertRequest(String modelName, long time, T storageData, + StorageHashMapBuilder<T> storageBuilder) { + this.modelName = modelName; + this.time = time; + indexes = IoTDBTableMetaInfo.get(modelName).getIndexes(); + indexValues = new ArrayList<>(indexes.size()); + Map<String, Object> storageMap = storageBuilder.entity2Storage(storageData); + + indexes.forEach(index -> { + if (index.equals(IoTDBClient.ID_IDX)) { + indexValues.add(storageData.id()); + } else if (storageMap.containsKey(index)) { + indexValues.add(String.valueOf(storageMap.get(index))); + storageMap.remove(index); + } + }); + + // time_bucket has changed to time before calling this method, so remove it from measurements + storageMap.remove(IoTDBClient.TIME_BUCKET); + // processing illegal value + Iterator<Map.Entry<String, Object>> entryIterator = storageMap.entrySet().iterator(); + while (entryIterator.hasNext()) { + Map.Entry<String, Object> entry = entryIterator.next(); + if (entry.getValue() == null) { + entryIterator.remove(); + } + if (entry.getValue() instanceof StorageDataComplexObject) { + storageMap.put(entry.getKey(), ((StorageDataComplexObject) entry.getValue()).toStorageData()); + } + } + + measurements = new ArrayList<>(storageMap.keySet()); + Map<String, TSDataType> columnAndTypeMap = IoTDBTableMetaInfo.get(modelName).getColumnAndTypeMap(); + measurementTypes = new ArrayList<>(measurements.size()); + for (String measurement : measurements) { + measurementTypes.add(columnAndTypeMap.get(measurement)); + } + measurementValues = new ArrayList<>(storageMap.values()); + + // processing illegal measurement Review comment: What do you mean illegal? I think this is a wrong word chosen? Maybe you mean to process several special fields which formats unsupported by IoTDB? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
