This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch DeleteWatermark012 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit a9414d6d25ad1a534b71ae489ac3fca13174d71e Author: Jackie Tien <[email protected]> AuthorDate: Fri Jun 16 16:39:32 2023 +0800 Remove water mark related configurarions in iotdb-common.properties (#10179) (cherry picked from commit bff402b99d36f40c6d82d5f46cdccdcb042cb09e) --- docs/UserGuide/Maintenance-Tools/Watermark-Tool.md | 196 ------------------ .../UserGuide/Maintenance-Tools/Watermark-Tool.md | 196 ------------------ integration-test/import-control.xml | 1 - .../iotdb/it/env/cluster/MppCommonConfig.java | 24 --- .../it/env/cluster/MppSharedCommonConfig.java | 28 --- .../iotdb/it/env/remote/RemoteCommonConfig.java | 20 -- .../org/apache/iotdb/itbase/env/CommonConfig.java | 8 - .../iotdb/db/it/watermark/IoTDBWatermarkIT.java | 229 --------------------- .../apache/iotdb/jdbc/IoTDBDatabaseMetadata.java | 14 -- .../iotdb/jdbc/IoTDBDatabaseMetadataTest.java | 4 - .../thrift/src/main/thrift/client.thrift | 12 +- .../resources/conf/iotdb-common.properties | 16 -- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 75 ------- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 10 - .../service/thrift/impl/ClientRPCServiceImpl.java | 8 - .../watermark/GroupedLSBWatermarkEncoder.java | 150 -------------- .../db/tools/watermark/WatermarkDetector.java | 220 -------------------- .../iotdb/db/tools/watermark/WatermarkEncoder.java | 36 ---- 18 files changed, 4 insertions(+), 1243 deletions(-) diff --git a/docs/UserGuide/Maintenance-Tools/Watermark-Tool.md b/docs/UserGuide/Maintenance-Tools/Watermark-Tool.md deleted file mode 100644 index 02a12042885..00000000000 --- a/docs/UserGuide/Maintenance-Tools/Watermark-Tool.md +++ /dev/null @@ -1,196 +0,0 @@ -<!-- -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. ---> - -## Watermark Tool - -This tool has two functions: 1) watermark embedding of the IoTDB query result and 2) watermark detection of the suspected data. - -### Watermark Embedding - -#### Configuration - -Watermark is disabled by default in IoTDB. To enable watermark embedding, the first thing is to modify the following fields in the configuration file `iotdb-datanode.properties`: - -| Name | Example | Explanation | -| ----------------------- | ------------------------------------------------------ | ------------------------------------------------------------ | -| watermark_module_opened | false | `true` to enable watermark embedding of the IoTDB server; `false` to disable | -| watermark_secret_key | IoTDB*2019@Beijing | self-defined secret key | -| watermark_bit_string | 100101110100 | 0-1 bit string to be embedded | -| watermark_method | GroupBasedLSBMethod(embed_row_cycle=2,embed_lsb_num=5) | specifies the watermark algorithm and its paramters | - -Notes: - -- `watermark_module_opened`: Set it to be true if you want to enable watermark embedding -- `watermark_secret_key`: Character '&' is not allowed. There is no constraint on the length of the secret key. Generally, the longer the key is, the higher the bar to intruders. -- `watermark_bit_string`: There is no constraint on the length of the bit string (except that it should not be empty). But note that it is difficult to reach the required significance level at the watermark detection phase if the bit string is way too short. -- `watermark_method`: Now only GroupBasedLSBMethod is supported, so actually you can only tune the two parameters of this method, which are `embed_row_cycle` and `embed_lsb_num`. - - Both of them should be positive integers. - - `embed_row_cycle` controls the ratio of rows watermarked. The smaller the `embed_row_cycle`, the larger the ratio of rows watermarked. When `embed_row_cycle` equals 1, every row is watermarked. - - GroupBasedLSBMethod uses LSB embedding. `embed_lsb_num` controls the number of least significant bits available for watermark embedding. The biggger the `embed_lsb_num`, the larger the varying range of a data point. -- `watermark_secret_key`, `watermark_bit_string` and `watermark_method` should be kept secret from possible attackers. That is, it is your responsiblity to take care of `iotdb-datanode.properties`. - -#### Usage Example - -* step 1. Create a new user Alice, grant read privilege and query - -A newly created user doesn't use watermark by default. So the query result is the original data. - -``` -.\start-cli.bat -u root -pw root -create user Alice 1234 -grant user Alice privileges READ_TIMESERIES on root.vehicle -exit - -.\start-cli.bat -u Alice -pw 1234 -select * from root -+-----------------------------------+------------------+ -| Time|root.vehicle.d0.s0| -+-----------------------------------+------------------+ -| 1970-01-01T08:00:00.001+08:00| 21.5| -| 1970-01-01T08:00:00.002+08:00| 22.5| -| 1970-01-01T08:00:00.003+08:00| 23.5| -| 1970-01-01T08:00:00.004+08:00| 24.5| -| 1970-01-01T08:00:00.005+08:00| 25.5| -| 1970-01-01T08:00:00.006+08:00| 26.5| -| 1970-01-01T08:00:00.007+08:00| 27.5| -| 1970-01-01T08:00:00.008+08:00| 28.5| -| 1970-01-01T08:00:00.009+08:00| 29.5| -| 1970-01-01T08:00:00.010+08:00| 30.5| -| 1970-01-01T08:00:00.011+08:00| 31.5| -| 1970-01-01T08:00:00.012+08:00| 32.5| -| 1970-01-01T08:00:00.013+08:00| 33.5| -| 1970-01-01T08:00:00.014+08:00| 34.5| -| 1970-01-01T08:00:00.015+08:00| 35.5| -| 1970-01-01T08:00:00.016+08:00| 36.5| -| 1970-01-01T08:00:00.017+08:00| 37.5| -| 1970-01-01T08:00:00.018+08:00| 38.5| -| 1970-01-01T08:00:00.019+08:00| 39.5| -| 1970-01-01T08:00:00.020+08:00| 40.5| -| 1970-01-01T08:00:00.021+08:00| 41.5| -| 1970-01-01T08:00:00.022+08:00| 42.5| -| 1970-01-01T08:00:00.023+08:00| 43.5| -| 1970-01-01T08:00:00.024+08:00| 44.5| -| 1970-01-01T08:00:00.025+08:00| 45.5| -| 1970-01-01T08:00:00.026+08:00| 46.5| -| 1970-01-01T08:00:00.027+08:00| 47.5| -| 1970-01-01T08:00:00.028+08:00| 48.5| -| 1970-01-01T08:00:00.029+08:00| 49.5| -| 1970-01-01T08:00:00.030+08:00| 50.5| -| 1970-01-01T08:00:00.031+08:00| 51.5| -| 1970-01-01T08:00:00.032+08:00| 52.5| -| 1970-01-01T08:00:00.033+08:00| 53.5| -+-----------------------------------+------------------+ -``` - -* step 2. grant watermark_embedding to Alice - -Usage: `grant watermark_embedding to Alice` - -Note that you can use `grant watermark_embedding to user1,user2,...` to grant watermark_embedding to multiple users. - -Only root can run this command. After root grants watermark_embedding to Alice, all query results of Alice are watermarked. - -``` -.\start-cli.bat -u root -pw root -grant watermark_embedding to Alice -exit - -.\start-cli.bat -u Alice -pw '1234' -select * from root -+-----------------------------------+------------------+ -| Time|root.vehicle.d0.s0| -+-----------------------------------+------------------+ -| 1970-01-01T08:00:00.001+08:00| 21.5| -| 1970-01-01T08:00:00.002+08:00| 22.5| -| 1970-01-01T08:00:00.003+08:00| 23.500008| -| 1970-01-01T08:00:00.004+08:00| 24.500015| -| 1970-01-01T08:00:00.005+08:00| 25.5| -| 1970-01-01T08:00:00.006+08:00| 26.500015| -| 1970-01-01T08:00:00.007+08:00| 27.5| -| 1970-01-01T08:00:00.008+08:00| 28.500004| -| 1970-01-01T08:00:00.009+08:00| 29.5| -| 1970-01-01T08:00:00.010+08:00| 30.5| -| 1970-01-01T08:00:00.011+08:00| 31.5| -| 1970-01-01T08:00:00.012+08:00| 32.5| -| 1970-01-01T08:00:00.013+08:00| 33.5| -| 1970-01-01T08:00:00.014+08:00| 34.5| -| 1970-01-01T08:00:00.015+08:00| 35.500004| -| 1970-01-01T08:00:00.016+08:00| 36.5| -| 1970-01-01T08:00:00.017+08:00| 37.5| -| 1970-01-01T08:00:00.018+08:00| 38.5| -| 1970-01-01T08:00:00.019+08:00| 39.5| -| 1970-01-01T08:00:00.020+08:00| 40.5| -| 1970-01-01T08:00:00.021+08:00| 41.5| -| 1970-01-01T08:00:00.022+08:00| 42.500015| -| 1970-01-01T08:00:00.023+08:00| 43.5| -| 1970-01-01T08:00:00.024+08:00| 44.500008| -| 1970-01-01T08:00:00.025+08:00| 45.50003| -| 1970-01-01T08:00:00.026+08:00| 46.500008| -| 1970-01-01T08:00:00.027+08:00| 47.500008| -| 1970-01-01T08:00:00.028+08:00| 48.5| -| 1970-01-01T08:00:00.029+08:00| 49.5| -| 1970-01-01T08:00:00.030+08:00| 50.5| -| 1970-01-01T08:00:00.031+08:00| 51.500008| -| 1970-01-01T08:00:00.032+08:00| 52.5| -| 1970-01-01T08:00:00.033+08:00| 53.5| -+-----------------------------------+------------------+ -``` - -* step 3. revoke watermark_embedding from Alice - -Usage: `revoke watermark_embedding from Alice` - -Note that you can use `revoke watermark_embedding from user1,user2,...` to revoke watermark_embedding from multiple users. - -Only root can run this command. After root revokes watermark_embedding from Alice, all query results of Alice are original again. - -### Watermark Detection - -`detect-watermark.sh` and `detect-watermark.bat` are provided for different platforms. - -Usage: ./detect-watermark.sh [filePath] [secretKey] [watermarkBitString] [embed_row_cycle] [embed_lsb_num] [alpha] [columnIndex] [dataType: int/float/double] - -Example: ./detect-watermark.sh /home/data/dump1.csv IoTDB*2019@Beijing 100101110100 2 5 0.05 1 float - -| Args | Example | Explanation | -| ------------------ | -------------------- | ------------------------------------------------------------ | -| filePath | /home/data/dump1.csv | suspected data file path | -| secretKey | IoTDB*2019@Beijing | see watermark embedding section | -| watermarkBitString | 100101110100 | see watermark embedding section | -| embed_row_cycle | 2 | see watermark embedding section | -| embed_lsb_num | 5 | see watermark embedding section | -| alpha | 0.05 | significance level | -| columnIndex | 1 | specifies one column of the data to detect | -| dataType | float | specifies the data type of the detected column; int/float/double | - -Notes: - -- `filePath`: You can use export-csv tool to generate such data file. The first row is header and the first column is time. Data in the file looks like this: - - | Time | root.vehicle.d0.s1 | root.vehicle.d0.s1 | - | ----------------------------- | ------------------ | ------------------ | - | 1970-01-01T08:00:00.001+08:00 | 100 | null | - | ... | ... | ... | - -- `watermark_secret_key`, `watermark_bit_string`, `embed_row_cycle` and `embed_lsb_num` should be consistent with those used in the embedding phase. - -- `alpha`: It should be in the range of [0,1]. The watermark detection is based on the significance test. The smaller the `alpha` is, the lower the probability that the data without the watermark is detected to be watermark embedded, and thus the higher the credibility of the result of detecting the existence of the watermark in data. - -- `columnIndex`: It should be a positive integer. - diff --git a/docs/zh/UserGuide/Maintenance-Tools/Watermark-Tool.md b/docs/zh/UserGuide/Maintenance-Tools/Watermark-Tool.md deleted file mode 100644 index e1fbb7bd7b2..00000000000 --- a/docs/zh/UserGuide/Maintenance-Tools/Watermark-Tool.md +++ /dev/null @@ -1,196 +0,0 @@ -<!-- -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. - ---> -## 水印工具 - -这个工具提供了 1）IoTDB 查询结果水印嵌入功能，2）可疑数据的水印检测功能。 - -### 水印嵌入 - -#### 配置 - -IoTDB 默认关闭水印嵌入功能。为了使用这个功能，第一步要做的事情是修改配置文件`iotdb-datanode.properties`中的以下各项： - -| 名称 | 示例 | 解释 | -| ----------------------- | ------------------------------------------------------ | ----------------------------------- | -| watermark_module_opened | false | `true`打开水印嵌入功能；`false`关闭 | -| watermark_secret_key | IoTDB*2019@Beijing | 自定义密钥 | -| watermark_bit_string | 100101110100 | 要被嵌入的 0-1 比特串 | -| watermark_method | GroupBasedLSBMethod(embed_row_cycle=2,embed_lsb_num=5) | 指定水印算法及其参数 | - -注意： - -- `watermark_module_opened`: 如果您想使用水印嵌入功能，请将其设置成`true`。 -- `watermark_secret_key`: 不能使用字符 '&'。密钥长度没有限制，一般来说密钥越长，攻击难度就越高。 -- `watermark_bit_string`: 比特串长度没有限制（除了不能为空字符串），但是当长度过短时，水印检测可能达不到要求的显著性水平。 -- `watermark_method`: 现在仅支持一种算法 GroupBasedLSBMethod，因此您实际上可以修改的只有这个算法的两个参数`embed_row_cycle`和`embed_lsb_num`的值： - - 均是正整数 - - `embed_row_cycle`控制了被嵌入水印的行占总行数的比例。`embed_row_cycle`越小，被嵌入水印的行的比例就越大。当`embed_row_cycle`等于 1 的时候，所有的行都将嵌入水印。 - - GroupBasedLSBMethod 使用 LSB 嵌入。`embed_lsb_num`控制了允许嵌入水印的最低有效位的数量。`embed_lsb_num`越大，数值的可变化范围就越大。 -- `watermark_secret_key`, `watermark_bit_string`和`watermark_method`都不应该被攻击者获得。您需要自己负责配置文件`iotdb-datanode.properties`的安全管理。 - -#### 使用示例 - - * 第一步：创建一个新用户 Alice，授予读权限，然后查询 - -一个新创建的用户默认不使用水印。因此查询结果就是数据库中的原始数据。 - -``` -.\start-cli.bat -u root -pw root -create user Alice '1234' -grant user Alice privileges READ_TIMESERIES on root.vehicle -exit - -.\start-cli.bat -u Alice -pw 1234 -select * from root -+-----------------------------------+------------------+ -| Time|root.vehicle.d0.s0| -+-----------------------------------+------------------+ -| 1970-01-01T08:00:00.001+08:00| 21.5| -| 1970-01-01T08:00:00.002+08:00| 22.5| -| 1970-01-01T08:00:00.003+08:00| 23.5| -| 1970-01-01T08:00:00.004+08:00| 24.5| -| 1970-01-01T08:00:00.005+08:00| 25.5| -| 1970-01-01T08:00:00.006+08:00| 26.5| -| 1970-01-01T08:00:00.007+08:00| 27.5| -| 1970-01-01T08:00:00.008+08:00| 28.5| -| 1970-01-01T08:00:00.009+08:00| 29.5| -| 1970-01-01T08:00:00.010+08:00| 30.5| -| 1970-01-01T08:00:00.011+08:00| 31.5| -| 1970-01-01T08:00:00.012+08:00| 32.5| -| 1970-01-01T08:00:00.013+08:00| 33.5| -| 1970-01-01T08:00:00.014+08:00| 34.5| -| 1970-01-01T08:00:00.015+08:00| 35.5| -| 1970-01-01T08:00:00.016+08:00| 36.5| -| 1970-01-01T08:00:00.017+08:00| 37.5| -| 1970-01-01T08:00:00.018+08:00| 38.5| -| 1970-01-01T08:00:00.019+08:00| 39.5| -| 1970-01-01T08:00:00.020+08:00| 40.5| -| 1970-01-01T08:00:00.021+08:00| 41.5| -| 1970-01-01T08:00:00.022+08:00| 42.5| -| 1970-01-01T08:00:00.023+08:00| 43.5| -| 1970-01-01T08:00:00.024+08:00| 44.5| -| 1970-01-01T08:00:00.025+08:00| 45.5| -| 1970-01-01T08:00:00.026+08:00| 46.5| -| 1970-01-01T08:00:00.027+08:00| 47.5| -| 1970-01-01T08:00:00.028+08:00| 48.5| -| 1970-01-01T08:00:00.029+08:00| 49.5| -| 1970-01-01T08:00:00.030+08:00| 50.5| -| 1970-01-01T08:00:00.031+08:00| 51.5| -| 1970-01-01T08:00:00.032+08:00| 52.5| -| 1970-01-01T08:00:00.033+08:00| 53.5| -+-----------------------------------+------------------+ -``` - - * 第二步：给 Alice 施加水印嵌入 - -sql 用法：`grant watermark_embedding to Alice` - -您可以使用`grant watermark_embedding to user1,user2,...`来同时给多个用户施加水印嵌入。 - -只有 root 用户有权限运行该指令。在 root 给 Alice 施加水印嵌入之后，Alice 的所有查询结果都将被嵌入水印。 - -``` -.\start-cli.bat -u root -pw root -grant watermark_embedding to Alice -exit - -.\start-cli.bat -u Alice -pw 1234 -select * from root - -+-----------------------------------+------------------+ -| Time|root.vehicle.d0.s0| -+-----------------------------------+------------------+ -| 1970-01-01T08:00:00.001+08:00| 21.5| -| 1970-01-01T08:00:00.002+08:00| 22.5| -| 1970-01-01T08:00:00.003+08:00| 23.500008| -| 1970-01-01T08:00:00.004+08:00| 24.500015| -| 1970-01-01T08:00:00.005+08:00| 25.5| -| 1970-01-01T08:00:00.006+08:00| 26.500015| -| 1970-01-01T08:00:00.007+08:00| 27.5| -| 1970-01-01T08:00:00.008+08:00| 28.500004| -| 1970-01-01T08:00:00.009+08:00| 29.5| -| 1970-01-01T08:00:00.010+08:00| 30.5| -| 1970-01-01T08:00:00.011+08:00| 31.5| -| 1970-01-01T08:00:00.012+08:00| 32.5| -| 1970-01-01T08:00:00.013+08:00| 33.5| -| 1970-01-01T08:00:00.014+08:00| 34.5| -| 1970-01-01T08:00:00.015+08:00| 35.500004| -| 1970-01-01T08:00:00.016+08:00| 36.5| -| 1970-01-01T08:00:00.017+08:00| 37.5| -| 1970-01-01T08:00:00.018+08:00| 38.5| -| 1970-01-01T08:00:00.019+08:00| 39.5| -| 1970-01-01T08:00:00.020+08:00| 40.5| -| 1970-01-01T08:00:00.021+08:00| 41.5| -| 1970-01-01T08:00:00.022+08:00| 42.500015| -| 1970-01-01T08:00:00.023+08:00| 43.5| -| 1970-01-01T08:00:00.024+08:00| 44.500008| -| 1970-01-01T08:00:00.025+08:00| 45.50003| -| 1970-01-01T08:00:00.026+08:00| 46.500008| -| 1970-01-01T08:00:00.027+08:00| 47.500008| -| 1970-01-01T08:00:00.028+08:00| 48.5| -| 1970-01-01T08:00:00.029+08:00| 49.5| -| 1970-01-01T08:00:00.030+08:00| 50.5| -| 1970-01-01T08:00:00.031+08:00| 51.500008| -| 1970-01-01T08:00:00.032+08:00| 52.5| -| 1970-01-01T08:00:00.033+08:00| 53.5| -+-----------------------------------+------------------+ -``` - - * 第三步：撤销 Alice 的水印嵌入 - -sql 用法：`revoke watermark_embedding from Alice` - -您可以使用`revoke watermark_embedding from user1,user2,...`来同时撤销多个用户的水印嵌入。 - -只有 root 用户有权限运行该指令。在 root 撤销 Alice 的水印嵌入之后，Alice 的所有查询结果就又是数据库中的原始数据了。 - -### 水印检测 - -`detect-watermark.sh` 和 `detect-watermark.bat` 是给不同平台提供的功能相同的工具脚本。 - -用法： ./detect-watermark.sh [filePath] [secretKey] [watermarkBitString] [embed_row_cycle] [embed_lsb_num] [alpha] [columnIndex] [dataType: int/float/double] - -示例： ./detect-watermark.sh /home/data/dump1.csv IoTDB*2019@Beijing 100101110100 2 5 0.05 1 float - -| Args | 示例 | 解释 | -| ------------------ | -------------------- | ---------------------------------------------- | -| filePath | /home/data/dump1.csv | 可疑数据的文件路径 | -| secretKey | IoTDB*2019@Beijing | 参见水印嵌入小节 | -| watermarkBitString | 100101110100 | 参见水印嵌入小节 | -| embed_row_cycle | 2 | 参见水印嵌入小节 | -| embed_lsb_num | 5 | 参见水印嵌入小节 | -| alpha | 0.05 | 显著性水平 | -| columnIndex | 1 | 指定可疑数据的某一列进行检测 | -| dataType | float | 指定检测列的数据类型；int/float/double 任选其一 | - -注意： - -- `filePath`: 您可以使用 export-csv 工具来生成这样的数据文件。第一行是表头， 第一列是时间列。文件中的数据示例如下： - -| Time | root.vehicle.d0.s1 | root.vehicle.d0.s1 | -| ----------------------------- | ------------------ | ------------------ | -| 1970-01-01T08:00:00.001+08:00 | 100 | null | -| ... | ... | ... | - -- `watermark_secret_key`, `watermark_bit_string`, `embed_row_cycle`和`embed_lsb_num`应该和水印嵌入过程使用的值保持一致。 - -- `alpha`: 取值范围 [0,1]。水印检测基于显著性检验，`alpha`越小，没有嵌入水印的数据被检测成嵌入水印的可能性越低，从而检测出嵌入水印的结果的可信度越高。 - -- `columnIndex`: 正整数 diff --git a/integration-test/import-control.xml b/integration-test/import-control.xml index 8f8aeb795ee..d776b09a248 100644 --- a/integration-test/import-control.xml +++ b/integration-test/import-control.xml @@ -39,7 +39,6 @@ <subpackage name="db.it"> <disallow pkg="org.apache.iotdb.jdbc.*"/> <allow class="org.apache.iotdb.db.constant.TestConstant" /> - <allow class="org.apache.iotdb.db.tools.watermark.WatermarkDetector" /> <allow class="org.apache.iotdb.db.utils.MathUtils" /> <allow class="org.apache.iotdb.db.wal.utils.WALMode" /> <allow class="org.fusesource.mqtt.client.QoS" /> diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/MppCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/MppCommonConfig.java index 0f3ff163403..4d7cf67b134 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/MppCommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/MppCommonConfig.java @@ -239,30 +239,6 @@ public class MppCommonConfig extends MppBaseConfig implements CommonConfig { return this; } - @Override - public CommonConfig setEnableWatermark(boolean enableWatermark) { - setProperty("watermark_module_opened", String.valueOf(enableWatermark)); - return this; - } - - @Override - public CommonConfig setWatermarkSecretKey(String watermarkSecretKey) { - setProperty("watermark_secret_key", watermarkSecretKey); - return this; - } - - @Override - public CommonConfig setWatermarkBitString(String watermarkBitString) { - setProperty("watermark_bit_string", watermarkBitString); - return this; - } - - @Override - public CommonConfig setWatermarkMethod(String watermarkMethod) { - setProperty("watermark_method", watermarkMethod); - return this; - } - @Override public CommonConfig setEnableMQTTService(boolean enableMQTTService) { setProperty("enable_mqtt_service", String.valueOf(enableMQTTService)); diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/MppSharedCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/MppSharedCommonConfig.java index c3a0521e6cd..93dc55a4317 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/MppSharedCommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/MppSharedCommonConfig.java @@ -243,34 +243,6 @@ public class MppSharedCommonConfig implements CommonConfig { return this; } - @Override - public CommonConfig setEnableWatermark(boolean enableWatermark) { - cnConfig.setEnableWatermark(enableWatermark); - dnConfig.setEnableWatermark(enableWatermark); - return this; - } - - @Override - public CommonConfig setWatermarkSecretKey(String watermarkSecretKey) { - cnConfig.setWatermarkSecretKey(watermarkSecretKey); - dnConfig.setWatermarkSecretKey(watermarkSecretKey); - return this; - } - - @Override - public CommonConfig setWatermarkBitString(String watermarkBitString) { - cnConfig.setWatermarkBitString(watermarkBitString); - dnConfig.setWatermarkBitString(watermarkBitString); - return this; - } - - @Override - public CommonConfig setWatermarkMethod(String watermarkMethod) { - cnConfig.setWatermarkMethod(watermarkMethod); - dnConfig.setWatermarkMethod(watermarkMethod); - return this; - } - @Override public CommonConfig setEnableMQTTService(boolean enableMQTTService) { cnConfig.setEnableMQTTService(enableMQTTService); diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/RemoteCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/RemoteCommonConfig.java index 993c4020b11..7a3885d4c3d 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/RemoteCommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/RemoteCommonConfig.java @@ -174,26 +174,6 @@ public class RemoteCommonConfig implements CommonConfig { return this; } - @Override - public CommonConfig setEnableWatermark(boolean enableWatermark) { - return this; - } - - @Override - public CommonConfig setWatermarkSecretKey(String watermarkSecretKey) { - return this; - } - - @Override - public CommonConfig setWatermarkBitString(String watermarkBitString) { - return this; - } - - @Override - public CommonConfig setWatermarkMethod(String watermarkMethod) { - return this; - } - @Override public CommonConfig setEnableMQTTService(boolean enableMQTTService) { return this; diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java index 8ed36aa387b..2dfdcd7f5b6 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java @@ -81,14 +81,6 @@ public interface CommonConfig { CommonConfig setMaxDegreeOfIndexNode(int maxDegreeOfIndexNode); - CommonConfig setEnableWatermark(boolean enableWatermark); - - CommonConfig setWatermarkSecretKey(String watermarkSecretKey); - - CommonConfig setWatermarkBitString(String watermarkBitString); - - CommonConfig setWatermarkMethod(String watermarkMethod); - CommonConfig setEnableMQTTService(boolean enableMQTTService); CommonConfig setSchemaEngineMode(String schemaEngineMode); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/watermark/IoTDBWatermarkIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/watermark/IoTDBWatermarkIT.java deleted file mode 100644 index 0bd2aceec6a..00000000000 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/watermark/IoTDBWatermarkIT.java +++ /dev/null @@ -1,229 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.db.it.watermark; - -import org.apache.iotdb.db.tools.watermark.WatermarkDetector; -import org.apache.iotdb.it.env.EnvFactory; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.ClusterIT; -import org.apache.iotdb.itbase.constant.TestConstant; - -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -import java.io.File; -import java.io.PrintWriter; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.Statement; - -import static org.apache.iotdb.itbase.constant.TestConstant.BASE_OUTPUT_PATH; -import static org.junit.Assert.fail; - -@Ignore // TODO add it back when we support watermark in mpp mode -@RunWith(IoTDBTestRunner.class) -@Category({ClusterIT.class}) -public class IoTDBWatermarkIT { - - private static final String filePath1 = BASE_OUTPUT_PATH.concat("watermarked_query_result.csv"); - private static final String filePath2 = - BASE_OUTPUT_PATH.concat(File.separator).concat("notWatermarked_query_result.csv"); - private static PrintWriter writer1; - private static PrintWriter writer2; - private static String secretKey = "ASDFGHJKL"; - private static String watermarkBitString = "10101000100"; - private static int embed_row_cycle = 5; - private static int embed_lsb_num = 5; - - @Before - public void setUp() throws Exception { - EnvFactory.getEnv() - .getConfig() - .getCommonConfig() - .setEnableWatermark(true) - .setWatermarkSecretKey(secretKey) - .setWatermarkBitString(watermarkBitString) - .setWatermarkMethod( - String.format( - "GroupBasedLSBMethod" + "(embed_row_cycle=%d,embed_lsb_num=%d)", - embed_row_cycle, embed_lsb_num)); - - EnvFactory.getEnv().initClusterEnvironment(); - - insertData(); - - File file1 = new File(filePath1); - if (file1.exists()) { - file1.delete(); - } - writer1 = new PrintWriter(file1); - writer1.println("time,root.vehicle.d0.s0,root.vehicle.d0.s1,root.vehicle.d0.s2"); - - File file2 = new File(filePath2); - if (file2.exists()) { - file2.delete(); - } - writer2 = new PrintWriter(file2); - writer2.println("time,root.vehicle.d0.s0,root.vehicle.d0.s1,root.vehicle.d0.s2"); - } - - @After - public void tearDown() throws Exception { - File file1 = new File(filePath1); - if (file1.exists()) { - file1.delete(); - } - File file2 = new File(filePath2); - if (file2.exists()) { - file2.delete(); - } - EnvFactory.getEnv().cleanClusterEnvironment(); - } - - private static void insertData() { - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - - String[] create_sql = - new String[] { - "CREATE DATABASE root.vehicle", - "CREATE TIMESERIES root.vehicle.d0.s0 WITH DATATYPE=INT32, ENCODING=RLE", - "CREATE TIMESERIES root.vehicle.d0.s1 WITH DATATYPE=INT64, ENCODING=RLE", - "CREATE TIMESERIES root.vehicle.d0.s2 WITH DATATYPE=FLOAT, ENCODING=RLE" - }; - for (String sql : create_sql) { - statement.execute(sql); - } - - for (int time = 1; time < 1000; time++) { - String sql = - String.format( - "insert into root.vehicle.d0(timestamp,s0,s2) values(%s,%s,%s)", - time, time % 50, time % 50); - statement.execute(sql); - if (time % 10 == 0) { - sql = - String.format( - "insert into root.vehicle.d0(timestamp,s1) values(%s,%s)", time, time % 50); - statement.execute(sql); - } - } - } catch (Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } - } - - @Test - public void EncodeAndDecodeTest1() { - // Watermark Embedding - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - statement.execute("GRANT WATERMARK_EMBEDDING TO root"); - try (ResultSet resultSet = statement.executeQuery("SELECT s0,s1,s2 FROM root.vehicle.d0")) { - while (resultSet.next()) { - String ans = - resultSet.getString(TestConstant.TIMESTAMP_STR) - + "," - + resultSet.getString(TestConstant.d0 + "." + TestConstant.s0) - + "," - + resultSet.getString(TestConstant.d0 + "." + TestConstant.s1) - + "," - + resultSet.getString(TestConstant.d0 + "." + TestConstant.s2); - writer1.println(ans); - } - writer1.close(); - } - } catch (Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } - - // Watermark Detection - double alpha = 0.1; - int columnIndex = 1; - try { - boolean isWatermarked = - WatermarkDetector.isWatermarked( - filePath1, - secretKey, - watermarkBitString, - embed_row_cycle, - embed_lsb_num, - alpha, - columnIndex, - "int"); - Assert.assertTrue(isWatermarked); - } catch (Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } - } - - @Test - public void EncodeAndDecodeTest2() { - // No Watermark Embedding - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { - statement.execute("REVOKE WATERMARK_EMBEDDING FROM root"); - try (ResultSet resultSet = statement.executeQuery("SELECT s0,s1,s2 FROM root.vehicle.d0")) { - while (resultSet.next()) { - String ans = - resultSet.getString(TestConstant.TIMESTAMP_STR) - + "," - + resultSet.getString(TestConstant.d0 + "." + TestConstant.s0) - + "," - + resultSet.getString(TestConstant.d0 + "." + TestConstant.s1) - + "," - + resultSet.getString(TestConstant.d0 + "." + TestConstant.s2); - writer2.println(ans); - } - writer2.close(); - } - } catch (Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } - - // Watermark Detection - double alpha = 0.1; - int columnIndex = 1; - try { - boolean isWatermarked = - WatermarkDetector.isWatermarked( - filePath2, - secretKey, - watermarkBitString, - embed_row_cycle, - embed_lsb_num, - alpha, - columnIndex, - "int"); - Assert.assertFalse(isWatermarked); - } catch (Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } - } -} diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadata.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadata.java index 1a91669b155..eb4f8e3b1d1 100644 --- a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadata.java +++ b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadata.java @@ -479,20 +479,6 @@ public class IoTDBDatabaseMetadata implements DatabaseMetaData { sqlKeywordsThatArentSQL92 = keywordBuf.toString(); } - private WatermarkEncoder getWatermarkEncoder() { - try { - groupedLSBWatermarkEncoder = - new GroupedLSBWatermarkEncoder( - client.getProperties().getWatermarkSecretKey(), - client.getProperties().getWatermarkBitString(), - client.getProperties().getWatermarkParamMarkRate(), - client.getProperties().getWatermarkParamMaxRightBit()); - } catch (TException e) { - e.printStackTrace(); - } - return groupedLSBWatermarkEncoder; - } - @Override public boolean isWrapperFor(Class<?> arg0) throws SQLException { throw new SQLException(METHOD_NOT_SUPPORTED_STRING); diff --git a/iotdb-client/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadataTest.java b/iotdb-client/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadataTest.java index 8eb17d266f8..ec2c494ed37 100644 --- a/iotdb-client/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadataTest.java +++ b/iotdb-client/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadataTest.java @@ -136,10 +136,6 @@ public class IoTDBDatabaseMetadataTest { when(execStatementResp.getDataTypeList()).thenReturn(dataTypeList); when(execStatementResp.getColumns()).thenReturn(columnsList); execStatementResp.columnNameIndexMap = columnNameIndexMap; - when(client.getProperties().getWatermarkSecretKey()).thenReturn("IoTDB*2019@Beijing"); - when(client.getProperties().getWatermarkBitString()).thenReturn("100101110100"); - when(client.getProperties().getWatermarkParamMarkRate()).thenReturn(5); - when(client.getProperties().getWatermarkParamMaxRightBit()).thenReturn(5); ResultSet rs = databaseMetaData.getCatalogs(); assertEquals(2, rs.findColumn("TYPE_CAT")); } diff --git a/iotdb-protocol/thrift/src/main/thrift/client.thrift b/iotdb-protocol/thrift/src/main/thrift/client.thrift index 8f6cf8e6297..e111c8d2a35 100644 --- a/iotdb-protocol/thrift/src/main/thrift/client.thrift +++ b/iotdb-protocol/thrift/src/main/thrift/client.thrift @@ -369,14 +369,10 @@ struct ServerProperties { 2: required list<string> supportedTimeAggregationOperations; 3: required string timestampPrecision; 4: i32 maxConcurrentClientNum; - 5: optional string watermarkSecretKey; - 6: optional string watermarkBitString - 7: optional i32 watermarkParamMarkRate; - 8: optional i32 watermarkParamMaxRightBit; - 9: optional i32 thriftMaxFrameSize; - 10: optional bool isReadOnly; - 11: optional string buildInfo; - 12: optional string logo; + 5: optional i32 thriftMaxFrameSize; + 6: optional bool isReadOnly; + 7: optional string buildInfo; + 8: optional string logo; } struct TSSetSchemaTemplateReq { diff --git a/node-commons/src/assembly/resources/conf/iotdb-common.properties b/node-commons/src/assembly/resources/conf/iotdb-common.properties index d6907a70e31..c181a7128b6 100644 --- a/node-commons/src/assembly/resources/conf/iotdb-common.properties +++ b/node-commons/src/assembly/resources/conf/iotdb-common.properties @@ -828,22 +828,6 @@ cluster_name=defaultCluster # Datatype: integer # freq_block_size=1024 -#################### -### Watermark Configuration -#################### - -# Datatype: boolean -# watermark_module_opened=false - -# Datatype: String -# watermark_secret_key=IoTDB*2019@Beijing - -# Datatype: String -# watermark_bit_string=100101110100 - -# Datatype: String -# watermark_method=GroupBasedLSBMethod(embed_row_cycle=2,embed_lsb_num=5) - #################### ### Authorization Configuration #################### diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index f0c4fa30d50..30478d714f5 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -60,7 +60,6 @@ import java.util.HashSet; import java.util.List; import java.util.Properties; import java.util.concurrent.TimeUnit; -import java.util.regex.Matcher; import java.util.regex.Pattern; import static org.apache.iotdb.commons.conf.IoTDBConstant.OBJECT_STORAGE_DIR; @@ -620,18 +619,6 @@ public class IoTDBConfig { /** whether use chunkBufferPool. */ private boolean chunkBufferPoolEnable = false; - /** Switch of watermark function */ - private boolean enableWatermark = false; - - /** Secret key for watermark */ - private String watermarkSecretKey = "IoTDB*2019@Beijing"; - - /** Bit string of watermark */ - private String watermarkBitString = "100101110100"; - - /** Watermark method and parameters */ - private String watermarkMethod = "GroupBasedLSBMethod(embed_row_cycle=2,embed_lsb_num=5)"; - /** Switch of creating schema automatically */ private boolean enableAutoCreateSchema = true; @@ -2255,68 +2242,6 @@ public class IoTDBConfig { this.lastCacheEnable = lastCacheEnable; } - public boolean isEnableWatermark() { - return enableWatermark; - } - - public void setEnableWatermark(boolean enableWatermark) { - this.enableWatermark = enableWatermark; - } - - public String getWatermarkSecretKey() { - return watermarkSecretKey; - } - - public void setWatermarkSecretKey(String watermarkSecretKey) { - this.watermarkSecretKey = watermarkSecretKey; - } - - public String getWatermarkBitString() { - return watermarkBitString; - } - - public void setWatermarkBitString(String watermarkBitString) { - this.watermarkBitString = watermarkBitString; - } - - public String getWatermarkMethod() { - return this.watermarkMethod; - } - - public void setWatermarkMethod(String watermarkMethod) { - this.watermarkMethod = watermarkMethod; - } - - public String getWatermarkMethodName() { - return watermarkMethod.split("\\(")[0]; - } - - public int getWatermarkParamMarkRate() { - return Integer.parseInt(getWatermarkParamValue("embed_row_cycle", "5")); - } - - public int getWatermarkParamMaxRightBit() { - return Integer.parseInt(getWatermarkParamValue("embed_lsb_num", "5")); - } - - private String getWatermarkParamValue(String key, String defaultValue) { - String res = getWatermarkParamValue(key); - if (res != null) { - return res; - } - return defaultValue; - } - - private String getWatermarkParamValue(String key) { - String pattern = key + "=(\\w*)"; - Pattern r = Pattern.compile(pattern); - Matcher m = r.matcher(watermarkMethod); - if (m.find() && m.groupCount() > 0) { - return m.group(1); - } - return null; - } - public boolean isAutoCreateSchemaEnabled() { return enableAutoCreateSchema; } diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 15a78fd231d..60507d8cf31 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -761,16 +761,6 @@ public class IoTDBDescriptor { conf.setRpcMaxConcurrentClientNum(maxConcurrentClientNum); - conf.setEnableWatermark( - Boolean.parseBoolean( - properties.getProperty( - "watermark_module_opened", Boolean.toString(conf.isEnableWatermark()).trim()))); - conf.setWatermarkSecretKey( - properties.getProperty("watermark_secret_key", conf.getWatermarkSecretKey())); - conf.setWatermarkBitString( - properties.getProperty("watermark_bit_string", conf.getWatermarkBitString())); - conf.setWatermarkMethod(properties.getProperty("watermark_method", conf.getWatermarkMethod())); - loadAutoCreateSchemaProps(properties); conf.setTsFileStorageFs( diff --git a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/ClientRPCServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/ClientRPCServiceImpl.java index 28bec0979bd..02f1c414fe0 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/ClientRPCServiceImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/ClientRPCServiceImpl.java @@ -738,14 +738,6 @@ public class ClientRPCServiceImpl implements IClientRPCServiceWithHandler { IoTDBDescriptor.getInstance().getConfig().getTimestampPrecision()); properties.setMaxConcurrentClientNum( IoTDBDescriptor.getInstance().getConfig().getRpcMaxConcurrentClientNum()); - properties.setWatermarkSecretKey( - IoTDBDescriptor.getInstance().getConfig().getWatermarkSecretKey()); - properties.setWatermarkBitString( - IoTDBDescriptor.getInstance().getConfig().getWatermarkBitString()); - properties.setWatermarkParamMarkRate( - IoTDBDescriptor.getInstance().getConfig().getWatermarkParamMarkRate()); - properties.setWatermarkParamMaxRightBit( - IoTDBDescriptor.getInstance().getConfig().getWatermarkParamMaxRightBit()); properties.setIsReadOnly(CommonDescriptor.getInstance().getConfig().isReadOnly()); properties.setThriftMaxFrameSize( IoTDBDescriptor.getInstance().getConfig().getThriftMaxFrameSize()); diff --git a/server/src/main/java/org/apache/iotdb/db/tools/watermark/GroupedLSBWatermarkEncoder.java b/server/src/main/java/org/apache/iotdb/db/tools/watermark/GroupedLSBWatermarkEncoder.java deleted file mode 100644 index 20d8f3b9b28..00000000000 --- a/server/src/main/java/org/apache/iotdb/db/tools/watermark/GroupedLSBWatermarkEncoder.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.db.tools.watermark; - -import org.apache.iotdb.db.conf.IoTDBConfig; -import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; -import org.apache.iotdb.tsfile.read.common.Field; -import org.apache.iotdb.tsfile.read.common.RowRecord; - -import org.apache.thrift.EncodingUtils; - -import java.math.BigInteger; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.List; - -public class GroupedLSBWatermarkEncoder implements WatermarkEncoder { - - private String secretKey; - private String bitString; - private int markRate = 2; - private int groupNumber; - private int maxBitPosition = 5; - private int minBitPosition = 0; - - public GroupedLSBWatermarkEncoder(String secretKey, String bitString) { - this.secretKey = secretKey; - this.bitString = bitString; - this.groupNumber = bitString.length(); - } - - public GroupedLSBWatermarkEncoder(IoTDBConfig conf) { - this(conf.getWatermarkSecretKey(), conf.getWatermarkBitString()); - this.markRate = conf.getWatermarkParamMarkRate(); - this.maxBitPosition = conf.getWatermarkParamMaxRightBit(); - } - - public static int hashMod(String val, Integer base) { - MessageDigest md; - try { - md = MessageDigest.getInstance("MD5"); - } catch (NoSuchAlgorithmException e) { - throw new RuntimeException("ERROR: Cannot find MD5 algorithm!"); - } - md.update(val.getBytes()); - BigInteger resultInteger = new BigInteger(1, md.digest()); - return resultInteger.mod(new BigInteger(base.toString())).intValue(); - } - - @Override - public boolean needEncode(long timestamp) { - return hashMod(String.format("%s%d", secretKey, timestamp), markRate) == 0; - } - - private int getGroupId(long timestamp) { - return hashMod(String.format("%d%s", timestamp, secretKey), groupNumber); - } - - private int getBitPosition(long timestamp) { - if (maxBitPosition <= minBitPosition) { - throw new RuntimeException("Error: minBitPosition is bigger than maxBitPosition"); - } - int range = maxBitPosition - minBitPosition; - return minBitPosition - + hashMod(String.format("%s%d%s", secretKey, timestamp, secretKey), range); - } - - private boolean getBitValue(long timestamp) { - int groupId = getGroupId(timestamp); - int bitIndex = groupId % bitString.length(); - return bitString.charAt(bitIndex) == '1'; - } - - @Override - public int encodeInt(int value, long timestamp) { - int targetBitPosition = getBitPosition(timestamp); - boolean targetBitValue = getBitValue(timestamp); - return EncodingUtils.setBit(value, targetBitPosition, targetBitValue); - } - - @Override - public long encodeLong(long value, long timestamp) { - int targetBitPosition = getBitPosition(timestamp); - boolean targetBitValue = getBitValue(timestamp); - return EncodingUtils.setBit(value, targetBitPosition, targetBitValue); - } - - @Override - public float encodeFloat(float value, long timestamp) { - int intBits = Float.floatToIntBits(value); - return Float.intBitsToFloat(encodeInt(intBits, timestamp)); - } - - @Override - public double encodeDouble(double value, long timestamp) { - long longBits = Double.doubleToLongBits(value); - return Double.longBitsToDouble(encodeLong(longBits, timestamp)); - } - - @Override - public RowRecord encodeRecord(RowRecord record) { - long timestamp = record.getTimestamp(); - if (!needEncode(timestamp)) { - return record; - } - List<Field> fields = record.getFields(); - for (Field field : fields) { - if (field == null || field.getDataType() == null) { - continue; - } - TSDataType dataType = field.getDataType(); - switch (dataType) { - case INT32: - int originIntValue = field.getIntV(); - field.setIntV(encodeInt(originIntValue, timestamp)); - break; - case INT64: - long originLongValue = field.getLongV(); - field.setLongV(encodeLong(originLongValue, timestamp)); - break; - case FLOAT: - float originFloatValue = field.getFloatV(); - field.setFloatV(encodeFloat(originFloatValue, timestamp)); - break; - case DOUBLE: - double originDoubleValue = field.getDoubleV(); - field.setDoubleV(encodeDouble(originDoubleValue, timestamp)); - break; - default: - } - } - return record; - } -} diff --git a/server/src/main/java/org/apache/iotdb/db/tools/watermark/WatermarkDetector.java b/server/src/main/java/org/apache/iotdb/db/tools/watermark/WatermarkDetector.java deleted file mode 100644 index c23964d70b6..00000000000 --- a/server/src/main/java/org/apache/iotdb/db/tools/watermark/WatermarkDetector.java +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.tools.watermark; - -import org.apache.iotdb.db.utils.DateTimeUtils; - -import org.apache.thrift.EncodingUtils; - -import java.io.BufferedReader; -import java.io.FileReader; -import java.io.IOException; -import java.math.BigDecimal; -import java.math.RoundingMode; -import java.time.ZoneId; - -public class WatermarkDetector { - - public static void main(String[] args) throws IOException { - if (args == null || args.length != 8) { - throw new IOException( - "Usage: ./detect-watermark.sh [filePath] [secretKey] " - + "[watermarkBitString] [embed_row_cycle] [embed_lsb_num] [alpha] [columnIndex] " - + "[dataType: int/float/double]"); - } - String filePath = args[0]; // data file path - String secretKey = args[1]; // watermark secret key - String watermarkBitString = args[2]; // watermark bit string - int embed_row_cycle = Integer.parseInt(args[3]); // watermark parameter - int embed_lsb_num = Integer.parseInt(args[4]); // watermark parameter - double alpha = Double.parseDouble(args[5]); // significance level of watermark detection - int columnIndex = Integer.parseInt(args[6]); // specify a column of data to detect - String dataType = args[7].toLowerCase(); - - if (embed_row_cycle < 1 || embed_lsb_num < 1 || alpha < 0 || alpha > 1 || columnIndex < 1) { - throw new IOException("Parameter out of range."); - } - - if (!"int".equals(dataType) && !"float".equals(dataType) && !"double".equals(dataType)) { - throw new IOException("invalid parameter: supported data types are int/float/double"); - } - - isWatermarked( - filePath, - secretKey, - watermarkBitString, - embed_row_cycle, - embed_lsb_num, - alpha, - columnIndex, - dataType); - } - - // Suppress high Cognitive Complexity warning - public static boolean isWatermarked( - String filePath, - String secretKey, - String watermarkBitString, - int embed_row_cycle, - int embed_lsb_num, - double alpha, - int columnIndex, - String dataType) - throws IOException { - System.out.println("-----Watermark detection begins-----"); - int[] trueNums = new int[watermarkBitString.length()]; // for majority vote - int[] falseNums = new int[watermarkBitString.length()]; // for majority vote - try (BufferedReader reader = new BufferedReader(new FileReader(filePath))) { - String line = reader.readLine(); // skip header - String[] items = line.split(","); - if (columnIndex < 1 || columnIndex > items.length - 1) { - throw new IOException("columnIndex is out of range."); - } - while ((line = reader.readLine()) != null) { - items = line.split(","); - long timestamp = parseTimestamp(items[0]); - if (GroupedLSBWatermarkEncoder.hashMod( - String.format("%s%d", secretKey, timestamp), embed_row_cycle) - == 0) { - String str = items[columnIndex]; - if ("null".equals(str)) { - continue; - } - - int targetBitPosition = - GroupedLSBWatermarkEncoder.hashMod( - String.format("%s%d%s", secretKey, timestamp, secretKey), embed_lsb_num); - int groupId = - GroupedLSBWatermarkEncoder.hashMod( - String.format("%d%s", timestamp, secretKey), watermarkBitString.length()); - - boolean isTrue = true; - switch (dataType) { - case "int": - isTrue = - EncodingUtils.testBit(Integer.parseInt(items[columnIndex]), targetBitPosition); - break; - case "float": - int floatToIntBits = Float.floatToIntBits(Float.parseFloat(items[columnIndex])); - isTrue = EncodingUtils.testBit(floatToIntBits, targetBitPosition); - break; - case "double": - long doubleToLongBits = - Double.doubleToLongBits(Double.parseDouble(items[columnIndex])); - isTrue = EncodingUtils.testBit(doubleToLongBits, targetBitPosition); - break; - default: - } - if (isTrue) { - trueNums[groupId] += 1; - } else { - falseNums[groupId] += 1; - } - } - } - } - - int cnt = 0; // total counted number - int hit_cnt = 0; // detected hit number - for (int i = 0; i < watermarkBitString.length(); i++) { - int res = trueNums[i] - falseNums[i]; // majority vote - if (res > 0 && watermarkBitString.charAt(i) == '1') { - hit_cnt += 1; - } else if (res < 0 && watermarkBitString.charAt(i) == '0') { - hit_cnt += 1; - } - if (res != 0) { - cnt += 1; - } // otherwise trueNums[i]==falseNums[i] then it is not counted - } - - int b = calMin(cnt, alpha); - System.out.println( - String.format("total counted number = %d, detected hit number = %d", cnt, hit_cnt)); - System.out.println( - String.format( - "To reach the significant level %f, the hit number should be not smaller than: %d", - alpha, b)); - boolean isWatermarked; - if (hit_cnt >= b) { - System.out.println("Therefore the detection result is: watermarked"); - isWatermarked = true; - } else { - System.out.println("Therefore the detection result is: not watermarked"); - isWatermarked = false; - } - System.out.println("-----Watermark detection finishes-----"); - return isWatermarked; - } - - /** Parses timestamp from string type to long type */ - private static long parseTimestamp(String str) { - long timestamp; - try { - timestamp = Long.parseLong(str); - } catch (NumberFormatException e) { - timestamp = DateTimeUtils.convertDatetimeStrToLong(str, ZoneId.systemDefault()); - } - return timestamp; - } - - /** - * Finds the minimum b that meets the formula: (C(l,b)+C(l,b+1)+C(l,b+2)+...+C(l,l))/2^l < alpha - * - * @param l the total number - * @param alpha significance level - * @return the minimum b - */ - private static int calMin(int l, double alpha) { - int b = l; - BigDecimal sum = new BigDecimal("1"); - - BigDecimal thrs = BigDecimal.valueOf(alpha); - for (int i = 0; i < l; i++) { - thrs = thrs.multiply(new BigDecimal("2")); - } - - while (sum.compareTo(thrs) < 0) { // sum < thrs - b -= 1; - sum = sum.add(Comb(l, b)); - } - - b++; - if (b > l) { - System.out.println( - "The total counted number or the alpha is too small to find b " - + "that meets the formula: (C(l,b)+C(l,b+1)+C(l,b+2)+...+C(l,l))/2^l < alpha"); - } - return b; - } - - /** Calculates combinatorial number C(n,m). */ - private static BigDecimal Comb(int n, int m) { - BigDecimal res1 = new BigDecimal("1"); - for (int i = n; i > m; i--) { - res1 = res1.multiply(new BigDecimal(i)); - } - BigDecimal res2 = new BigDecimal("1"); - for (int i = 2; i <= n - m; i++) { - res2 = res2.multiply(new BigDecimal(i)); - } - return res1.divide(res2, 10, RoundingMode.HALF_EVEN); - } -} diff --git a/server/src/main/java/org/apache/iotdb/db/tools/watermark/WatermarkEncoder.java b/server/src/main/java/org/apache/iotdb/db/tools/watermark/WatermarkEncoder.java deleted file mode 100644 index 78d2a0ab286..00000000000 --- a/server/src/main/java/org/apache/iotdb/db/tools/watermark/WatermarkEncoder.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.db.tools.watermark; - -import org.apache.iotdb.tsfile.read.common.RowRecord; - -public interface WatermarkEncoder { - - int encodeInt(int value, long time); - - long encodeLong(long value, long time); - - float encodeFloat(float value, long time); - - double encodeDouble(double value, long time); - - boolean needEncode(long timestamp); - - RowRecord encodeRecord(RowRecord record); -}
