This is an automated email from the ASF dual-hosted git repository.
spricoder pushed a commit to branch object_type
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/object_type by this push:
new 7ee4df735cf add compress tiff version1
7ee4df735cf is described below
commit 7ee4df735cf2d2b194ed8a5e1abc9b90980bc23d
Author: spricoder <[email protected]>
AuthorDate: Thu Jul 10 11:47:51 2025 +0800
add compress tiff version1
---
iotdb-client/service-rpc/pom.xml | 10 ++
.../rpc/model/CompressedTiffModelProcessor.java | 114 +++++++++++++++++++++
.../org/apache/iotdb/rpc/model/ModelProcessor.java | 15 +--
.../apache/iotdb/rpc/model/ModelProcessorType.java | 25 -----
iotdb-core/datanode/pom.xml | 5 +
.../db/utils/model/CompressedTiffModelReader.java | 71 +++++++++++++
.../apache/iotdb/db/utils/model/ModelReader.java | 2 +
7 files changed, 211 insertions(+), 31 deletions(-)
diff --git a/iotdb-client/service-rpc/pom.xml b/iotdb-client/service-rpc/pom.xml
index c525afa7cfc..17c52de5d91 100644
--- a/iotdb-client/service-rpc/pom.xml
+++ b/iotdb-client/service-rpc/pom.xml
@@ -89,6 +89,16 @@
<artifactId>ij</artifactId>
<version>1.54g</version>
</dependency>
+ <dependency>
+ <groupId>org.gdal</groupId>
+ <artifactId>gdal</artifactId>
+ <version>3.11.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.gdal</groupId>
+ <artifactId>vsi</artifactId>
+ <version>1.0</version>
+ </dependency>
</dependencies>
<build>
<plugins>
diff --git
a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/model/CompressedTiffModelProcessor.java
b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/model/CompressedTiffModelProcessor.java
new file mode 100644
index 00000000000..6f3c57faf38
--- /dev/null
+++
b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/model/CompressedTiffModelProcessor.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.rpc.model;
+
+import org.gdal.VsiGdalNative;
+import org.gdal.gdal.Band;
+import org.gdal.gdal.Dataset;
+import org.gdal.gdal.Driver;
+import org.gdal.gdal.gdal;
+import org.gdal.gdalconst.gdalconstConstants;
+
+public class CompressedTiffModelProcessor extends ModelProcessor {
+ private static final Driver DRIVER;
+ // Specifying compression options
+ private static String compressOption = "COMPRESS=LZW";
+ // Specifying block x size options
+ private static String blockXSize = "BLOCKXSIZE=256";
+ // Specifying block y size options
+ private static String blockYSize = "BLOCKYSIZE=256";
+
+ static {
+ gdal.AllRegister();
+ DRIVER = gdal.GetDriverByName("GTiff");
+ if (DRIVER == null) {
+ throw new RuntimeException("Failed to get GTiff driver: " +
gdal.GetLastErrorMsg());
+ }
+ }
+
+ @Override
+ public byte[] write(float[] values, int width, int height) {
+ String virtualFilePath = getVirtualFilePath();
+ return write(virtualFilePath, values, width, height);
+ }
+
+ private byte[] write(String filePath, float[] values, int width, int height)
{
+ // floating point data should use predictor 2 (for difference prediction),
and use block storage
+ // (recommended for LZW)
+ String[] options =
+ new String[] {compressOption, "PREDICTOR=2", "TILED=YES", blockXSize,
blockYSize};
+
+ Dataset dataset = null;
+ try {
+ // Create dataset with specified options
+ dataset = DRIVER.Create(filePath, width, height, 1,
gdalconstConstants.GDT_Float32, options);
+
+ if (dataset == null) {
+ throw new RuntimeException("Failed to create dataset: " +
gdal.GetLastErrorMsg());
+ }
+
+ Band band = dataset.GetRasterBand(1);
+ int result = band.WriteRaster(0, 0, width, height, values);
+ if (result != gdalconstConstants.CE_None) {
+ throw new RuntimeException("Failed to write data to tiff file: " +
gdal.GetLastErrorMsg());
+ }
+ band.FlushCache();
+ return VsiGdalNative.readVsiMemFile(filePath, false);
+ } finally {
+ if (dataset != null) {
+ dataset.delete();
+ }
+ }
+ }
+
+ @Override
+ public float[] readAll(byte[] fileBytes) {
+ // TODO @spricoder
+ String virtualFilePath = getVirtualFilePath();
+ VsiGdalNative.writeVsiMemFile(virtualFilePath, fileBytes);
+ return readAll(virtualFilePath);
+ }
+
+ @Override
+ public float[] readAll(String filePath) {
+ Dataset dataset = gdal.Open(filePath, gdalconstConstants.GA_ReadOnly);
+ if (dataset == null) {
+ throw new RuntimeException("Failed to open tiff file: " +
gdal.GetLastErrorMsg());
+ }
+ try {
+ Band band = dataset.GetRasterBand(1);
+ if (band == null) {
+ throw new RuntimeException(
+ "Failed to get raster band from dataset" + gdal.GetLastErrorMsg());
+ }
+ int width = band.getXSize();
+ int height = band.getYSize();
+ float[] result = new float[width * height];
+ band.ReadRaster(0, 0, width, height, gdalconstConstants.GDT_Float32,
result);
+ return result;
+ } finally {
+ dataset.delete();
+ }
+ }
+
+ private String getVirtualFilePath() {
+ return "";
+ }
+}
diff --git
a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/model/ModelProcessor.java
b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/model/ModelProcessor.java
index 45af293a699..a7f9ef50805 100644
---
a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/model/ModelProcessor.java
+++
b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/model/ModelProcessor.java
@@ -46,12 +46,15 @@ public abstract class ModelProcessor {
*/
public abstract float[] readAll(String filePath);
- public static ModelProcessor getInstance(ModelProcessorType modelFileType) {
- switch (modelFileType) {
- case UNCOMPRESSED_TIFF:
- return new UnCompressedTiffModelProcessor();
- default:
- return new CompressedTsFileModelProcessor();
+ public static ModelProcessor getInstance(String modelFileType) {
+ if (modelFileType.equalsIgnoreCase("COMPRESSED_TSFILE")) {
+ return new CompressedTsFileModelProcessor();
+ } else if (modelFileType.equalsIgnoreCase("UNCOMPRESSED_TIFF")) {
+ return new UnCompressedTiffModelProcessor();
+ } else if (modelFileType.equalsIgnoreCase("COMPRESSED_TIFF")) {
+ return new CompressedTiffModelProcessor();
+ } else {
+ throw new IllegalArgumentException("Unsupported model file type: " +
modelFileType);
}
}
}
diff --git
a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/model/ModelProcessorType.java
b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/model/ModelProcessorType.java
deleted file mode 100644
index 1966a14484e..00000000000
---
a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/model/ModelProcessorType.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.rpc.model;
-
-public enum ModelProcessorType {
- COMPRESSED_TSFILE,
- UNCOMPRESSED_TIFF
-}
diff --git a/iotdb-core/datanode/pom.xml b/iotdb-core/datanode/pom.xml
index f1cd4a4fe46..c40553cde81 100644
--- a/iotdb-core/datanode/pom.xml
+++ b/iotdb-core/datanode/pom.xml
@@ -392,6 +392,11 @@
<version>1.3.0</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.gdal</groupId>
+ <artifactId>gdal</artifactId>
+ <version>3.11.0</version>
+ </dependency>
</dependencies>
<build>
<plugins>
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/model/CompressedTiffModelReader.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/model/CompressedTiffModelReader.java
new file mode 100644
index 00000000000..d78caa4e467
--- /dev/null
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/model/CompressedTiffModelReader.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.utils.model;
+
+import org.gdal.gdal.Band;
+import org.gdal.gdal.Dataset;
+import org.gdal.gdal.gdal;
+import org.gdal.gdalconst.gdalconstConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class CompressedTiffModelReader extends ModelReader {
+ private static final Logger LOGGER =
LoggerFactory.getLogger(CompressedTiffModelReader.class);
+
+ static {
+ gdal.AllRegister();
+ }
+
+ @Override
+ public List<float[]> penetrate(String filePath, List<List<Integer>>
startAndEndTimeArray) {
+ Dataset dataset = gdal.Open(filePath, gdalconstConstants.GA_ReadOnly);
+ if (dataset == null) {
+ LOGGER.error("Failed to open tiff file: {}", gdal.GetLastErrorMsg());
+ throw new RuntimeException("Failed to open tiff file: " +
gdal.GetLastErrorMsg());
+ }
+ try {
+ Band band = dataset.GetRasterBand(1);
+ if (band == null) {
+ LOGGER.error("Failed to get raster band from dataset: {}",
gdal.GetLastErrorMsg());
+ throw new RuntimeException(
+ "Failed to get raster band from dataset" + gdal.GetLastErrorMsg());
+ }
+ int width = band.getXSize();
+ List<float[]> result = new ArrayList<>();
+ for (List<Integer> startAndEndTime : startAndEndTimeArray) {
+ int xIndex = startAndEndTime.get(0);
+ int yIndex = startAndEndTime.get(1);
+ float[] tmp = new float[yIndex - xIndex + 1];
+ int xOff = xIndex % width;
+ int yOff = xIndex / width;
+ int xSize = yIndex - xIndex + 1;
+ int ySize = 1;
+ band.ReadRaster(xOff, yOff, xSize, ySize,
gdalconstConstants.GDT_Float32, tmp);
+ result.add(tmp);
+ }
+ return result;
+ } finally {
+ dataset.delete();
+ }
+ }
+}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/model/ModelReader.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/model/ModelReader.java
index 18a6f5aed43..3d5c043a125 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/model/ModelReader.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/model/ModelReader.java
@@ -44,6 +44,8 @@ public abstract class ModelReader {
return new UnCompressedTiffModelReader();
} else if (modelFileType.equalsIgnoreCase("COMPRESSED_TSFILE")) {
return new CompressedTsFileModelReader();
+ } else if (modelFileType.equalsIgnoreCase("COMPRESSED_TIFF")) {
+ return new CompressedTiffModelReader();
} else {
throw new SemanticException("Unsupported model file type: " +
modelFileType);
}
