This is an automated email from the ASF dual-hosted git repository. bchapuis pushed a commit to branch calcite-schema-ddl in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit 19f8856ff4e11918f28db212d885f8ae1862662f Author: Bertil Chapuis <[email protected]> AuthorDate: Mon Apr 14 16:54:37 2025 +0200 Add schema for openstreetmap formats --- .../baremaps/calcite/BaremapsTableFactory.java | 32 +--- .../apache/baremaps/calcite/data/DataSchema.java | 2 +- .../calcite/openstreetmap/OpenStreetMapSchema.java | 184 ++++++++++++++++++++ .../calcite/openstreetmap/OpenStreetMapTable.java | 19 ++- .../openstreetmap/OpenStreetMapSchemaTest.java | 187 +++++++++++++++++++++ .../openstreetmap/OpenStreetMapTableTest.java | 4 +- 6 files changed, 395 insertions(+), 33 deletions(-) diff --git a/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/BaremapsTableFactory.java b/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/BaremapsTableFactory.java index 857f46261..c2ddc3ad3 100644 --- a/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/BaremapsTableFactory.java +++ b/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/BaremapsTableFactory.java @@ -143,9 +143,9 @@ public class BaremapsTableFactory implements TableFactory<Table> { // Create an entity reader based on the file extension if (filePath.endsWith(".pbf") || filePath.endsWith(".osm.pbf")) { - return createTableFromPbf(inputStream); + return createTableFromPbf(Paths.get(filePath)); } else if (filePath.endsWith(".xml") || filePath.endsWith(".osm")) { - return createTableFromXml(inputStream); + return createTableFromXml(Paths.get(filePath)); } else { throw new IllegalArgumentException( "Unsupported file format. Supported formats are .pbf, .osm.pbf, .xml, and .osm"); @@ -249,16 +249,6 @@ public class BaremapsTableFactory implements TableFactory<Table> { } } - /** - * Create a table from a PBF file. - * - * @param inputStream the input stream - * @return the table - */ - public static OpenStreetMapTable createTableFromPbf(InputStream inputStream) { - return new OpenStreetMapTable(new PbfEntityReader().setGeometries(true), inputStream); - } - /** * Create a table from a PBF file. * @@ -267,17 +257,9 @@ public class BaremapsTableFactory implements TableFactory<Table> { * @throws IOException if an I/O error occurs */ public static OpenStreetMapTable createTableFromPbf(Path path) throws IOException { - return createTableFromPbf(new FileInputStream(path.toFile())); - } - - /** - * Create a table from an XML file. - * - * @param inputStream the input stream - * @return the table - */ - public static OpenStreetMapTable createTableFromXml(InputStream inputStream) { - return new OpenStreetMapTable(new XmlEntityReader().setGeometries(true), inputStream); + PbfEntityReader reader = new PbfEntityReader(); + reader.setGeometries(true); + return new OpenStreetMapTable(path.toFile(), reader); } /** @@ -288,7 +270,9 @@ public class BaremapsTableFactory implements TableFactory<Table> { * @throws IOException if an I/O error occurs */ public static OpenStreetMapTable createTableFromXml(Path path) throws IOException { - return createTableFromXml(new FileInputStream(path.toFile())); + XmlEntityReader reader = new XmlEntityReader(); + reader.setGeometries(true); + return new OpenStreetMapTable(path.toFile(), reader); } private Table createGeoParquetTable(Map<String, Object> operand) { diff --git a/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/data/DataSchema.java b/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/data/DataSchema.java index 35e9f4c89..8f3bafe84 100644 --- a/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/data/DataSchema.java +++ b/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/data/DataSchema.java @@ -88,4 +88,4 @@ public class DataSchema extends AbstractSchema { protected Map<String, Table> getTableMap() { return tableMap; } -} \ No newline at end of file +} diff --git a/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/openstreetmap/OpenStreetMapSchema.java b/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/openstreetmap/OpenStreetMapSchema.java new file mode 100644 index 000000000..322a50592 --- /dev/null +++ b/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/openstreetmap/OpenStreetMapSchema.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.baremaps.calcite.openstreetmap; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import org.apache.baremaps.openstreetmap.OpenStreetMapFormat; +import org.apache.baremaps.openstreetmap.model.Entity; +import org.apache.baremaps.openstreetmap.pbf.PbfEntityReader; +import org.apache.baremaps.openstreetmap.xml.XmlEntityReader; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.schema.Table; +import org.apache.calcite.schema.impl.AbstractSchema; + +/** + * A Calcite schema implementation for OpenStreetMap data. This schema provides access to OpenStreetMap + * files through the Apache Calcite framework for SQL querying. + */ +public class OpenStreetMapSchema extends AbstractSchema { + + private final File directory; + private final Map<String, Table> tableMap; + private final RelDataTypeFactory typeFactory; + + /** + * Constructs an OpenStreetMapSchema with the specified directory. + * + * @param directory the directory containing OpenStreetMap files + * @param typeFactory the type factory to use for creating tables + * @throws IOException if an I/O error occurs + */ + public OpenStreetMapSchema(File directory, RelDataTypeFactory typeFactory) throws IOException { + this.directory = Objects.requireNonNull(directory, "Directory cannot be null"); + this.typeFactory = Objects.requireNonNull(typeFactory, "Type factory cannot be null"); + this.tableMap = new HashMap<>(); + + // Process files in the directory + File[] files = directory.listFiles((dir, name) -> + name.toLowerCase().endsWith(".pbf") || + name.toLowerCase().endsWith(".osm.pbf") || + name.toLowerCase().endsWith(".xml") || + name.toLowerCase().endsWith(".osm")); + + if (files != null) { + for (File file : files) { + // Extract the base name without extension (e.g., "sample" from "sample.osm.pbf") + String fileName = file.getName(); + String tableName = fileName; + + // Remove all extensions (e.g., "sample.osm.pbf" -> "sample") + while (tableName.contains(".")) { + int lastDotIndex = tableName.lastIndexOf('.'); + if (lastDotIndex > 0) { + tableName = tableName.substring(0, lastDotIndex); + } else { + break; + } + } + + // Create the table with the file reference + tableMap.put(tableName, createTable(file)); + } + } + } + + /** + * Constructs an OpenStreetMapSchema with a single file. + * + * @param file the OpenStreetMap file + * @param typeFactory the type factory to use for creating tables + * @throws IOException if an I/O error occurs + */ + public OpenStreetMapSchema(File file, RelDataTypeFactory typeFactory, boolean isDirectory) throws IOException { + if (isDirectory) { + // If isDirectory is true, treat the file as a directory + this.directory = Objects.requireNonNull(file, "Directory cannot be null"); + this.typeFactory = Objects.requireNonNull(typeFactory, "Type factory cannot be null"); + this.tableMap = new HashMap<>(); + + // Process files in the directory + File[] files = file.listFiles((dir, name) -> + name.toLowerCase().endsWith(".pbf") || + name.toLowerCase().endsWith(".osm.pbf") || + name.toLowerCase().endsWith(".xml") || + name.toLowerCase().endsWith(".osm")); + + if (files != null) { + for (File osmFile : files) { + // Extract the base name without extension (e.g., "sample" from "sample.osm.pbf") + String fileName = osmFile.getName(); + String tableName = fileName; + + // Remove all extensions (e.g., "sample.osm.pbf" -> "sample") + while (tableName.contains(".")) { + int lastDotIndex = tableName.lastIndexOf('.'); + if (lastDotIndex > 0) { + tableName = tableName.substring(0, lastDotIndex); + } else { + break; + } + } + + // Create the table with the file reference + tableMap.put(tableName, createTable(osmFile)); + } + } + } else { + // If isDirectory is false, treat the file as a single file + this.directory = Objects.requireNonNull(file, "File cannot be null"); + this.typeFactory = Objects.requireNonNull(typeFactory, "Type factory cannot be null"); + this.tableMap = new HashMap<>(); + + // Extract the base name without extension (e.g., "sample" from "sample.osm.pbf") + String fileName = file.getName(); + String tableName = fileName; + + // Remove all extensions (e.g., "sample.osm.pbf" -> "sample") + while (tableName.contains(".")) { + int lastDotIndex = tableName.lastIndexOf('.'); + if (lastDotIndex > 0) { + tableName = tableName.substring(0, lastDotIndex); + } else { + break; + } + } + + // Create the table with the file reference + tableMap.put(tableName, createTable(file)); + } + } + + /** + * Creates a table for the given file. + * + * @param file the OpenStreetMap file + * @return the created table + */ + private Table createTable(File file) { + // Determine the appropriate entity reader based on file extension + OpenStreetMapFormat.EntityReader<Entity> entityReader; + if (file.getName().toLowerCase().endsWith(".pbf") || + file.getName().toLowerCase().endsWith(".osm.pbf")) { + PbfEntityReader pbfReader = new PbfEntityReader(); + pbfReader.setGeometries(true); + pbfReader.setCoordinateMap(new HashMap<>()); + pbfReader.setReferenceMap(new HashMap<>()); + entityReader = pbfReader; + } else { + XmlEntityReader xmlReader = new XmlEntityReader(); + xmlReader.setGeometries(true); + xmlReader.setCoordinateMap(new HashMap<>()); + xmlReader.setReferenceMap(new HashMap<>()); + entityReader = xmlReader; + } + + // Create the table with the file reference + return new OpenStreetMapTable(file, entityReader); + } + + @Override + protected Map<String, Table> getTableMap() { + return tableMap; + } +} \ No newline at end of file diff --git a/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/openstreetmap/OpenStreetMapTable.java b/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/openstreetmap/OpenStreetMapTable.java index 93cd87d44..6be494888 100644 --- a/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/openstreetmap/OpenStreetMapTable.java +++ b/baremaps-calcite/src/main/java/org/apache/baremaps/calcite/openstreetmap/OpenStreetMapTable.java @@ -17,6 +17,9 @@ package org.apache.baremaps.calcite.openstreetmap; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; import java.io.InputStream; import java.util.*; import java.util.stream.Stream; @@ -40,19 +43,19 @@ import org.locationtech.jts.geom.Geometry; */ public class OpenStreetMapTable extends AbstractTable implements ScannableTable { + private final File file; private final EntityReader<Entity> entityReader; - private final InputStream inputStream; private RelDataType rowType; /** * Constructs an OpenStreetMapTable with the specified parameters. * + * @param file the OpenStreetMap file * @param entityReader the EntityReader for parsing the OSM data - * @param inputStream the input stream containing the OSM data */ - public OpenStreetMapTable(EntityReader<Entity> entityReader, InputStream inputStream) { - this.entityReader = entityReader; - this.inputStream = inputStream; + public OpenStreetMapTable(File file, EntityReader<Entity> entityReader) { + this.file = Objects.requireNonNull(file, "File cannot be null"); + this.entityReader = Objects.requireNonNull(entityReader, "Entity reader cannot be null"); } @Override @@ -97,7 +100,11 @@ public class OpenStreetMapTable extends AbstractTable implements ScannableTable return new AbstractEnumerable<Object[]>() { @Override public Enumerator<Object[]> enumerator() { - return new OpenStreetMapEnumerator(entityReader, inputStream); + try { + return new OpenStreetMapEnumerator(entityReader, new FileInputStream(file)); + } catch (IOException e) { + throw new RuntimeException("Failed to open input stream", e); + } } }; } diff --git a/baremaps-calcite/src/test/java/org/apache/baremaps/calcite/openstreetmap/OpenStreetMapSchemaTest.java b/baremaps-calcite/src/test/java/org/apache/baremaps/calcite/openstreetmap/OpenStreetMapSchemaTest.java new file mode 100644 index 000000000..8cedb78ad --- /dev/null +++ b/baremaps-calcite/src/test/java/org/apache/baremaps/calcite/openstreetmap/OpenStreetMapSchemaTest.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.baremaps.calcite.openstreetmap; + +import static org.junit.jupiter.api.Assertions.*; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.HashMap; +import java.util.Properties; +import org.apache.baremaps.openstreetmap.pbf.PbfEntityReader; +import org.apache.baremaps.testing.TestFiles; +import org.apache.calcite.jdbc.CalciteConnection; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.schema.SchemaPlus; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +public class OpenStreetMapSchemaTest { + + @TempDir + Path tempDir; + + private Path sampleDataDir; + private RelDataTypeFactory typeFactory; + + @BeforeEach + void setup() throws IOException, SQLException { + // Create a temporary directory for test files + sampleDataDir = tempDir.resolve("osm-data"); + Files.createDirectories(sampleDataDir); + + // Get the absolute paths to the sample files + Path pbfSourcePath = TestFiles.SAMPLE_OSM_PBF.toAbsolutePath(); + Path xmlSourcePath = TestFiles.SAMPLE_OSM_XML.toAbsolutePath(); + + // Copy sample OSM files to the test directory + Path pbfPath = sampleDataDir.resolve("sample.osm.pbf"); + Path xmlPath = sampleDataDir.resolve("sample.osm.xml"); + + // Check if source files exist + if (!Files.exists(pbfSourcePath)) { + throw new IOException("Sample PBF file not found: " + pbfSourcePath); + } + if (!Files.exists(xmlSourcePath)) { + throw new IOException("Sample XML file not found: " + xmlSourcePath); + } + + Files.copy(pbfSourcePath, pbfPath); + Files.copy(xmlSourcePath, xmlPath); + + // Set up Calcite connection to get a RelDataTypeFactory + Properties info = new Properties(); + info.setProperty("lex", "MYSQL"); + + try (Connection connection = DriverManager.getConnection("jdbc:calcite:", info)) { + CalciteConnection calciteConnection = connection.unwrap(CalciteConnection.class); + typeFactory = calciteConnection.getTypeFactory(); + } + } + + @Test + void testSchemaCreation() throws IOException { + // Create an OpenStreetMapSchema with the test directory + OpenStreetMapSchema schema = new OpenStreetMapSchema(sampleDataDir.toFile(), typeFactory); + + // Verify that the schema contains the expected tables + // The table name is based on the filename without extension + assertTrue(schema.getTableMap().containsKey("sample"), "Schema should contain 'sample' table"); + + // Verify that the table has the expected structure + OpenStreetMapTable table = (OpenStreetMapTable) schema.getTableMap().get("sample"); + assertNotNull(table, "Table should not be null"); + + // Verify the schema structure + int fieldCount = table.getRowType(typeFactory).getFieldCount(); + assertEquals(9, fieldCount, "Schema should have 9 columns"); + } + + @Test + void testSqlQueryWithDirectory() throws Exception { + // Create an OpenStreetMapSchema with the test directory + OpenStreetMapSchema schema = new OpenStreetMapSchema(sampleDataDir.toFile(), typeFactory); + + // Configure Calcite connection properties + Properties info = new Properties(); + info.setProperty("lex", "MYSQL"); + + // Set up a connection and register our schema + try (Connection connection = DriverManager.getConnection("jdbc:calcite:", info)) { + CalciteConnection calciteConnection = connection.unwrap(CalciteConnection.class); + SchemaPlus rootSchema = calciteConnection.getRootSchema(); + + // Add the schema to the root schema + rootSchema.add("osm", schema); + + // Test a simple query to select a limited number of entities + // The table name is based on the filename without extension + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery("SELECT id, type FROM osm.sample LIMIT 10")) { + int rowCount = 0; + + while (resultSet.next()) { + rowCount++; + long id = resultSet.getLong("id"); + String type = resultSet.getString("type"); + + // Verify basic properties + assertTrue(id != 0, "Entity should have non-zero ID"); + assertNotNull(type, "Entity should have a type"); + } + + // Verify that we got some rows + assertTrue(rowCount > 0, "Should have retrieved at least one entity"); + } + } + } + + @Test + void testSqlQueryWithSingleFile() throws Exception { + // Create a properly configured PbfEntityReader + PbfEntityReader entityReader = new PbfEntityReader(); + entityReader.setGeometries(true); + entityReader.setCoordinateMap(new HashMap<>()); + entityReader.setReferenceMap(new HashMap<>()); + + // Create an OpenStreetMapSchema with a single file + File pbfFile = sampleDataDir.resolve("sample.osm.pbf").toFile(); + OpenStreetMapSchema schema = new OpenStreetMapSchema(pbfFile, typeFactory, false); + + // Configure Calcite connection properties + Properties info = new Properties(); + info.setProperty("lex", "MYSQL"); + + // Set up a connection and register our schema + try (Connection connection = DriverManager.getConnection("jdbc:calcite:", info)) { + CalciteConnection calciteConnection = connection.unwrap(CalciteConnection.class); + SchemaPlus rootSchema = calciteConnection.getRootSchema(); + + // Add the schema to the root schema + rootSchema.add("osm", schema); + + // Test a simple query to select a limited number of entities + // For a single file, the table name is "sample" (not "osm") + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery("SELECT id, type FROM osm.sample LIMIT 10")) { + int rowCount = 0; + + while (resultSet.next()) { + rowCount++; + long id = resultSet.getLong("id"); + String type = resultSet.getString("type"); + + // Verify basic properties + assertTrue(id != 0, "Entity should have non-zero ID"); + assertNotNull(type, "Entity should have a type"); + } + + // Verify that we got some rows + assertTrue(rowCount > 0, "Should have retrieved at least one entity"); + } + } + } +} \ No newline at end of file diff --git a/baremaps-calcite/src/test/java/org/apache/baremaps/calcite/openstreetmap/OpenStreetMapTableTest.java b/baremaps-calcite/src/test/java/org/apache/baremaps/calcite/openstreetmap/OpenStreetMapTableTest.java index 73aa021ae..74e2f1eaa 100644 --- a/baremaps-calcite/src/test/java/org/apache/baremaps/calcite/openstreetmap/OpenStreetMapTableTest.java +++ b/baremaps-calcite/src/test/java/org/apache/baremaps/calcite/openstreetmap/OpenStreetMapTableTest.java @@ -51,7 +51,7 @@ public class OpenStreetMapTableTest { entityReader.setGeometries(false); // Don't generate geometries to avoid errors // Create the OpenStreetMapTable - OpenStreetMapTable osmTable = new OpenStreetMapTable(entityReader, inputStream); + OpenStreetMapTable osmTable = new OpenStreetMapTable(SAMPLE_OSM_PATH.toFile(), entityReader); // Verify the schema structure RelDataTypeFactory typeFactory = new JavaTypeFactoryImpl(); @@ -91,7 +91,7 @@ public class OpenStreetMapTableTest { try (var inputStream = new FileInputStream(SAMPLE_OSM_PATH.toFile())) { // Create the table with our configured reader - OpenStreetMapTable osmTable = new OpenStreetMapTable(entityReader, inputStream); + OpenStreetMapTable osmTable = new OpenStreetMapTable(SAMPLE_OSM_PATH.toFile(), entityReader); // Configure Calcite connection properties Properties info = new Properties();
