github-advanced-security[bot] commented on code in PR #895: URL: https://github.com/apache/incubator-baremaps/pull/895#discussion_r1795434479
########## baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/GeoParquetGroup.java: ########## @@ -0,0 +1,639 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.baremaps.geoparquet; + +import java.util.ArrayList; +import java.util.List; +import org.apache.parquet.io.api.Binary; +import org.apache.parquet.schema.GroupType; +import org.locationtech.jts.geom.Envelope; +import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.io.ParseException; +import org.locationtech.jts.io.WKBReader; +import org.locationtech.jts.io.WKBWriter; + +public class GeoParquetGroup { + + private final GroupType schema; + private final GeoParquetMetadata metadata; + private final Schema geoParquetSchema; + private final Object[] data; + + public GeoParquetGroup(GroupType schema, GeoParquetMetadata metadata, Schema geoParquetSchema) { + this.schema = schema; + this.metadata = metadata; + this.geoParquetSchema = geoParquetSchema; + this.data = new Object[schema.getFields().size()]; + for (int i = 0; i < schema.getFieldCount(); i++) { + Field field = geoParquetSchema.fields().get(i); + if (field.cardinality() == Cardinality.REPEATED) { + this.data[i] = new ArrayList<>(); + } else { + this.data[i] = null; // For REQUIRED or OPTIONAL fields + } + } + } + + public GeoParquetGroup addGroup(int fieldIndex) { + GeoParquetGroup group = createGroup(fieldIndex); + add(fieldIndex, group); + return group; + } + + public GeoParquetGroup addGroup(String field) { + return addGroup(getParquetSchema().getFieldIndex(field)); + } + + public GeoParquetGroup getGroup(int fieldIndex, int index) { + return (GeoParquetGroup) getValue(fieldIndex, index); + } + + public GeoParquetGroup getGroup(String field, int index) { + return getGroup(getParquetSchema().getFieldIndex(field), index); + } + + public int getFieldRepetitionCount(int fieldIndex) { + Object value = data[fieldIndex]; + if (value instanceof List<?>list) { + return list.size(); + } else { + return value == null ? 0 : 1; + } + } + + private Object getValue(int fieldIndex, int index) { + Object value = data[fieldIndex]; + if (value instanceof List<?>list) { + return list.get(index); + } else if (index == 0) { + return value; + } else { + throw createGeoParquetException(fieldIndex, "element number " + index); + } + } + + private Object getValue(int fieldIndex) { + Field field = geoParquetSchema.fields().get(fieldIndex); + if (field.cardinality() == Cardinality.REPEATED) { + throw new IllegalStateException("Field " + fieldIndex + " (" + field.name() + + ") is repeated. Use getValues() instead."); + } + return data[fieldIndex]; + } + + public List<Object> getValues(int fieldIndex) { + Field field = geoParquetSchema.fields().get(fieldIndex); + if (field.cardinality() != Cardinality.REPEATED) { + return List.of(getValue(fieldIndex)); + } + return (List<Object>) data[fieldIndex]; + } + + private GeoParquetGroup getGroup(int fieldIndex) { + return (GeoParquetGroup) data[fieldIndex]; + } + + private void addValue(int fieldIndex, Object value) { + Object currentValue = data[fieldIndex]; + if (currentValue instanceof List<?>) { + ((List<Object>) currentValue).add(value); + } else { + data[fieldIndex] = value; + } + } + + public void add(int fieldIndex, int value) { + addValue(fieldIndex, value); + } + + public void add(int fieldIndex, long value) { + addValue(fieldIndex, value); + } + + public void add(int fieldIndex, String value) { + addValue(fieldIndex, Binary.fromString(value)); + } + + public void add(int fieldIndex, boolean value) { + addValue(fieldIndex, value); + } + + public void add(int fieldIndex, Binary value) { + addValue(fieldIndex, value); + } + + public void add(int fieldIndex, float value) { + addValue(fieldIndex, value); + } + + public void add(int fieldIndex, double value) { + addValue(fieldIndex, value); + } + + public void add(int fieldIndex, GeoParquetGroup value) { + addValue(fieldIndex, value); + } + + public void add(int fieldIndex, Geometry geometry) { + byte[] bytes = new WKBWriter().write(geometry); + add(fieldIndex, Binary.fromConstantByteArray(bytes)); + } + + public void add(String field, int value) { + add(getParquetSchema().getFieldIndex(field), value); + } + + public void add(String field, long value) { + add(getParquetSchema().getFieldIndex(field), value); + } + + public void add(String field, float value) { + add(getParquetSchema().getFieldIndex(field), value); + } + + public void add(String field, double value) { + add(getParquetSchema().getFieldIndex(field), value); + } + + public void add(String field, String value) { + add(getParquetSchema().getFieldIndex(field), value); + } + + public void add(String field, boolean value) { + add(getParquetSchema().getFieldIndex(field), value); + } + + public void add(String field, Binary value) { + add(getParquetSchema().getFieldIndex(field), value); + } + + public void add(String field, GeoParquetGroup value) { + add(getParquetSchema().getFieldIndex(field), value); + } + + public void add(String field, Geometry geometry) { + add(getParquetSchema().getFieldIndex(field), geometry); + } + + private GeoParquetException createGeoParquetException(int fieldIndex, String elementText) { + String msg = String.format("Not found %d (%s) %s in group%n%s", fieldIndex, + schema.getFieldName(fieldIndex), elementText, this); + return new GeoParquetException(msg); + } + + public String toString() { Review Comment: ## Missing Override annotation This method overrides [Object.toString](1); it is advisable to add an Override annotation. [Show more details](https://github.com/apache/incubator-baremaps/security/code-scanning/1574) ########## baremaps-geoparquet/src/main/java/org/apache/baremaps/geoparquet/GeoParquetReader.java: ########## @@ -22,149 +22,178 @@ import java.io.IOException; import java.net.URI; import java.util.*; +import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.stream.StreamSupport; -import org.apache.baremaps.geoparquet.data.GeoParquetGroup; -import org.apache.baremaps.geoparquet.data.GeoParquetGroup.Schema; -import org.apache.baremaps.geoparquet.data.GeoParquetGroupFactory; -import org.apache.baremaps.geoparquet.data.GeoParquetMetadata; +import org.apache.baremaps.geoparquet.GeoParquetGroup.Schema; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider; import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.parquet.hadoop.ParquetFileReader; +import org.apache.parquet.hadoop.metadata.BlockMetaData; +import org.apache.parquet.hadoop.metadata.FileMetaData; +import org.apache.parquet.hadoop.metadata.ParquetMetadata; import org.apache.parquet.schema.MessageType; - +import org.locationtech.jts.geom.Envelope; /** * This reader is based on the parquet example code located at: org.apache.parquet.example.data.*. */ public class GeoParquetReader { - private final URI uri; - final Configuration configuration; - private List<FileStatus> files; - private Long groupCount; + protected final Configuration configuration; + protected final List<FileStatus> files; + private final AtomicLong groupCount = new AtomicLong(-1); + private final Envelope envelope; - record FileInfo(FileStatus file, Long recordCount, Map<String, String> keyValueMetadata, - MessageType messageType, GeoParquetMetadata metadata, - GeoParquetGroup.Schema geoParquetSchema) { + public GeoParquetReader(URI uri) { + this(uri, null, createDefaultConfiguration()); } - public GeoParquetReader(URI uri) { - this(uri, createConfiguration()); + public GeoParquetReader(URI uri, Envelope envelope) { + this(uri, envelope, createDefaultConfiguration()); } - public GeoParquetReader(URI uri, Configuration configuration) { - this.uri = uri; + public GeoParquetReader(URI uri, Envelope envelope, Configuration configuration) { this.configuration = configuration; + this.files = initializeFiles(uri, configuration); + this.envelope = envelope; + } + + private static List<FileStatus> initializeFiles(URI uri, Configuration configuration) { + try { + Path globPath = new Path(uri.getPath()); + FileSystem fileSystem = FileSystem.get(uri, configuration); + FileStatus[] fileStatuses = fileSystem.globStatus(globPath); + if (fileStatuses == null) { + throw new GeoParquetException("No files found at the specified URI."); + } + return Collections.unmodifiableList(Arrays.asList(fileStatuses)); + } catch (IOException e) { + throw new GeoParquetException("IOException while attempting to list files.", e); + } } public MessageType getParquetSchema() { - return files().stream() + return files.stream() .findFirst() .map(this::getFileInfo) - .orElseThrow() - .messageType(); + .orElseThrow( + () -> new GeoParquetException("No files available to read schema.")).messageType; } private FileInfo getFileInfo(FileStatus fileStatus) { try { - return buildFileInfo(fileStatus); + ParquetMetadata parquetMetadata = + ParquetFileReader.readFooter(configuration, fileStatus.getPath()); Review Comment: ## Deprecated method or constructor invocation Invoking [ParquetFileReader.readFooter](1) should be avoided because it has been deprecated. [Show more details](https://github.com/apache/incubator-baremaps/security/code-scanning/1575) -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
