http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/org/apache/sdap/ningester/processors/PythonChainProcessor.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sdap/ningester/processors/PythonChainProcessor.java b/src/main/java/org/apache/sdap/ningester/processors/PythonChainProcessor.java new file mode 100644 index 0000000..a194114 --- /dev/null +++ b/src/main/java/org/apache/sdap/ningester/processors/PythonChainProcessor.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.sdap.ningester.processors; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; +import org.apache.sdap.nexusproto.NexusTile; +import org.apache.sdap.ningester.processors.properties.PythonProcessorModule; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; +import org.springframework.web.client.RestTemplate; + +import java.util.Collections; +import java.util.List; + +public class PythonChainProcessor { + + private RestTemplate restTemplate; + + private List<PythonProcessorModule> processorList; + + private String uriPath; + + public PythonChainProcessor(RestTemplate restTemplate) { + this.restTemplate = restTemplate; + } + + public NexusTile nexusTileProcessor(NexusTile nexusTile) { + + HttpHeaders headers = new HttpHeaders(); + headers.setAccept(Collections.singletonList(MediaType.APPLICATION_OCTET_STREAM)); + headers.setContentType(MediaType.APPLICATION_JSON); + + PythonChainProcessorRequest chainProcessorRequest = new PythonChainProcessorRequest(); + chainProcessorRequest.setProcessorList(processorList); + try { + chainProcessorRequest.setNexusTile(JsonFormat.printer().print(nexusTile)); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + + HttpEntity<PythonChainProcessorRequest> requestEntity = new HttpEntity<>(chainProcessorRequest, headers); + + return restTemplate.exchange( + uriPath, + HttpMethod.POST, + requestEntity, + NexusTile.class).getBody(); + } + + public void setProcessorList(List<PythonProcessorModule> processorList) { + this.processorList = processorList; + } + + public void setUriPath(String uriPath) { + this.uriPath = uriPath; + } + + public class PythonChainProcessorRequest { + + @JsonProperty("processor_list") + private List<PythonProcessorModule> processorList; + + @JsonProperty("input_data") + private String nexusTile; + + public List<PythonProcessorModule> getProcessorList() { + return processorList; + } + + public void setProcessorList(List<PythonProcessorModule> processorList) { + this.processorList = processorList; + } + + public String getNexusTile() { + return nexusTile; + } + + public void setNexusTile(String nexusTile) { + this.nexusTile = nexusTile; + } + } + +}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/org/apache/sdap/ningester/processors/properties/AddDatasetName.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sdap/ningester/processors/properties/AddDatasetName.java b/src/main/java/org/apache/sdap/ningester/processors/properties/AddDatasetName.java new file mode 100644 index 0000000..e2a75c1 --- /dev/null +++ b/src/main/java/org/apache/sdap/ningester/processors/properties/AddDatasetName.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.sdap.ningester.processors.properties; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@ConfigurationProperties +@Component("addDatasetNameProperties") +public class AddDatasetName { + + private String datasetName; + + public String getDatasetName() { + return datasetName; + } + + public void setDatasetName(String datasetName) { + this.datasetName = datasetName; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/org/apache/sdap/ningester/processors/properties/AddDayOfYearAttribute.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sdap/ningester/processors/properties/AddDayOfYearAttribute.java b/src/main/java/org/apache/sdap/ningester/processors/properties/AddDayOfYearAttribute.java new file mode 100644 index 0000000..9a0737e --- /dev/null +++ b/src/main/java/org/apache/sdap/ningester/processors/properties/AddDayOfYearAttribute.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.sdap.ningester.processors.properties; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@ConfigurationProperties +@Component("addDayOfYearAttributeProperties") +public class AddDayOfYearAttribute { + + private String regex; + + + public String getRegex() { + return regex; + } + + public void setRegex(String regex) { + this.regex = regex; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/org/apache/sdap/ningester/processors/properties/AddTimeFromGranuleName.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sdap/ningester/processors/properties/AddTimeFromGranuleName.java b/src/main/java/org/apache/sdap/ningester/processors/properties/AddTimeFromGranuleName.java new file mode 100644 index 0000000..b2cea56 --- /dev/null +++ b/src/main/java/org/apache/sdap/ningester/processors/properties/AddTimeFromGranuleName.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.sdap.ningester.processors.properties; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@ConfigurationProperties +@Component("addTimeFromGranuleNameProperties") +public class AddTimeFromGranuleName { + + private String regex; + private String dateFormat; + + public String getRegex() { + return regex; + } + + public void setRegex(String regex) { + this.regex = regex; + } + + public String getDateFormat() { + return dateFormat; + } + + public void setDateFormat(String dateFormat) { + this.dateFormat = dateFormat; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/org/apache/sdap/ningester/processors/properties/GenerateTileId.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sdap/ningester/processors/properties/GenerateTileId.java b/src/main/java/org/apache/sdap/ningester/processors/properties/GenerateTileId.java new file mode 100644 index 0000000..09cd7a3 --- /dev/null +++ b/src/main/java/org/apache/sdap/ningester/processors/properties/GenerateTileId.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.sdap.ningester.processors.properties; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@ConfigurationProperties +@Component("generateTileIdProperties") +public class GenerateTileId { + + private String salt = ""; + + public String getSalt() { + return salt; + } + + public void setSalt(String salt) { + this.salt = salt; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/org/apache/sdap/ningester/processors/properties/PythonChainProcessor.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sdap/ningester/processors/properties/PythonChainProcessor.java b/src/main/java/org/apache/sdap/ningester/processors/properties/PythonChainProcessor.java new file mode 100644 index 0000000..5b36fe7 --- /dev/null +++ b/src/main/java/org/apache/sdap/ningester/processors/properties/PythonChainProcessor.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.sdap.ningester.processors.properties; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +import java.net.URL; +import java.util.ArrayList; +import java.util.List; + +@ConfigurationProperties +@Component("pythonChainProcessorProperties") +public class PythonChainProcessor { + private URL baseUrl; + private String uriPath; + + private List<PythonProcessorModule> processorList = new ArrayList<>(); + + public String getUriPath() { + return uriPath; + } + + public void setUriPath(String uriPath) { + this.uriPath = uriPath; + } + + public List<PythonProcessorModule> getProcessorList() { + return processorList; + } + + public URL getBaseUrl() { + return baseUrl; + } + + public void setBaseUrl(URL baseUrl) { + this.baseUrl = baseUrl; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/org/apache/sdap/ningester/processors/properties/PythonProcessorModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sdap/ningester/processors/properties/PythonProcessorModule.java b/src/main/java/org/apache/sdap/ningester/processors/properties/PythonProcessorModule.java new file mode 100644 index 0000000..fd13178 --- /dev/null +++ b/src/main/java/org/apache/sdap/ningester/processors/properties/PythonProcessorModule.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.sdap.ningester.processors.properties; + +import java.util.HashMap; +import java.util.Map; + +public class PythonProcessorModule { + + private String name; + private Map<String, String> config = new HashMap<>(); + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Map<String, String> getConfig() { + return config; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/org/apache/sdap/ningester/writer/CassandraStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sdap/ningester/writer/CassandraStore.java b/src/main/java/org/apache/sdap/ningester/writer/CassandraStore.java new file mode 100644 index 0000000..4130113 --- /dev/null +++ b/src/main/java/org/apache/sdap/ningester/writer/CassandraStore.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.sdap.ningester.writer; + +import org.apache.sdap.nexusproto.NexusTile; +import org.apache.sdap.nexusproto.TileData; +import org.springframework.data.cassandra.core.CassandraOperations; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; + +public class CassandraStore implements DataStore { + + private CassandraOperations cassandraTemplate; + + //TODO This will be refactored at some point to be dynamic per-message. Or maybe per-group. + private String tableName = "sea_surface_temp"; + + public CassandraStore(CassandraOperations cassandraTemplate) { + this.cassandraTemplate = cassandraTemplate; + } + + @Override + public void saveData(List<? extends NexusTile> nexusTiles) { + + String query = "insert into " + this.tableName + " (tile_id, tile_blob) VALUES (?, ?)"; + cassandraTemplate.ingest(query, nexusTiles.stream() + .map(nexusTile -> getCassandraRowFromTileData(nexusTile.getTile())) + .collect(Collectors.toList())); + } + + private List<Object> getCassandraRowFromTileData(TileData tile) { + + UUID tileId = UUID.fromString(tile.getTileId()); + return Arrays.asList(tileId, ByteBuffer.wrap(tile.toByteArray())); + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/org/apache/sdap/ningester/writer/DataStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sdap/ningester/writer/DataStore.java b/src/main/java/org/apache/sdap/ningester/writer/DataStore.java new file mode 100644 index 0000000..c5e1a36 --- /dev/null +++ b/src/main/java/org/apache/sdap/ningester/writer/DataStore.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.sdap.ningester.writer; + +import org.apache.sdap.nexusproto.NexusTile; + +import java.util.List; + +public interface DataStore { + + void saveData(List<? extends NexusTile> nexusTiles); +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/org/apache/sdap/ningester/writer/DataStoreException.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sdap/ningester/writer/DataStoreException.java b/src/main/java/org/apache/sdap/ningester/writer/DataStoreException.java new file mode 100644 index 0000000..e68cc40 --- /dev/null +++ b/src/main/java/org/apache/sdap/ningester/writer/DataStoreException.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sdap.ningester.writer; + +public class DataStoreException extends RuntimeException { + + public DataStoreException() { + } + + public DataStoreException(String message) { + super(message); + } + + public DataStoreException(String message, Throwable cause) { + super(message, cause); + } + + public DataStoreException(Throwable cause) { + super(cause); + } + + public DataStoreException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/org/apache/sdap/ningester/writer/DynamoStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sdap/ningester/writer/DynamoStore.java b/src/main/java/org/apache/sdap/ningester/writer/DynamoStore.java new file mode 100644 index 0000000..3f4860a --- /dev/null +++ b/src/main/java/org/apache/sdap/ningester/writer/DynamoStore.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sdap.ningester.writer; + +import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; +import com.amazonaws.services.dynamodbv2.document.DynamoDB; +import com.amazonaws.services.dynamodbv2.document.Item; +import com.amazonaws.services.dynamodbv2.document.Table; +import org.apache.sdap.nexusproto.NexusTile; + +import java.util.List; + +/** + * Created by djsilvan on 6/26/17. + */ +public class DynamoStore implements DataStore { + + private DynamoDB dynamoDB; + private String tableName; + private String primaryKey; + + public DynamoStore(AmazonDynamoDB dynamoClient, String tableName, String primaryKey) { + dynamoDB = new DynamoDB(dynamoClient); + this.tableName = tableName; + this.primaryKey = primaryKey; + } + + public void saveData(List<? extends NexusTile> nexusTiles) { + + Table table = dynamoDB.getTable(tableName); + + for (NexusTile tile : nexusTiles) { + String tileId = getTileId(tile); + byte[] tileData = getTileData(tile); + + try { + table.putItem(new Item().withPrimaryKey(primaryKey, tileId).withBinary("data", tileData)); + } catch (Exception e) { + throw new DataStoreException("Unable to add item: " + tileId, e); + } + } + } + + private String getTileId(NexusTile tile) { + return tile.getTile().getTileId(); + } + + private byte[] getTileData(NexusTile tile) { + return tile.getTile().toByteArray(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/org/apache/sdap/ningester/writer/MetadataStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sdap/ningester/writer/MetadataStore.java b/src/main/java/org/apache/sdap/ningester/writer/MetadataStore.java new file mode 100644 index 0000000..b44bb70 --- /dev/null +++ b/src/main/java/org/apache/sdap/ningester/writer/MetadataStore.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.sdap.ningester.writer; + +import org.apache.sdap.nexusproto.NexusTile; + +import java.util.List; + +public interface MetadataStore { + + void saveMetadata(List<? extends NexusTile> nexusTiles); + + void deleteMetadata(List<? extends NexusTile> nexusTiles); +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/org/apache/sdap/ningester/writer/NexusWriter.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sdap/ningester/writer/NexusWriter.java b/src/main/java/org/apache/sdap/ningester/writer/NexusWriter.java new file mode 100644 index 0000000..69aee86 --- /dev/null +++ b/src/main/java/org/apache/sdap/ningester/writer/NexusWriter.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sdap.ningester.writer; + +import org.apache.sdap.nexusproto.NexusTile; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + +public class NexusWriter { + + private static final Logger log = LoggerFactory.getLogger(NexusWriter.class); + private MetadataStore metadataStore; + private DataStore dataStore; + + public NexusWriter(MetadataStore metadataStore, DataStore dataStore) { + this.metadataStore = metadataStore; + this.dataStore = dataStore; + } + + public void saveToNexus(List<? extends NexusTile> nexusTiles) { + if (nexusTiles.size() > 0) { + metadataStore.saveMetadata(nexusTiles); + + try { + dataStore.saveData(nexusTiles); + + } catch (RuntimeException e) { + try { + metadataStore.deleteMetadata(nexusTiles); + } catch (RuntimeException e2) { + log.error("During exception while saving data, could not rollback metadata", e2); + } + throw e; + } + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/org/apache/sdap/ningester/writer/S3Store.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sdap/ningester/writer/S3Store.java b/src/main/java/org/apache/sdap/ningester/writer/S3Store.java new file mode 100644 index 0000000..fd2b2c0 --- /dev/null +++ b/src/main/java/org/apache/sdap/ningester/writer/S3Store.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sdap.ningester.writer; + +import com.amazonaws.AmazonClientException; +import com.amazonaws.AmazonServiceException; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.PutObjectRequest; +import org.apache.sdap.nexusproto.NexusTile; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.util.List; + +/** + * Created by djsilvan on 6/26/17. + */ +public class S3Store implements DataStore { + + private AmazonS3 s3; + private String bucketName; + + public S3Store(AmazonS3Client s3client, String bucketName) { + s3 = s3client; + this.bucketName = bucketName; + } + + public void saveData(List<? extends NexusTile> nexusTiles) { + + for (NexusTile tile : nexusTiles) { + String tileId = getTileId(tile); + byte[] tileData = getTileData(tile); + Long contentLength = (long) tileData.length; + InputStream stream = new ByteArrayInputStream(tileData); + ObjectMetadata meta = new ObjectMetadata(); + meta.setContentLength(contentLength); + + try { + s3.putObject(new PutObjectRequest(bucketName, tileId, stream, meta)); + } catch (AmazonServiceException ase) { + throw new DataStoreException("Caught an AmazonServiceException, which means your request made it " + + "to Amazon S3, but was rejected with an error response for some reason.", ase); + } catch (AmazonClientException ace) { + throw new DataStoreException("Caught an AmazonClientException, which means the client encountered " + + "a serious internal problem while trying to communicate with S3, " + + "such as not being able to access the network.", ace); + } + } + } + + private String getTileId(NexusTile tile) { + return tile.getTile().getTileId(); + } + + private byte[] getTileData(NexusTile tile) { + return tile.getTile().toByteArray(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/org/apache/sdap/ningester/writer/SolrStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sdap/ningester/writer/SolrStore.java b/src/main/java/org/apache/sdap/ningester/writer/SolrStore.java new file mode 100644 index 0000000..1eaffc4 --- /dev/null +++ b/src/main/java/org/apache/sdap/ningester/writer/SolrStore.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.sdap.ningester.writer; + +import org.apache.sdap.nexusproto.NexusTile; +import org.apache.sdap.nexusproto.TileSummary; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.SolrInputField; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.data.solr.core.SolrOperations; + +import java.math.BigDecimal; +import java.nio.file.Paths; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.stream.Collectors; + +public class SolrStore implements MetadataStore { + + //TODO This will be refactored at some point to be dynamic per-message. Or maybe per-group. + private static final String TABLE_NAME = "sea_surface_temp"; + private static final SimpleDateFormat iso = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); + + static { + iso.setTimeZone(TimeZone.getTimeZone("UTC")); + } + + private Integer commitWithin = 1000; + private Integer geoPrecision = 3; + private String collection = "nexustiles"; + private SolrOperations solr; + private Logger log = LoggerFactory.getLogger(SolrStore.class); + + public SolrStore(SolrOperations solr) { + this.solr = solr; + } + + @Override + public void saveMetadata(List<? extends NexusTile> nexusTiles) { + + List<SolrInputDocument> solrdocs = nexusTiles.stream() + .map(nexusTile -> getSolrDocFromTileSummary(nexusTile.getSummary())) + .collect(Collectors.toList()); + solr.saveDocuments(this.collection, solrdocs, commitWithin); + } + + @Override + public void deleteMetadata(List<? extends NexusTile> nexusTiles) { + + List<String> tileIds = nexusTiles.stream() + .map(nexusTile -> nexusTile.getSummary().getDatasetName() + "!" + nexusTile.getSummary().getTileId()) + .collect(Collectors.toList()); + solr.deleteById(this.collection, tileIds); + } + + public SolrInputDocument getSolrDocFromTileSummary(TileSummary summary) { + + TileSummary.BBox bbox = summary.getBbox(); + TileSummary.DataStats stats = summary.getStats(); + + Calendar startCal = Calendar.getInstance(TimeZone.getTimeZone("UTC")); + startCal.setTime(new Date(stats.getMinTime() * 1000)); + Calendar endCal = Calendar.getInstance(TimeZone.getTimeZone("UTC")); + endCal.setTime(new Date(stats.getMaxTime() * 1000)); + + String minTime = iso.format(startCal.getTime()); + String maxTime = iso.format(endCal.getTime()); + + String geo = determineGeo(summary); + + String granuleFileName = Paths.get(summary.getGranule()).getFileName().toString(); + + SolrInputDocument inputDocument = new SolrInputDocument(); + inputDocument.addField("table_s", TABLE_NAME); + inputDocument.addField("geo", geo); + inputDocument.addField("id", summary.getTileId()); + inputDocument.addField("solr_id_s", summary.getDatasetName() + "!" + summary.getTileId()); + inputDocument.addField("dataset_id_s", summary.getDatasetUuid()); + inputDocument.addField("sectionSpec_s", summary.getSectionSpec()); + inputDocument.addField("dataset_s", summary.getDatasetName()); + inputDocument.addField("granule_s", granuleFileName); + inputDocument.addField("tile_var_name_s", summary.getDataVarName()); + inputDocument.addField("tile_min_lon", bbox.getLonMin()); + inputDocument.addField("tile_max_lon", bbox.getLonMax()); + inputDocument.addField("tile_min_lat", bbox.getLatMin()); + inputDocument.addField("tile_max_lat", bbox.getLatMax()); + inputDocument.addField("tile_min_time_dt", minTime); + inputDocument.addField("tile_max_time_dt", maxTime); + inputDocument.addField("tile_min_val_d", stats.getMin()); + inputDocument.addField("tile_max_val_d", stats.getMax()); + inputDocument.addField("tile_avg_val_d", stats.getMean()); + inputDocument.addField("tile_count_i", Long.valueOf(stats.getCount()).intValue()); + + summary.getGlobalAttributesList().forEach(attribute -> + inputDocument.addField(attribute.getName(), attribute.getValuesCount() == 1 ? attribute.getValues(0) : attribute.getValuesList()) + ); + return inputDocument; + } + + private String determineGeo(TileSummary summary) { + //Solr cannot index a POLYGON where all corners are the same point or when there are only 2 distinct points (line). + //Solr is configured for a specific precision so we need to round to that precision before checking equality. + Integer geoPrecision = this.geoPrecision; + + BigDecimal latMin = BigDecimal.valueOf(summary.getBbox().getLatMin()).setScale(geoPrecision, BigDecimal.ROUND_HALF_UP); + BigDecimal latMax = BigDecimal.valueOf(summary.getBbox().getLatMax()).setScale(geoPrecision, BigDecimal.ROUND_HALF_UP); + BigDecimal lonMin = BigDecimal.valueOf(summary.getBbox().getLonMin()).setScale(geoPrecision, BigDecimal.ROUND_HALF_UP); + BigDecimal lonMax = BigDecimal.valueOf(summary.getBbox().getLonMax()).setScale(geoPrecision, BigDecimal.ROUND_HALF_UP); + + String geo; + //If lat min = lat max and lon min = lon max, index the 'geo' bounding box as a POINT instead of a POLYGON + if (latMin.equals(latMax) && lonMin.equals(lonMax)) { + geo = "POINT(" + lonMin + " " + latMin + ")"; + log.debug("{}\t{}[{}] geo={}", summary.getTileId(), summary.getGranule(), summary.getSectionSpec(), geo); + } + //If lat min = lat max but lon min != lon max, then we essentially have a line. + else if (latMin.equals(latMax)) { + geo = "LINESTRING (" + lonMin + " " + latMin + ", " + lonMax + " " + latMin + ")"; + log.debug("{}\t{}[{}] geo={}", summary.getTileId(), summary.getGranule(), summary.getSectionSpec(), geo); + } + //Same if lon min = lon max but lat min != lat max + else if (lonMin.equals(lonMax)) { + geo = "LINESTRING (" + lonMin + " " + latMin + ", " + lonMin + " " + latMax + ")"; + log.debug("{}\t{}[{}] geo={}", summary.getTileId(), summary.getGranule(), summary.getSectionSpec(), geo); + } + //All other cases should use POLYGON + else { + geo = "POLYGON((" + + lonMin + " " + latMin + ", " + + lonMax + " " + latMin + ", " + + lonMax + " " + latMax + ", " + + lonMin + " " + latMax + ", " + + lonMin + " " + latMin + "))"; + } + + return geo; + } + + private SolrInputDocument toSolrInputDocument(Map<String, Object> doc) { + SolrInputDocument solrDoc = new SolrInputDocument(); + + solrDoc.putAll(doc.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, entry -> { + SolrInputField field = new SolrInputField(entry.getKey()); + field.setValue(entry.getValue(), 0); + return field; + }))); + + return solrDoc; + } + + public void setCollection(String collection) { + this.collection = collection; + } + + public void setCommitWithin(Integer commitWithin) { + this.commitWithin = commitWithin; + } + + public void setGeoPrecision(Integer geoPrecision) { + this.geoPrecision = geoPrecision; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/org/apache/sdap/ningester/writer/properties/CassandraStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sdap/ningester/writer/properties/CassandraStore.java b/src/main/java/org/apache/sdap/ningester/writer/properties/CassandraStore.java new file mode 100644 index 0000000..bc866c7 --- /dev/null +++ b/src/main/java/org/apache/sdap/ningester/writer/properties/CassandraStore.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sdap.ningester.writer.properties; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@ConfigurationProperties +@Component("cassandraStoreProperties") +public class CassandraStore { + + private String tableName = "sea_surface_temp"; + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/org/apache/sdap/ningester/writer/properties/DynamoStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sdap/ningester/writer/properties/DynamoStore.java b/src/main/java/org/apache/sdap/ningester/writer/properties/DynamoStore.java new file mode 100644 index 0000000..9b0324c --- /dev/null +++ b/src/main/java/org/apache/sdap/ningester/writer/properties/DynamoStore.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.sdap.ningester.writer.properties; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@ConfigurationProperties("dynamo") +@Component("dynamoStoreProperties") +public class DynamoStore { + + private String tableName; + + private String region; + + private String primaryKey = "tile_id"; + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public String getRegion() { + return region; + } + + public void setRegion(String region) { + this.region = region; + } + + public String getPrimaryKey() { + return primaryKey; + } + + public void setPrimaryKey(String primaryKey) { + this.primaryKey = primaryKey; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/org/apache/sdap/ningester/writer/properties/S3Store.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sdap/ningester/writer/properties/S3Store.java b/src/main/java/org/apache/sdap/ningester/writer/properties/S3Store.java new file mode 100644 index 0000000..b3f9eed --- /dev/null +++ b/src/main/java/org/apache/sdap/ningester/writer/properties/S3Store.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.sdap.ningester.writer.properties; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@ConfigurationProperties("s3") +@Component("s3StoreProperties") +public class S3Store { + + private String bucketName; + + private String region; + + public String getRegion() { + return region; + } + + public void setRegion(String region) { + this.region = region; + } + + public String getBucketName() { + return bucketName; + } + + public void setBucketName(String bucketName) { + this.bucketName = bucketName; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/org/apache/sdap/ningester/writer/properties/SolrStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sdap/ningester/writer/properties/SolrStore.java b/src/main/java/org/apache/sdap/ningester/writer/properties/SolrStore.java new file mode 100644 index 0000000..cb70bd7 --- /dev/null +++ b/src/main/java/org/apache/sdap/ningester/writer/properties/SolrStore.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.sdap.ningester.writer.properties; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@ConfigurationProperties +@Component("solrStoreProperties") +public class SolrStore { + + + private Integer commitWithin = 1000; + private Integer geoPrecision = 3; + private String collection = "nexustiles"; + + + public String getCollection() { + return collection; + } + + public void setCollection(String collection) { + this.collection = collection; + } + + public Integer getCommitWithin() { + return commitWithin; + } + + public void setCommitWithin(Integer commitWithin) { + this.commitWithin = commitWithin; + } + + public Integer getGeoPrecision() { + return geoPrecision; + } + + public void setGeoPrecision(Integer geoPrecision) { + this.geoPrecision = geoPrecision; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/test/java/gov/nasa/jpl/nexus/ningester/NingesterApplicationTests.java ---------------------------------------------------------------------- diff --git a/src/test/java/gov/nasa/jpl/nexus/ningester/NingesterApplicationTests.java b/src/test/java/gov/nasa/jpl/nexus/ningester/NingesterApplicationTests.java deleted file mode 100644 index ab6735f..0000000 --- a/src/test/java/gov/nasa/jpl/nexus/ningester/NingesterApplicationTests.java +++ /dev/null @@ -1,16 +0,0 @@ -package gov.nasa.jpl.nexus.ningester; - -import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit4.SpringRunner; - -@RunWith(SpringRunner.class) -@SpringBootTest -public class NingesterApplicationTests { - - @Test - public void contextLoads() { - } - -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReaderTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReaderTest.java b/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReaderTest.java deleted file mode 100644 index a874ac0..0000000 --- a/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReaderTest.java +++ /dev/null @@ -1,80 +0,0 @@ -package gov.nasa.jpl.nexus.ningester.datatiler; - -import org.apache.sdap.nexusproto.NexusTile; -import org.junit.Test; -import org.springframework.batch.item.ExecutionContext; -import org.springframework.core.io.ClassPathResource; -import org.springframework.core.io.Resource; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.Assert.assertTrue; - -public class NetCDFItemReaderTest { - - @Test - public void testOpen() { - SliceFileByTilesDesired slicer = new SliceFileByTilesDesired(); - slicer.setTilesDesired(5184); - slicer.setDimensions(Arrays.asList("lat", "lon")); - - NetCDFItemReader reader = new NetCDFItemReader(slicer); - reader.setResource(new ClassPathResource("granules/20050101120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc")); - - ExecutionContext context = new ExecutionContext(); - reader.open(context); - - assertTrue(context.containsKey(NetCDFItemReader.CURRENT_TILE_SPEC_INDEX_KEY)); - } - - @Test - public void testReadOne() throws Exception { - SliceFileByTilesDesired slicer = new SliceFileByTilesDesired(); - slicer.setTilesDesired(5184); - slicer.setDimensions(Arrays.asList("lat", "lon")); - - Resource testResource = new ClassPathResource("granules/20050101120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc"); - NetCDFItemReader reader = new NetCDFItemReader(slicer); - reader.setResource(testResource); - - ExecutionContext context = new ExecutionContext(); - reader.open(context); - - NexusTile result = reader.read(); - - assertThat(result.getSummary().getSectionSpec(), is("lat:0:10,lon:0:20")); - assertThat(result.getSummary().getGranule(), is(testResource.getURL().toString())); - - } - - @Test - public void testReadAll() { - Integer tilesDesired = 5184; - - SliceFileByTilesDesired slicer = new SliceFileByTilesDesired(); - slicer.setTilesDesired(tilesDesired); - slicer.setDimensions(Arrays.asList("lat", "lon")); - slicer.setTimeDimension("time"); - - Resource testResource = new ClassPathResource("granules/20050101120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc"); - NetCDFItemReader reader = new NetCDFItemReader(slicer); - reader.setResource(testResource); - - ExecutionContext context = new ExecutionContext(); - reader.open(context); - - List<NexusTile> results = new ArrayList<>(); - NexusTile result; - while ((result = reader.read()) != null) { - results.add(result); - } - - assertThat(results.size(), is(tilesDesired)); - - } - -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByDimensionTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByDimensionTest.java b/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByDimensionTest.java deleted file mode 100644 index 26d6059..0000000 --- a/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByDimensionTest.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * **************************************************************************** - * Copyright (c) 2016 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - * ****************************************************************************/ -package gov.nasa.jpl.nexus.ningester.datatiler; - -import org.junit.Test; -import org.springframework.core.io.ClassPathResource; -import org.springframework.core.io.Resource; - -import java.io.IOException; -import java.util.Arrays; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -import static junit.framework.Assert.assertEquals; -import static org.hamcrest.CoreMatchers.hasItems; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.MatcherAssert.assertThat; - -public class SliceFileByDimensionTest { - - @Test - public void testGenerateTileBoundrySlices() { - - SliceFileByDimension slicer = new SliceFileByDimension(); - slicer.setSliceByDimension("NUMROWS"); - - Map<String, Integer> dimensionNameToLength = new LinkedHashMap<>(); - dimensionNameToLength.put("NUMROWS", 3163); - dimensionNameToLength.put("NUMCELLS", 82); - - List<String> result = slicer.generateTileBoundrySlices("NUMROWS", dimensionNameToLength); - - assertEquals(3163, result.size()); - - assertThat(result, hasItems("NUMROWS:0:1,NUMCELLS:0:82", "NUMROWS:1:2,NUMCELLS:0:82", "NUMROWS:3162:3163,NUMCELLS:0:82")); - - } - - @Test - public void testGenerateTileBoundrySlices2() { - - SliceFileByDimension slicer = new SliceFileByDimension(); - slicer.setSliceByDimension("NUMROWS"); - - Map<String, Integer> dimensionNameToLength = new LinkedHashMap<>(); - dimensionNameToLength.put("NUMROWS", 2); - dimensionNameToLength.put("NUMCELLS", 82); - - List<String> result = slicer.generateTileBoundrySlices("NUMROWS", dimensionNameToLength); - - assertEquals(2, result.size()); - - assertThat(result, hasItems("NUMROWS:0:1,NUMCELLS:0:82", "NUMROWS:1:2,NUMCELLS:0:82")); - - } - - @Test - public void testGenerateSlicesByInteger() throws IOException { - - - Integer expectedTiles = 1624; - - SliceFileByDimension slicer = new SliceFileByDimension(); - slicer.setDimensions(Arrays.asList("0", "1")); - slicer.setSliceByDimension("1"); - - Resource testResource = new ClassPathResource("granules/SMAP_L2B_SSS_04892_20160101T005507_R13080.h5"); - - List<String> results = slicer.generateSlices(testResource.getFile()); - - assertThat(results.size(), is(expectedTiles)); - - } - - @Test - public void testGenerateSlicesByName() throws IOException { - - - Integer expectedTiles = 3163; - - SliceFileByDimension slicer = new SliceFileByDimension(); - slicer.setDimensions(Arrays.asList("NUMROWS", "NUMCELLS")); - slicer.setSliceByDimension("NUMROWS"); - - Resource testResource = new ClassPathResource("granules/ascat_20121029_010301_metopb_00588_eps_o_coa_2101_ovw.l2.nc"); - - List<String> results = slicer.generateSlices(testResource.getFile()); - - assertThat(results.size(), is(expectedTiles)); - - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesiredTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesiredTest.java b/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesiredTest.java deleted file mode 100644 index ff0077c..0000000 --- a/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesiredTest.java +++ /dev/null @@ -1,179 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2017 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.datatiler; - -import org.junit.Test; -import org.springframework.core.io.ClassPathResource; -import org.springframework.core.io.Resource; - -import java.io.IOException; -import java.util.Arrays; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -import static org.hamcrest.CoreMatchers.hasItems; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.junit.Assert.assertEquals; - -public class SliceFileByTilesDesiredTest { - - @Test - public void testGenerateChunkBoundrySlicesWithDivisibileTiles() { - - SliceFileByTilesDesired slicer = new SliceFileByTilesDesired(); - - Integer tilesDesired = 4; - - Map<String, Integer> dimensionNameToLength = new LinkedHashMap<>(); - dimensionNameToLength.put("lat", 8); - dimensionNameToLength.put("lon", 8); - - List<String> result = slicer.generateChunkBoundrySlices(tilesDesired, dimensionNameToLength); - - assertEquals(tilesDesired.intValue(), result.size()); - - String[] expected = new String[]{ - "lat:0:4,lon:0:4", - "lat:0:4,lon:4:8", - "lat:4:8,lon:0:4", - "lat:4:8,lon:4:8"}; - assertThat(result, containsInAnyOrder(expected)); - assertThat(result, contains(expected)); - - } - - @Test - public void testGenerateChunkBoundrySlicesWithNonDivisibileTiles() { - - SliceFileByTilesDesired slicer = new SliceFileByTilesDesired(); - - Integer tilesDesired = 5; - - Map<String, Integer> dimensionNameToLength = new LinkedHashMap<>(); - dimensionNameToLength.put("lat", 8); - dimensionNameToLength.put("lon", 8); - - List<String> result = slicer.generateChunkBoundrySlices(tilesDesired, dimensionNameToLength); - - assertEquals(9, result.size()); - - String[] expected = new String[]{ - "lat:0:3,lon:0:3", - "lat:0:3,lon:3:6", - "lat:0:3,lon:6:8", - "lat:3:6,lon:0:3", - "lat:3:6,lon:3:6", - "lat:3:6,lon:6:8", - "lat:6:8,lon:0:3", - "lat:6:8,lon:3:6", - "lat:6:8,lon:6:8"}; - assertThat(result, containsInAnyOrder(expected)); - assertThat(result, contains(expected)); - - } - - @Test - public void testAddTimeDimension() { - - SliceFileByTilesDesired slicer = new SliceFileByTilesDesired(); - slicer.setTimeDimension("time"); - - Integer tilesDesired = 4; - Integer timeLen = 3; - - Map<String, Integer> dimensionNameToLength = new LinkedHashMap<>(); - dimensionNameToLength.put("lat", 8); - dimensionNameToLength.put("lon", 8); - - List<String> result = slicer.generateChunkBoundrySlices(tilesDesired, dimensionNameToLength); - result = slicer.addTimeDimension(result, timeLen); - - assertEquals(tilesDesired * timeLen, result.size()); - - String[] expected = new String[]{ - "time:0:1,lat:0:4,lon:0:4", - "time:1:2,lat:0:4,lon:0:4", - "time:2:3,lat:0:4,lon:0:4", - - "time:0:1,lat:0:4,lon:4:8", - "time:1:2,lat:0:4,lon:4:8", - "time:2:3,lat:0:4,lon:4:8", - - "time:0:1,lat:4:8,lon:0:4", - "time:1:2,lat:4:8,lon:0:4", - "time:2:3,lat:4:8,lon:0:4", - - "time:0:1,lat:4:8,lon:4:8", - "time:1:2,lat:4:8,lon:4:8", - "time:2:3,lat:4:8,lon:4:8"}; - assertThat(result, containsInAnyOrder(expected)); - assertThat(result, contains(expected)); - - } - - @Test - public void testGenerateChunkBoundrySlicesWithMurDimensions() { - - SliceFileByTilesDesired slicer = new SliceFileByTilesDesired(); - - Integer tilesDesired = 5184; - - Map<String, Integer> dimensionNameToLength = new LinkedHashMap<>(); - dimensionNameToLength.put("lat", 17999); - dimensionNameToLength.put("lon", 36000); - - List<String> result = slicer.generateChunkBoundrySlices(tilesDesired, dimensionNameToLength); - - assertEquals(tilesDesired + 72, result.size()); - - assertThat(result, hasItems("lat:0:249,lon:0:500", "lat:0:249,lon:500:1000", "lat:17928:17999,lon:35500:36000")); - - } - - @Test - public void testAddTimeDimensionWithMurDimensionsAndTime() { - - SliceFileByTilesDesired slicer = new SliceFileByTilesDesired(); - - Integer tilesDesired = 5184; - - Map<String, Integer> dimensionNameToLength = new LinkedHashMap<>(); - dimensionNameToLength.put("lat", 17999); - dimensionNameToLength.put("lon", 36000); - - slicer.setTimeDimension("time"); - - List<String> result = slicer.generateChunkBoundrySlices(tilesDesired, dimensionNameToLength); - result = slicer.addTimeDimension(result, 1); - - assertEquals(tilesDesired + 72, result.size()); - - assertThat(result, hasItems("time:0:1,lat:0:249,lon:0:500", "time:0:1,lat:0:249,lon:500:1000", "time:0:1,lat:17928:17999,lon:35500:36000")); - - } - - @Test - public void testGenerateSlicesCcmp() throws IOException { - Integer tilesDesired = 270; - Integer expectedTiles = 289 * 4; // 4 time slices and 289 tiles per time slice - - SliceFileByTilesDesired slicer = new SliceFileByTilesDesired(); - slicer.setTilesDesired(tilesDesired); - slicer.setDimensions(Arrays.asList("latitude", "longitude")); - slicer.setTimeDimension("time"); - - Resource testResource = new ClassPathResource("granules/CCMP_Wind_Analysis_20050101_V02.0_L3.0_RSS.nc"); - - List<String> results = slicer.generateSlices(testResource.getFile()); - - assertThat(results.size(), is(expectedTiles)); - - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/test/java/gov/nasa/jpl/nexus/ningester/processors/AddDatasetNameTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/gov/nasa/jpl/nexus/ningester/processors/AddDatasetNameTest.java b/src/test/java/gov/nasa/jpl/nexus/ningester/processors/AddDatasetNameTest.java deleted file mode 100644 index 71b5c46..0000000 --- a/src/test/java/gov/nasa/jpl/nexus/ningester/processors/AddDatasetNameTest.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - ****************************************************************************** - * Copyright (c) 2018 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.processors; - -import org.apache.sdap.nexusproto.NexusTile; -import org.junit.Test; - -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.MatcherAssert.assertThat; - -public class AddDatasetNameTest { - - @Test - public void testDatasetName() { - - String datasetName = "testDataset"; - - NexusTile input = NexusTile.newBuilder().build(); - - AddDatasetName processor = new AddDatasetName(datasetName); - - NexusTile result = processor.addDatasetName(input); - - assertThat(result.getSummary().getDatasetName(), is(datasetName)); - - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/test/java/gov/nasa/jpl/nexus/ningester/processors/AddDayOfYearAttributeTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/gov/nasa/jpl/nexus/ningester/processors/AddDayOfYearAttributeTest.java b/src/test/java/gov/nasa/jpl/nexus/ningester/processors/AddDayOfYearAttributeTest.java deleted file mode 100644 index 4722eaf..0000000 --- a/src/test/java/gov/nasa/jpl/nexus/ningester/processors/AddDayOfYearAttributeTest.java +++ /dev/null @@ -1,84 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2018 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.processors; - -import org.apache.sdap.nexusproto.NexusTile; -import org.apache.sdap.nexusproto.TileSummary; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.beans.HasPropertyWithValue.hasProperty; - -public class AddDayOfYearAttributeTest { - - @Rule - public ExpectedException thrown = ExpectedException.none(); - - @Test - public void testSuccessfulMatch() { - String regex = "^(\\d{3})"; - String granuleName = "001.L4_5day_avhrr_clim_sst_pixelMean.nc"; - NexusTile nexusTile = NexusTile.newBuilder().setSummary( - TileSummary.newBuilder() - .setGranule(granuleName) - .build() - ).build(); - - AddDayOfYearAttribute processor = new AddDayOfYearAttribute(regex); - - NexusTile result = processor.setDayOfYearFromGranuleName(nexusTile); - - assertThat(result.getSummary().getGlobalAttributesList(), contains( - hasProperty("name", is("day_of_year_i")) - )); - - String actualDayOfYear = result.getSummary().getGlobalAttributes(0).getValues(0); - assertThat(actualDayOfYear, is("001")); - } - - @Test() - public void testUnsuccessfulMatch() { - - thrown.expect(RuntimeException.class); - thrown.expectMessage("regex did not match granuleName."); - - String regex = "^(\\d{3})"; - String granuleName = "L4_5day_avhrr_clim_sst_pixelMean.nc"; - NexusTile nexusTile = NexusTile.newBuilder().setSummary( - TileSummary.newBuilder() - .setGranule(granuleName) - .build() - ).build(); - - AddDayOfYearAttribute processor = new AddDayOfYearAttribute(regex); - - processor.setDayOfYearFromGranuleName(nexusTile); - } - - @Test() - public void testUnsuccessfulGroupMatch() { - - thrown.expect(RuntimeException.class); - thrown.expectMessage("regex does not have exactly one capturing group."); - - String regex = "^\\d{3}"; - String granuleName = "001.L4_5day_avhrr_clim_sst_pixelMean.nc"; - NexusTile nexusTile = NexusTile.newBuilder().setSummary( - TileSummary.newBuilder() - .setGranule(granuleName) - .build() - ).build(); - - AddDayOfYearAttribute processor = new AddDayOfYearAttribute(regex); - - processor.setDayOfYearFromGranuleName(nexusTile); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/test/java/gov/nasa/jpl/nexus/ningester/processors/AddTimeFromGranuleNameTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/gov/nasa/jpl/nexus/ningester/processors/AddTimeFromGranuleNameTest.java b/src/test/java/gov/nasa/jpl/nexus/ningester/processors/AddTimeFromGranuleNameTest.java deleted file mode 100644 index b59be7d..0000000 --- a/src/test/java/gov/nasa/jpl/nexus/ningester/processors/AddTimeFromGranuleNameTest.java +++ /dev/null @@ -1,85 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2018 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ -package gov.nasa.jpl.nexus.ningester.processors; - -import org.apache.sdap.nexusproto.GridTile; -import org.apache.sdap.nexusproto.NexusTile; -import org.apache.sdap.nexusproto.TileData; -import org.apache.sdap.nexusproto.TileSummary; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -import java.text.ParseException; - -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.isA; -import static org.hamcrest.MatcherAssert.assertThat; - - -public class AddTimeFromGranuleNameTest { - - - @Rule - public ExpectedException thrown = ExpectedException.none(); - - @Test - public void testSuccessfulMatch() { - String regex = "^.*(\\d{7})\\."; - String dateFormat = "yyyyDDD"; - - String granuleName = "A2012001.L3m_DAY_NSST_sst_4km.nc"; - Long expectedTime = 1325376000L; // 01/01/2012 00:00:00 in epoch time - NexusTile nexusTile = NexusTile.newBuilder().setSummary( - TileSummary.newBuilder() - .setGranule(granuleName) - .build() - ).setTile( - TileData.newBuilder() - .setGridTile( - GridTile.newBuilder( - - ).build() - ).build() - ).build(); - - AddTimeFromGranuleName processor = new AddTimeFromGranuleName(regex, dateFormat); - - NexusTile result = processor.setTimeFromGranuleName(nexusTile); - - assertThat(result.getTile().getGridTile().getTime(), is(expectedTime)); - assertThat(result.getSummary().getStats().getMinTime(), is(expectedTime)); - assertThat(result.getSummary().getStats().getMaxTime(), is(expectedTime)); - } - - @Test - public void testUnparseable() { - String regex = "^.*(\\d{7})\\."; - String dateFormat = "yyyyDDDss"; - - String granuleName = "A2012001.L3m_DAY_NSST_sst_4km.nc"; - - thrown.expect(RuntimeException.class); - thrown.expectCause(isA(ParseException.class)); - - NexusTile nexusTile = NexusTile.newBuilder().setSummary( - TileSummary.newBuilder() - .setGranule(granuleName) - .build() - ).setTile( - TileData.newBuilder() - .setGridTile( - GridTile.newBuilder( - - ).build() - ).build() - ).build(); - - AddTimeFromGranuleName processor = new AddTimeFromGranuleName(regex, dateFormat); - - NexusTile result = processor.setTimeFromGranuleName(nexusTile); - - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/test/java/gov/nasa/jpl/nexus/ningester/processors/GenerateTileIdTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/gov/nasa/jpl/nexus/ningester/processors/GenerateTileIdTest.java b/src/test/java/gov/nasa/jpl/nexus/ningester/processors/GenerateTileIdTest.java deleted file mode 100644 index 5cfdad2..0000000 --- a/src/test/java/gov/nasa/jpl/nexus/ningester/processors/GenerateTileIdTest.java +++ /dev/null @@ -1,65 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2018 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.processors; - -import org.apache.sdap.nexusproto.NexusTile; -import org.apache.sdap.nexusproto.TileData; -import org.junit.Test; - -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.MatcherAssert.assertThat; - -public class GenerateTileIdTest { - - - @Test - public void testGenerateId() { - - String granuleFileName = "file:/path/to/some/file/19960421120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc"; - String sectionSpec = "time:0:1,lat:140:160,lon:640:680"; - - String expectedId = "c031a9c4-9e1d-32e9-9d5c-d2497ce74920"; - - NexusTile.Builder inputTileBuilder = NexusTile.newBuilder(); - inputTileBuilder.getSummaryBuilder().setGranule(granuleFileName); - inputTileBuilder.getSummaryBuilder().setSectionSpec(sectionSpec); - inputTileBuilder.setTile(TileData.newBuilder()); - - GenerateTileId processor = new GenerateTileId(); - - NexusTile result = processor.addTileId(inputTileBuilder.build()); - - assertThat(result.getSummary().getTileId(), is(expectedId)); - assertThat(result.getTile().getTileId(), is(expectedId)); - - } - - @Test - public void testGenerateIdWithSalt() { - - String granuleFileName = "file:/path/to/some/file/CCMP_Wind_Analysis_19990928_V02.0_L3.0_RSS.nc"; - String sectionSpec = "time:3:4,longitude:174:261,latitude:152:190"; - String salt = "wind_u"; - - String expectedId = "48da50ef-e92c-3562-89f9-470561a06482"; - - NexusTile.Builder inputTileBuilder = NexusTile.newBuilder(); - inputTileBuilder.getSummaryBuilder().setGranule(granuleFileName); - inputTileBuilder.getSummaryBuilder().setSectionSpec(sectionSpec); - inputTileBuilder.setTile(TileData.newBuilder()); - - GenerateTileId processor = new GenerateTileId(); - processor.setSalt(salt); - - NexusTile result = processor.addTileId(inputTileBuilder.build()); - - assertThat(result.getSummary().getTileId(), is(expectedId)); - assertThat(result.getTile().getTileId(), is(expectedId)); - - } - -} - http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/test/java/gov/nasa/jpl/nexus/ningester/writer/SolrStoreTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/gov/nasa/jpl/nexus/ningester/writer/SolrStoreTest.java b/src/test/java/gov/nasa/jpl/nexus/ningester/writer/SolrStoreTest.java deleted file mode 100644 index 9fd5173..0000000 --- a/src/test/java/gov/nasa/jpl/nexus/ningester/writer/SolrStoreTest.java +++ /dev/null @@ -1,137 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2018 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.writer; - -import org.apache.sdap.nexusproto.TileSummary; -import org.apache.solr.common.SolrInputDocument; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -public class SolrStoreTest { - - @Test - public void testGetSolrDocFromTileSummary() { - SolrStore solrStore = new SolrStore(null); - - TileSummary tileSummary = TileSummary.newBuilder() - .setTileId("1") - .setBbox(TileSummary.BBox.newBuilder() - .setLatMin(51) - .setLatMax(55) - .setLonMin(22) - .setLonMax(30) - .build()) - .setDatasetName("test") - .setDatasetUuid("4") - .setDataVarName("sst") - .setGranule("test.nc") - .setSectionSpec("0:1,0:1") - .setStats(TileSummary.DataStats.newBuilder() - .setCount(10) - .setMax(50) - .setMin(50) - .setMean(50) - .setMaxTime(1429142399) - .setMinTime(1429142399) - .build()) - .build(); - - SolrInputDocument doc = solrStore.getSolrDocFromTileSummary(tileSummary); - - assertEquals("2015-04-15T23:59:59Z", doc.get("tile_min_time_dt").getValue()); - assertEquals("2015-04-15T23:59:59Z", doc.get("tile_max_time_dt").getValue()); - assertEquals("sea_surface_temp", doc.get("table_s").getValue()); - assertEquals("POLYGON((22.000 51.000, 30.000 51.000, 30.000 55.000, 22.000 55.000, 22.000 51.000))", doc.get("geo").getValue()); - assertEquals("1", doc.get("id").getValue()); - assertEquals("4", doc.get("dataset_id_s").getValue()); - assertEquals("0:1,0:1", doc.get("sectionSpec_s").getValue()); - assertEquals("test", doc.get("dataset_s").getValue()); - assertEquals("test.nc", doc.get("granule_s").getValue()); - assertEquals("sst", doc.get("tile_var_name_s").getValue()); - assertEquals(22.0f, (Float) doc.get("tile_min_lon").getValue(), 0.01f); - assertEquals(30.0f, (Float) doc.get("tile_max_lon").getValue(), 0.01f); - assertEquals(51.0f, (Float) doc.get("tile_min_lat").getValue(), 0.01f); - assertEquals(55.0f, (Float) doc.get("tile_max_lat").getValue(), 0.01f); - assertEquals(50.0f, (Float) doc.get("tile_min_val_d").getValue(), 0.01f); - assertEquals(50.0f, (Float) doc.get("tile_max_val_d").getValue(), 0.01f); - assertEquals(50.0f, (Float) doc.get("tile_avg_val_d").getValue(), 0.01f); - assertEquals(10, doc.get("tile_count_i").getValue()); - assertEquals("test!1", doc.get("solr_id_s").getValue()); - } - - @Test - public void testGeoIsPointWhenLatMinMaxEqualAndLonMinMaxEqual() { - SolrStore solrStore = new SolrStore(null); - - TileSummary tileSummary = TileSummary.newBuilder() - .setBbox(TileSummary.BBox.newBuilder() - .setLatMin(51) - .setLatMax(51) - .setLonMin(22) - .setLonMax(22) - .build()) - .build(); - - SolrInputDocument doc = solrStore.getSolrDocFromTileSummary(tileSummary); - - assertEquals("POINT(22.000 51.000)", doc.get("geo").getValue()); - } - - @Test - public void testGeoIsLineStringWhenLatMinMaxEqualAndLonMinMaxNotEqual() { - SolrStore solrStore = new SolrStore(null); - - TileSummary tileSummary = TileSummary.newBuilder() - .setBbox(TileSummary.BBox.newBuilder() - .setLatMin(51) - .setLatMax(51) - .setLonMin(22) - .setLonMax(29) - .build()) - .build(); - - SolrInputDocument doc = solrStore.getSolrDocFromTileSummary(tileSummary); - - assertEquals("LINESTRING (22.000 51.000, 29.000 51.000)", doc.get("geo").getValue()); - } - - @Test - public void testGeoIsLineStringWhenLatMinMaxNotEqualAndLonMinMaxEqual() { - SolrStore solrStore = new SolrStore(null); - - TileSummary tileSummary = TileSummary.newBuilder() - .setBbox(TileSummary.BBox.newBuilder() - .setLatMin(51) - .setLatMax(59) - .setLonMin(22) - .setLonMax(22) - .build()) - .build(); - - SolrInputDocument doc = solrStore.getSolrDocFromTileSummary(tileSummary); - - assertEquals("LINESTRING (22.000 51.000, 22.000 59.000)", doc.get("geo").getValue()); - } - - @Test - public void testGeoIsLineStringWhenLatMinMaxAlmostEqualAndLonMinMaxNotEqual() { - SolrStore solrStore = new SolrStore(null); - - TileSummary tileSummary = TileSummary.newBuilder() - .setBbox(TileSummary.BBox.newBuilder() - .setLatMin(-56.135883f) - .setLatMax(-56.135674f) - .setLonMin(-9.229431f) - .setLonMax(-8.934967f) - .build()) - .build(); - - SolrInputDocument doc = solrStore.getSolrDocFromTileSummary(tileSummary); - - assertEquals("LINESTRING (-9.229 -56.136, -8.935 -56.136)", doc.get("geo").getValue()); - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/test/java/org/apache/sdap/ningester/NingesterApplicationTests.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sdap/ningester/NingesterApplicationTests.java b/src/test/java/org/apache/sdap/ningester/NingesterApplicationTests.java new file mode 100644 index 0000000..289b0b5 --- /dev/null +++ b/src/test/java/org/apache/sdap/ningester/NingesterApplicationTests.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sdap.ningester; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; + +@RunWith(SpringRunner.class) +@SpringBootTest +public class NingesterApplicationTests { + + @Test + public void contextLoads() { + } + +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/test/java/org/apache/sdap/ningester/datatiler/NetCDFItemReaderTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sdap/ningester/datatiler/NetCDFItemReaderTest.java b/src/test/java/org/apache/sdap/ningester/datatiler/NetCDFItemReaderTest.java new file mode 100644 index 0000000..ee710c3 --- /dev/null +++ b/src/test/java/org/apache/sdap/ningester/datatiler/NetCDFItemReaderTest.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sdap.ningester.datatiler; + +import org.apache.sdap.nexusproto.NexusTile; +import org.junit.Test; +import org.springframework.batch.item.ExecutionContext; +import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.Resource; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertTrue; + +public class NetCDFItemReaderTest { + + @Test + public void testOpen() { + SliceFileByTilesDesired slicer = new SliceFileByTilesDesired(); + slicer.setTilesDesired(5184); + slicer.setDimensions(Arrays.asList("lat", "lon")); + + NetCDFItemReader reader = new NetCDFItemReader(slicer); + reader.setResource(new ClassPathResource("granules/20050101120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc")); + + ExecutionContext context = new ExecutionContext(); + reader.open(context); + + assertTrue(context.containsKey(NetCDFItemReader.CURRENT_TILE_SPEC_INDEX_KEY)); + } + + @Test + public void testReadOne() throws Exception { + SliceFileByTilesDesired slicer = new SliceFileByTilesDesired(); + slicer.setTilesDesired(5184); + slicer.setDimensions(Arrays.asList("lat", "lon")); + + Resource testResource = new ClassPathResource("granules/20050101120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc"); + NetCDFItemReader reader = new NetCDFItemReader(slicer); + reader.setResource(testResource); + + ExecutionContext context = new ExecutionContext(); + reader.open(context); + + NexusTile result = reader.read(); + + assertThat(result.getSummary().getSectionSpec(), is("lat:0:10,lon:0:20")); + assertThat(result.getSummary().getGranule(), is(testResource.getURL().toString())); + + } + + @Test + public void testReadAll() { + Integer tilesDesired = 5184; + + SliceFileByTilesDesired slicer = new SliceFileByTilesDesired(); + slicer.setTilesDesired(tilesDesired); + slicer.setDimensions(Arrays.asList("lat", "lon")); + slicer.setTimeDimension("time"); + + Resource testResource = new ClassPathResource("granules/20050101120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc"); + NetCDFItemReader reader = new NetCDFItemReader(slicer); + reader.setResource(testResource); + + ExecutionContext context = new ExecutionContext(); + reader.open(context); + + List<NexusTile> results = new ArrayList<>(); + NexusTile result; + while ((result = reader.read()) != null) { + results.add(result); + } + + assertThat(results.size(), is(tilesDesired)); + + } + +}
