This is an automated email from the ASF dual-hosted git repository. wiener pushed a commit to branch edge-extensions in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
commit 1f5959ffca8aa3c174d47170e9994e8a3c2b0097 Author: Patrick Wiener <[email protected]> AuthorDate: Tue Sep 29 11:08:48 2020 +0100 update branch with dev changes --- pom.xml | 1 - streampipes-app-file-export/pom.xml | 79 ----- .../app/file/export/ElasticsearchAppData.java | 83 ------ .../app/file/export/ElasticsearchConfig.java | 60 ---- .../app/file/export/api/IElasticsearch.java | 36 --- .../application/AppFileExportApplication.java | 30 -- .../app/file/export/converter/JsonConverter.java | 65 ----- .../app/file/export/impl/Elasticsearch.java | 320 --------------------- .../app/file/export/model/IndexInfo.java | 45 --- .../org/apache/streampipes/model/node/Node.java | 3 + .../streampipes/model/node/NodeBrokerInfo.java | 3 + .../apache/streampipes/model/node/NodeInfo.java | 3 + .../streampipes/model/node/NodeMetadata.java | 3 + .../streampipes/model/node/NodeResources.java | 2 + .../model/node/resources/hardware/CPU.java | 3 + .../model/node/resources/hardware/DISK.java | 3 + .../model/node/resources/hardware/GPU.java | 3 + .../node/resources/hardware/HardwareResource.java | 2 + .../model/node/resources/hardware/MEM.java | 3 + .../AccessibleSensorActuatorResource.java | 2 + .../model/node/resources/software/Cuda.java | 3 + .../model/node/resources/software/Docker.java | 3 + .../node/resources/software/SoftwareResource.java | 3 + .../development/env | 2 +- .../app/configuration/configuration.component.html | 4 +- .../configuration/shared/configuration.service.ts | 9 +- .../app/core-model/gen/streampipes-model-client.ts | 18 ++ ui/src/app/core-model/gen/streampipes-model.ts | 18 ++ .../save-pipeline/save-pipeline.component.html | 69 +++-- .../save-pipeline/save-pipeline.component.ts | 29 +- .../save-pipeline/save-pipeline.controller.ts | 218 -------------- .../save-pipeline/submitPipelineModal.tmpl.html | 115 -------- 32 files changed, 145 insertions(+), 1095 deletions(-) diff --git a/pom.xml b/pom.xml index 27a20dd..a52be90 100644 --- a/pom.xml +++ b/pom.xml @@ -843,7 +843,6 @@ <module>archetypes/streampipes-archetype-pe-processors-jvm</module> <module>archetypes/streampipes-archetype-pe-sinks-flink</module> <module>archetypes/streampipes-archetype-pe-processors-flink</module> - <module>streampipes-app-file-export</module> <module>streampipes-backend</module> <module>streampipes-code-generation</module> <module>streampipes-commons</module> diff --git a/streampipes-app-file-export/pom.xml b/streampipes-app-file-export/pom.xml deleted file mode 100644 index 211640b..0000000 --- a/streampipes-app-file-export/pom.xml +++ /dev/null @@ -1,79 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - ~ Licensed to the Apache Software Foundation (ASF) under one or more - ~ contributor license agreements. See the NOTICE file distributed with - ~ this work for additional information regarding copyright ownership. - ~ The ASF licenses this file to You under the Apache License, Version 2.0 - ~ (the "License"); you may not use this file except in compliance with - ~ the License. You may obtain a copy of the License at - ~ - ~ http://www.apache.org/licenses/LICENSE-2.0 - ~ - ~ Unless required by applicable law or agreed to in writing, software - ~ distributed under the License is distributed on an "AS IS" BASIS, - ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - ~ See the License for the specific language governing permissions and - ~ limitations under the License. - ~ - --> - -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <modelVersion>4.0.0</modelVersion> - <parent> - <artifactId>streampipes-parent</artifactId> - <groupId>org.apache.streampipes</groupId> - <version>0.68.0-SNAPSHOT</version> - </parent> - - <name>StreamPipes App File Export</name> - <artifactId>streampipes-app-file-export</artifactId> - <packaging>jar</packaging> - - <dependencies> - <!-- StreamPipes dependencies --> - <dependency> - <groupId>org.apache.streampipes</groupId> - <artifactId>streampipes-storage-couchdb</artifactId> - <version>0.68.0-SNAPSHOT</version> - </dependency> - - <!-- External dependencies --> - <dependency> - <groupId>commons-logging</groupId> - <artifactId>commons-logging</artifactId> - </dependency> - <dependency> - <groupId>javax.ws.rs</groupId> - <artifactId>javax.ws.rs-api</artifactId> - </dependency> - <dependency> - <groupId>org.glassfish.jersey.containers</groupId> - <artifactId>jersey-container-servlet</artifactId> - </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>fluent-hc</artifactId> - </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpcore</artifactId> - </dependency> - <dependency> - <groupId>org.elasticsearch.client</groupId> - <artifactId>elasticsearch-rest-high-level-client</artifactId> - </dependency> - <dependency> - <groupId>org.lightcouch</groupId> - <artifactId>lightcouch</artifactId> - </dependency> - - <!-- dependency convergence --> - <dependency> - <groupId>org.yaml</groupId> - <artifactId>snakeyaml</artifactId> - </dependency> - - <!-- Test dependencies --> - - </dependencies> -</project> \ No newline at end of file diff --git a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/ElasticsearchAppData.java b/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/ElasticsearchAppData.java deleted file mode 100644 index 794f50b..0000000 --- a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/ElasticsearchAppData.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.app.file.export; - -public class ElasticsearchAppData { - - private String index; - - private long timestampFrom; - - private long timestampTo; - - private String output; - - private boolean allData; - - public ElasticsearchAppData() { - } - - public ElasticsearchAppData(String index, long timestampFrom, long timeStampTo, String output, boolean allData) { - this.index = index; - this.timestampFrom = timestampFrom; - this.timestampTo = timeStampTo; - this.output = output; - this.allData = allData; - } - - public String getIndex() { - return index; - } - - public void setIndex(String index) { - this.index = index; - } - - public long getTimestampFrom() { - return timestampFrom; - } - - public void setTimestampFrom(long timestampFrom) { - this.timestampFrom = timestampFrom; - } - - public long getTimestampTo() { - return timestampTo; - } - - public void setTimeStampTo(long timestampTo) { - this.timestampTo = timestampTo; - } - - public String getOutput() { - return output; - } - - public void setOutput(String output) { - this.output = output; - } - - public boolean isAllData() { - return allData; - } - - public void setAllData(boolean allData) { - this.allData = allData; - } -} diff --git a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/ElasticsearchConfig.java b/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/ElasticsearchConfig.java deleted file mode 100644 index 0460e18..0000000 --- a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/ElasticsearchConfig.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.app.file.export; - -import org.apache.streampipes.config.SpConfig; - -public enum ElasticsearchConfig { - INSTANCE; - - private SpConfig config; - private final static String HOST = "host"; - private final static String PORT = "port"; - private final static String PROTOCOL = "protocol"; - private final static String DATA_LOCATION = "data_location"; - - ElasticsearchConfig() { - config = SpConfig.getSpConfig("storage/elasticsearch"); - - config.register(HOST, "elasticsearch", "Hostname for the elasticsearch service"); - config.register(PORT, 9200, "Port for the elasticsearch service"); - config.register(PROTOCOL, "http", "Protocol the elasticsearch service"); - config.register(DATA_LOCATION,"/home/user/", "Folder that stores all the created data blobs"); - } - - public String getElasticsearchHost() { - return config.getString(HOST); - } - - public Integer getElasticsearchPort() { - return config.getInteger(PORT); - } - - public String getElasticsearchURL() { - return getElasticsearchProtocol()+ "://" + getElasticsearchHost() + ":" + getElasticsearchPort(); - } - - public String getElasticsearchProtocol() { - return config.getString(PROTOCOL); - } - - public String getDataLocation() { - return config.getString(DATA_LOCATION); - } -} diff --git a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/api/IElasticsearch.java b/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/api/IElasticsearch.java deleted file mode 100644 index c61f670..0000000 --- a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/api/IElasticsearch.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.app.file.export.api; - -import org.apache.streampipes.app.file.export.ElasticsearchAppData; - -import javax.ws.rs.core.Response; - -public interface IElasticsearch { - - Response createFiles(ElasticsearchAppData data); - - Response getFile(String fileName); - - Response deleteFile(String fileName); - - Response getEndpoints(); - - -} diff --git a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/application/AppFileExportApplication.java b/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/application/AppFileExportApplication.java deleted file mode 100644 index 3168892..0000000 --- a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/application/AppFileExportApplication.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.app.file.export.application; - -import org.glassfish.jersey.server.ResourceConfig; -import org.apache.streampipes.app.file.export.impl.Elasticsearch; - - -public class AppFileExportApplication extends ResourceConfig { - - public AppFileExportApplication() { - register(Elasticsearch.class); - } -} diff --git a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/converter/JsonConverter.java b/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/converter/JsonConverter.java deleted file mode 100644 index 5909d42..0000000 --- a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/converter/JsonConverter.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package org.apache.streampipes.app.file.export.converter; - -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; - -import java.util.Map; -import java.util.Set; -import java.util.StringJoiner; - -public class JsonConverter { - - private JsonObject elasticJsonRepresentation; - private JsonParser jsonParser; - - public JsonConverter() { - this.jsonParser = new JsonParser(); - } - - public String getCsvHeader(String elasticJsonRepresentation) { - JsonObject inContent = jsonParser.parse(elasticJsonRepresentation).getAsJsonObject(); - - Set<Map.Entry<String, JsonElement>> elements = inContent.entrySet(); - StringJoiner sj = new StringJoiner(";"); - - for (Map.Entry<String, JsonElement> entry : elements) { - sj.add(entry.getKey().toString()); - } - - return sj.toString() + "\n"; - - } - - public String convertToCsv(String elasticJsonRepresentation) { - JsonObject inContent = jsonParser.parse(elasticJsonRepresentation).getAsJsonObject(); - - Set<Map.Entry<String, JsonElement>> elements = inContent.entrySet(); - StringJoiner sj = new StringJoiner(";"); - - for (Map.Entry<String, JsonElement> entry : elements) { - sj.add(entry.getValue().toString()); - } - - return sj.toString() + "\n"; - - } - -} diff --git a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/impl/Elasticsearch.java b/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/impl/Elasticsearch.java deleted file mode 100644 index c3fb684..0000000 --- a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/impl/Elasticsearch.java +++ /dev/null @@ -1,320 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.app.file.export.impl; - -import com.google.gson.Gson; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import org.apache.http.HttpHost; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.fluent.Request; -import org.elasticsearch.action.search.ClearScrollRequest; -import org.elasticsearch.action.search.ClearScrollResponse; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.SearchScrollRequest; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.RestClientBuilder; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.Scroll; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.lightcouch.CouchDbClient; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.apache.streampipes.app.file.export.ElasticsearchAppData; -import org.apache.streampipes.app.file.export.ElasticsearchConfig; -import org.apache.streampipes.app.file.export.api.IElasticsearch; -import org.apache.streampipes.app.file.export.converter.JsonConverter; -import org.apache.streampipes.app.file.export.model.IndexInfo; -import org.apache.streampipes.storage.couchdb.utils.Utils; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import javax.ws.rs.Consumes; -import javax.ws.rs.DELETE; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; - -@Path("/v1/elasticsearch") -public class Elasticsearch implements IElasticsearch { - - private static String mainFilePath = ElasticsearchConfig.INSTANCE.getDataLocation(); - private static final List<String> excludedIndices = Collections.singletonList(".kibana"); - - Logger LOG = LoggerFactory.getLogger(Elasticsearch.class); - - @POST - @Produces(MediaType.APPLICATION_JSON) - @Consumes(MediaType.APPLICATION_JSON) - @Path("/file") - @Override - public Response createFiles(ElasticsearchAppData data) { - String index = data.getIndex(); - long timestampFrom = data.getTimestampFrom(); - long timeStampTo = data.getTimestampTo(); - String output = data.getOutput(); - boolean allData = data.isAllData(); - - try { - RestHighLevelClient client = getRestHighLevelClient(); - - final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(1L)); - SearchRequest searchRequest = new SearchRequest(index); - searchRequest.scroll(scroll); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - - if (!allData) { - searchSourceBuilder.query(QueryBuilders.rangeQuery("timestamp").from(timestampFrom).to(timeStampTo)); - } - - searchRequest.source(searchSourceBuilder); - SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); - String scrollId = searchResponse.getScrollId(); - SearchHit[] searchHits = searchResponse.getHits().getHits(); - - //Time created in milli sec, index, from, to - long timestamp = System.currentTimeMillis(); - String fileName = System.currentTimeMillis() + "-" + index + "-" + timestampFrom + "-" + timeStampTo + "." + output; - String filePath = mainFilePath + fileName; - FileOutputStream fileStream = this.getFileStream(filePath); - - if(("csv").equals(output)) { - processCSV(client, fileStream, scrollId, scroll, searchHits); - } else { - processJSON(client, fileStream, scrollId, scroll, searchHits); - } - - fileStream.close(); - - CouchDbClient couchDbClient = getCouchDbClient(); - Map<String, Object> map = new HashMap<>(); - map.put("_id", fileName); - map.put("fileName", fileName); - map.put("filePath", filePath); - map.put("createAt", timestamp); - map.put("from", timestampFrom); - map.put("to", timeStampTo); - couchDbClient.save(map); - - LOG.info("Created file: " + fileName); - - return Response.ok().build(); - - } catch (IOException e) { - e.printStackTrace(); - LOG.error(e.getMessage()); - return Response.status(500).entity(e).build(); - } - } - - @GET - @Produces(MediaType.APPLICATION_JSON) - @Path("/file/{fileName}") - public Response getFile(@PathParam("fileName") String fileName) { - File file = new File(mainFilePath + fileName); - if (file.exists()) { - LOG.info("Downloaded file: " + fileName); - return Response.ok(file, MediaType.APPLICATION_OCTET_STREAM) - .header("Content-Disposition", "attachment; filename=\"" + fileName + "\"") - .build(); - } else { - LOG.info("Download - File not found"); - return Response.status(404).entity("File not found").build(); - } - } - - @GET - @Produces(MediaType.APPLICATION_JSON) - @Path("/indices") - public Response getIndices() { - String url = ElasticsearchConfig.INSTANCE.getElasticsearchURL() + "/_cat/indices?v"; - try { - JsonElement jsonResponse = get(url); - - JsonArray response = jsonResponse.getAsJsonArray(); - List<IndexInfo> availableIndices = new ArrayList<>(); - for(int i = 0; i < response.size(); i++) { - JsonObject object = response.get(i).getAsJsonObject(); - String index = object.get("index").getAsString(); - if (!shouldExclude(index)) { - Integer documentCount = Integer.parseInt(object.get("docs.count").getAsString()); - availableIndices.add(new IndexInfo(index, documentCount)); - } - } - return Response.ok(availableIndices).build(); - } catch (IOException e) { - e.printStackTrace(); - return Response.serverError().build(); - } - } - - private boolean shouldExclude(String index) { - return excludedIndices.stream().anyMatch(i -> i.equals(index)); - } - - @DELETE - @Path("/file/{fileName}") - @Override - public Response deleteFile(@PathParam("fileName") String fileName) { - CouchDbClient couchDbClient = getCouchDbClient(); - JsonObject found = couchDbClient.find(JsonObject.class, fileName); - couchDbClient.remove(found.get("_id").getAsString(), found.get("_rev").getAsString()); - File file = new File(mainFilePath + fileName); - file.delete(); - LOG.info("Deleted: " + fileName); - - return Response.ok().build(); - } - - @GET - @Path("/files") - @Override - public Response getEndpoints() { - CouchDbClient couchDbClient = getCouchDbClient(); - List<JsonObject> endpoints = couchDbClient.view("_all_docs").includeDocs(true).query(JsonObject.class); - String json = new Gson().toJson(endpoints); - - return Response.ok(json).build(); - } - - private CouchDbClient getCouchDbClient() { - return Utils.getCouchDbElasticsearchFilesEndppointClient(); - } - - private FileOutputStream getFileStream(String filePath) throws IOException { - File file = new File(filePath); - file.getParentFile().mkdirs(); - return new FileOutputStream(filePath); - } - - private JsonElement get(String url) throws IOException { - String jsonResponse = Request.Get(url) - .addHeader("accept", "application/json") - .addHeader("Content-Type", "application/json") - .execute() - .returnContent().asString(); - return new JsonParser().parse(jsonResponse); - } - - private RestHighLevelClient getRestHighLevelClient() { - String host = ElasticsearchConfig.INSTANCE.getElasticsearchHost(); - int port = ElasticsearchConfig.INSTANCE.getElasticsearchPort(); - - return new RestHighLevelClient( - RestClient.builder( - new HttpHost(host, port, "http")) - .setRequestConfigCallback( - new RestClientBuilder.RequestConfigCallback() { - @Override - public RequestConfig.Builder customizeRequestConfig( - RequestConfig.Builder requestConfigBuilder) { - return requestConfigBuilder - .setConnectTimeout(5000) - .setSocketTimeout(60000); - } - }) - ); - - } - - private void processJSON(RestHighLevelClient client, FileOutputStream fileStream, String scrollId, Scroll scroll, SearchHit[] searchHits) throws IOException { - fileStream.write("[".getBytes()); - boolean isFirstElement = true; - for (SearchHit hit : searchHits) { - if(!isFirstElement) - fileStream.write(",".getBytes()); - fileStream.write(hit.getSourceAsString().getBytes()); - isFirstElement = false; - } - - while (searchHits != null && searchHits.length > 0) { - - SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); - scrollRequest.scroll(scroll); - SearchResponse searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT); - scrollId = searchResponse.getScrollId(); - searchHits = searchResponse.getHits().getHits(); - for (SearchHit hit : searchHits) { - fileStream.write(",".getBytes()); - fileStream.write(hit.getSourceAsString().getBytes()); - } - } - fileStream.write("]".getBytes()); - - ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); - clearScrollRequest.addScrollId(scrollId); - ClearScrollResponse clearScrollResponse = client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); - - - } - - private void processCSV(RestHighLevelClient client, FileOutputStream fileStream, String scrollId, Scroll scroll, - SearchHit[] searchHits) throws IOException { - JsonConverter jsonConverter = new JsonConverter(); - - boolean isFirstElement = true; - for (SearchHit hit : searchHits) { - if (isFirstElement) - fileStream.write(jsonConverter.getCsvHeader(hit.getSourceAsString()).getBytes()); - String response = jsonConverter.convertToCsv(hit.getSourceAsString()); - fileStream.write(response.getBytes()); - isFirstElement = false; - - } - - while (searchHits != null && searchHits.length > 0) { - - SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); - scrollRequest.scroll(scroll); - SearchResponse searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT); - scrollId = searchResponse.getScrollId(); - searchHits = searchResponse.getHits().getHits(); - for (SearchHit hit : searchHits) { - fileStream.write(jsonConverter.convertToCsv(hit.getSourceAsString()).getBytes()); - } - - } - - ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); - clearScrollRequest.addScrollId(scrollId); - ClearScrollResponse clearScrollResponse = client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); - - } - - - - } diff --git a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/model/IndexInfo.java b/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/model/IndexInfo.java deleted file mode 100644 index b23e82b..0000000 --- a/streampipes-app-file-export/src/main/java/org/apache/streampipes/app/file/export/model/IndexInfo.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package org.apache.streampipes.app.file.export.model; - -public class IndexInfo { - - private String indexName; - private Integer documentCount; - - public IndexInfo(String indexName, Integer documentCount) { - this.indexName = indexName; - this.documentCount = documentCount; - } - - public String getIndexName() { - return indexName; - } - - public void setIndexName(String indexName) { - this.indexName = indexName; - } - - public Integer getDocumentCount() { - return documentCount; - } - - public void setDocumentCount(Integer documentCount) { - this.documentCount = documentCount; - } -} diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/node/Node.java b/streampipes-model/src/main/java/org/apache/streampipes/model/node/Node.java index 184f444..ba2a175 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/node/Node.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/node/Node.java @@ -16,6 +16,9 @@ package org.apache.streampipes.model.node;/* * */ +import org.apache.streampipes.model.shared.annotation.TsModel; + +@TsModel public class Node { public NodeInfo nodeInfo; diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/node/NodeBrokerInfo.java b/streampipes-model/src/main/java/org/apache/streampipes/model/node/NodeBrokerInfo.java index cc8538c..b46da5d 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/node/NodeBrokerInfo.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/node/NodeBrokerInfo.java @@ -17,6 +17,9 @@ */ package org.apache.streampipes.model.node; +import org.apache.streampipes.model.shared.annotation.TsModel; + +@TsModel public class NodeBrokerInfo { private String host; diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/node/NodeInfo.java b/streampipes-model/src/main/java/org/apache/streampipes/model/node/NodeInfo.java index cf4b202..e79f254 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/node/NodeInfo.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/node/NodeInfo.java @@ -17,8 +17,11 @@ */ package org.apache.streampipes.model.node; +import org.apache.streampipes.model.shared.annotation.TsModel; + import java.util.List; +@TsModel public class NodeInfo { private String nodeControllerId; diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/node/NodeMetadata.java b/streampipes-model/src/main/java/org/apache/streampipes/model/node/NodeMetadata.java index b1b17db..9195389 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/node/NodeMetadata.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/node/NodeMetadata.java @@ -17,8 +17,11 @@ */ package org.apache.streampipes.model.node; +import org.apache.streampipes.model.shared.annotation.TsModel; + import java.util.List; +@TsModel public class NodeMetadata { private String nodeAddress; diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/node/NodeResources.java b/streampipes-model/src/main/java/org/apache/streampipes/model/node/NodeResources.java index a9510e3..79db2b0 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/node/NodeResources.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/node/NodeResources.java @@ -19,9 +19,11 @@ package org.apache.streampipes.model.node;/* import org.apache.streampipes.model.node.resources.hardware.HardwareResource; import org.apache.streampipes.model.node.resources.interfaces.AccessibleSensorActuatorResource; import org.apache.streampipes.model.node.resources.software.SoftwareResource; +import org.apache.streampipes.model.shared.annotation.TsModel; import java.util.List; +@TsModel public class NodeResources { public HardwareResource hardwareResource; public SoftwareResource softwareResource; diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/node/resources/hardware/CPU.java b/streampipes-model/src/main/java/org/apache/streampipes/model/node/resources/hardware/CPU.java index 7f9c131..7772cfe 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/node/resources/hardware/CPU.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/node/resources/hardware/CPU.java @@ -16,6 +16,9 @@ package org.apache.streampipes.model.node.resources.hardware;/* * */ +import org.apache.streampipes.model.shared.annotation.TsModel; + +@TsModel public class CPU { public int cores; public String arch; diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/node/resources/hardware/DISK.java b/streampipes-model/src/main/java/org/apache/streampipes/model/node/resources/hardware/DISK.java index 6f53373..841cf9d 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/node/resources/hardware/DISK.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/node/resources/hardware/DISK.java @@ -16,6 +16,9 @@ package org.apache.streampipes.model.node.resources.hardware;/* * */ +import org.apache.streampipes.model.shared.annotation.TsModel; + +@TsModel public class DISK { public long diskTotal; diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/node/resources/hardware/GPU.java b/streampipes-model/src/main/java/org/apache/streampipes/model/node/resources/hardware/GPU.java index 74e3068..0003a0f 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/node/resources/hardware/GPU.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/node/resources/hardware/GPU.java @@ -16,6 +16,9 @@ package org.apache.streampipes.model.node.resources.hardware;/* * */ +import org.apache.streampipes.model.shared.annotation.TsModel; + +@TsModel public class GPU { public boolean hasGPU; public int cudaCores; diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/node/resources/hardware/HardwareResource.java b/streampipes-model/src/main/java/org/apache/streampipes/model/node/resources/hardware/HardwareResource.java index 101a036..cb93970 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/node/resources/hardware/HardwareResource.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/node/resources/hardware/HardwareResource.java @@ -16,7 +16,9 @@ package org.apache.streampipes.model.node.resources.hardware;/* * */ +import org.apache.streampipes.model.shared.annotation.TsModel; +@TsModel public class HardwareResource { public CPU cpu; diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/node/resources/hardware/MEM.java b/streampipes-model/src/main/java/org/apache/streampipes/model/node/resources/hardware/MEM.java index 34e5eae..beab137 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/node/resources/hardware/MEM.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/node/resources/hardware/MEM.java @@ -16,6 +16,9 @@ package org.apache.streampipes.model.node.resources.hardware;/* * */ +import org.apache.streampipes.model.shared.annotation.TsModel; + +@TsModel public class MEM { public Long memTotal; diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/node/resources/interfaces/AccessibleSensorActuatorResource.java b/streampipes-model/src/main/java/org/apache/streampipes/model/node/resources/interfaces/AccessibleSensorActuatorResource.java index ecf38ac..f21987f 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/node/resources/interfaces/AccessibleSensorActuatorResource.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/node/resources/interfaces/AccessibleSensorActuatorResource.java @@ -16,7 +16,9 @@ package org.apache.streampipes.model.node.resources.interfaces;/* * */ +import org.apache.streampipes.model.shared.annotation.TsModel; +@TsModel public class AccessibleSensorActuatorResource { public String name; public String type; diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/node/resources/software/Cuda.java b/streampipes-model/src/main/java/org/apache/streampipes/model/node/resources/software/Cuda.java index 7327da5..fcf68f9 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/node/resources/software/Cuda.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/node/resources/software/Cuda.java @@ -16,6 +16,9 @@ package org.apache.streampipes.model.node.resources.software;/* * */ +import org.apache.streampipes.model.shared.annotation.TsModel; + +@TsModel public class Cuda { private boolean hasCuda; diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/node/resources/software/Docker.java b/streampipes-model/src/main/java/org/apache/streampipes/model/node/resources/software/Docker.java index fa0294a..2a18984 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/node/resources/software/Docker.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/node/resources/software/Docker.java @@ -16,6 +16,9 @@ package org.apache.streampipes.model.node.resources.software;/* * */ +import org.apache.streampipes.model.shared.annotation.TsModel; + +@TsModel public class Docker { public boolean hasDocker; diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/node/resources/software/SoftwareResource.java b/streampipes-model/src/main/java/org/apache/streampipes/model/node/resources/software/SoftwareResource.java index e9ef299..7a0591b 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/node/resources/software/SoftwareResource.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/node/resources/software/SoftwareResource.java @@ -16,6 +16,9 @@ package org.apache.streampipes.model.node.resources.software;/* * */ +import org.apache.streampipes.model.shared.annotation.TsModel; + +@TsModel public class SoftwareResource { public String os; public String kernelVersion; diff --git a/streampipes-node-controller-container/development/env b/streampipes-node-controller-container/development/env index 16ce003..8a9c1a5 100644 --- a/streampipes-node-controller-container/development/env +++ b/streampipes-node-controller-container/development/env @@ -27,7 +27,7 @@ SP_NODE_ACCESSIBLE_SENSOR_ACTUATOR_ZED=zed;sensor;/dev/video1;usb SP_NODE_ACCESSIBLE_SENSOR_ACTUATOR_ROS=ros;universalrobot;192.168.0.80;network SP_NODE_SUPPORTED_PE_APP_ID_FILTER=org.apache.streampipes.processors.filters.jvm.numericalfilter SP_NODE_SUPPORTED_PE_APP_ID_ENRICH=org.apache.streampipes.processors.filters.jvm.enrich -SP_NODE_SUPPORTED_PE_APP_ID_STATS=org.streampipes.pe.processors.internal.universalrobot.statscollector +SP_NODE_SUPPORTED_PE_APP_ID_STATS=org.streampipes.pe.ptrocessors.internal.universalrobot.statscollector SP_NODE_SUPPORTED_PE_APP_ID_KPI=org.streampipes.pe.processors.internal.universalrobot.kpi SP_NODE_SUPPORTED_PE_APP_ID_ROSSINK=org.streampipes.pe.sinks.internal.universalrobot.ros SP_NODE_HAS_GPU=false diff --git a/ui/src/app/configuration/configuration.component.html b/ui/src/app/configuration/configuration.component.html index 3a8ebbe..39116ef 100644 --- a/ui/src/app/configuration/configuration.component.html +++ b/ui/src/app/configuration/configuration.component.html @@ -22,8 +22,8 @@ <mat-tab-group [selectedIndex]="selectedIndex" (selectedIndexChange)="selectedIndexChange($event)"> <mat-tab label="Pipeline Element Configuration"></mat-tab> <mat-tab label="Messaging"></mat-tab> - <mat-tab label="Nodes"></mat-tab> <mat-tab label="DataLake"></mat-tab> + <mat-tab label="Nodes"></mat-tab> </mat-tab-group> </div> </div> @@ -37,7 +37,7 @@ <div class="fixed-height page-container-padding-inner" fxLayout="column" fxFlex="100" *ngIf="selectedIndex == 2"> <sp-datalake-configuration fxFlex="100"></sp-datalake-configuration> </div> - <div class="page-container-padding-inner" fxLayout="column" fxFlex="100" *ngIf="selectedIndex == 2"> + <div class="page-container-padding-inner" fxLayout="column" fxFlex="100" *ngIf="selectedIndex == 3"> <edge-configuration fxFlex="100"></edge-configuration> </div> </div> diff --git a/ui/src/app/configuration/shared/configuration.service.ts b/ui/src/app/configuration/shared/configuration.service.ts index 4eb809a..8578799 100644 --- a/ui/src/app/configuration/shared/configuration.service.ts +++ b/ui/src/app/configuration/shared/configuration.service.ts @@ -49,10 +49,11 @@ export class ConfigurationService { getAvailableEdgeNodes(): Observable<Array<NodeInfo>> { return this.http.get(this.getServerUrl() + '/api/v2/users/' + this.authStatusService.email + "/nodes") - .map(data => { - return data as NodeInfo[]; - }); - + .pipe( + map(response => { + return response as NodeInfo[]; + }) + ) } getConsulServices(): Observable<StreampipesPeContainer[]> { diff --git a/ui/src/app/core-model/gen/streampipes-model-client.ts b/ui/src/app/core-model/gen/streampipes-model-client.ts index 4a19f55..d10df05 100644 --- a/ui/src/app/core-model/gen/streampipes-model-client.ts +++ b/ui/src/app/core-model/gen/streampipes-model-client.ts @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + /* tslint:disable */ /* eslint-disable */ // @ts-nocheck diff --git a/ui/src/app/core-model/gen/streampipes-model.ts b/ui/src/app/core-model/gen/streampipes-model.ts index 7dca972..02c2eee 100644 --- a/ui/src/app/core-model/gen/streampipes-model.ts +++ b/ui/src/app/core-model/gen/streampipes-model.ts @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + /* tslint:disable */ /* eslint-disable */ // @ts-nocheck diff --git a/ui/src/app/editor/dialog/save-pipeline/save-pipeline.component.html b/ui/src/app/editor/dialog/save-pipeline/save-pipeline.component.html index 19102f9..f8c2199 100644 --- a/ui/src/app/editor/dialog/save-pipeline/save-pipeline.component.html +++ b/ui/src/app/editor/dialog/save-pipeline/save-pipeline.component.html @@ -43,47 +43,46 @@ </mat-form-field> </div> </form> - <mat-checkbox (click)="triggerTutorial()" color="primary" [(ngModel)]="startPipelineAfterStorage"> + <mat-slide-toggle (click)="triggerTutorial()" color="primary" [(ngModel)]="startPipelineAfterStorage"> Start pipeline immediately - </mat-checkbox> - <md-checkbox ng-disabled="false" aria-label="Start Pipeline" - [(ng-model)]="advancedSettings"> - Deployment settings - </md-checkbox> - <div *ng-if="advancedSettings"> - <h4>Nodes</h4> - <div ng-repeat="pipelineElement in pipeline.sepas"> - <div flex="100" layout="row"> - <div flex="50" layout="row" layout-align="start center"> - <span>{{pipelineElement.name}}</span> - </div> - <div flex="50" layout="row" layout-align="start center"> - <md-select ng-model="pipelineElement.deploymentTargetNodeId" style="margin: 0;"> - <md-option - ng-value="node.nodeControllerId" ng-repeat="node in deploymentOptions[pipelineElement.appId]"> - <em>{{node.nodeMetadata.nodeAddress}} - </em> - </md-option> - </md-select> + </mat-slide-toggle> + <mat-slide-toggle color="primary" [(ngModel)]="advancedSettings"> + Choose deployment options + </mat-slide-toggle> + <mat-divider style="margin: 2em 0 2em 0;"></mat-divider> + <div *ngIf="advancedSettings"> + <b>Node selection</b> + <div *ngFor="let processors of pipeline.sepas"> + <div fxFlex="100" fxLayout="row"> + <div fxFlex="50" fxLayout="row" fxLayoutAlign="center center"> + <span>{{processors.name}}</span> </div> </div> + <div fxFlex="50" fxLayout="row" fxLayoutAlign="start center"> + <mat-form-field appearance="outline"> + <mat-select [(ngModel)]="processors.deploymentTargetNodeId" style="margin: 0;height: 20px;"> + <mat-option [value]="node.nodeControllerId" *ngFor="let node of deploymentOptions[processors.appId]"> + <em>{{node.nodeMetadata.nodeAddress}}</em> + </mat-option> + </mat-select> + </mat-form-field> + </div> </div> - <div ng-repeat="pipelineElement in pipeline.actions"> - <div flex="100" layout="row"> - <div flex="50" layout="row" layout-align="start center"> - <span>{{pipelineElement.name}}</span> - </div> - <div flex="50" layout="row" layout-align="start center"> - <md-select ng-model="pipelineElement.deploymentTargetNodeId" style="margin: 0;"> - <md-option - ng-value = "node.nodeControllerId" ng-repeat="node in - deploymentOptions[pipelineElement.appId]"> - <em>{{node.nodeMetadata.nodeAddress}} - </em> - </md-option> - </md-select> + <div *ngFor="let sinks of pipeline.actions"> + <div fxFlex="100" fxLayout="row"> + <div fxFlex="50" fxLayout="row" fxLayoutAlign="center center"> + <span>{{sinks.name}}</span> </div> </div> + <div fxFlex="50" fxLayout="row" fxLayoutAlign="start center"> + <mat-form-field appearance="outline"> + <mat-select [(ngModel)]="sinks.deploymentTargetNodeId" style="margin: 0; height: 20px;"> + <mat-option [value]="node.nodeControllerId" *ngFor="let node of deploymentOptions[sinks.appId]"> + <em>{{node.nodeMetadata.nodeAddress}}</em> + </mat-option> + </mat-select> + </mat-form-field> + </div> </div> </div> </div> diff --git a/ui/src/app/editor/dialog/save-pipeline/save-pipeline.component.ts b/ui/src/app/editor/dialog/save-pipeline/save-pipeline.component.ts index ef9701d..cf9b9f3 100644 --- a/ui/src/app/editor/dialog/save-pipeline/save-pipeline.component.ts +++ b/ui/src/app/editor/dialog/save-pipeline/save-pipeline.component.ts @@ -55,7 +55,7 @@ export class SavePipelineComponent implements OnInit { storageError: boolean = false; errorMessage: string = ''; - edgeNodes: NodeInfo[] + edgeNodes: NodeInfo[]; advancedSettings: boolean = false; deploymentOptions: Array<any> = new Array<any>(); @@ -71,6 +71,7 @@ export class SavePipelineComponent implements OnInit { ngOnInit() { this.getPipelineCategories(); + this.loadAndPrepareEdgeNodes(); this.submitPipelineForm.addControl("pipelineName", new FormControl(this.pipeline.name, [Validators.required, Validators.maxLength(40)])) @@ -137,6 +138,28 @@ export class SavePipelineComponent implements OnInit { return nodeInfo; } + modifyPipelineElementsDeployments(pipelineElements) { + pipelineElements.forEach(p => { + let selectedTargetNodeId = p.deploymentTargetNodeId + console.log(selectedTargetNodeId); + if(selectedTargetNodeId != "default") { + let selectedNode = this.edgeNodes + .filter(node => node.nodeControllerId === selectedTargetNodeId) + + p.deploymentTargetNodeHostname = selectedNode + .map(node => node.nodeMetadata.nodeAddress)[0] + + p.deploymentTargetNodePort = selectedNode + .map(node => node.nodeControllerPort)[0] + } + else { + console.log('null'); + p.deploymentTargetNodeHostname = null + p.deploymentTargetNodePort = null + } + }) + } + savePipeline(switchTab) { if (this.pipeline.name == "") { //this.showToast("error", "Please enter a name for your pipeline"); @@ -146,9 +169,13 @@ export class SavePipelineComponent implements OnInit { let storageRequest; if (this.currentModifiedPipelineId && this.updateMode === 'update') { + this.modifyPipelineElementsDeployments(this.pipeline.sepas) + this.modifyPipelineElementsDeployments(this.pipeline.actions) storageRequest = this.pipelineService.updatePipeline(this.pipeline); } else { this.pipeline._id = undefined; + this.modifyPipelineElementsDeployments(this.pipeline.sepas) + this.modifyPipelineElementsDeployments(this.pipeline.actions) storageRequest = this.pipelineService.storePipeline(this.pipeline); } diff --git a/ui/src/app/editor/dialog/save-pipeline/save-pipeline.controller.ts b/ui/src/app/editor/dialog/save-pipeline/save-pipeline.controller.ts deleted file mode 100644 index 2054f78..0000000 --- a/ui/src/app/editor/dialog/save-pipeline/save-pipeline.controller.ts +++ /dev/null @@ -1,218 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -import {RestApi} from "../../../services/rest-api.service"; -import {NodeInfo, NodeMetadata} from "../../../configuration/model/NodeInfo.model"; -import {Browser} from "leaflet"; -import edge = Browser.edge; - -export class SavePipelineController { - - RestApi: RestApi; - $mdToast: any; - $state: any; - $mdDialog: any; - pipelineCategories: any; - pipeline: any; - ObjectProvider: any; - startPipelineAfterStorage: any; - modificationMode: any; - updateMode: any; - submitPipelineForm: any; - TransitionService: any; - ShepherdService: any; - edgeNodes: NodeInfo[] - - advancedSettings: boolean = false; - - deploymentOptions: Array<any> = new Array<any>(); - - constructor($mdDialog, - $state, - RestApi, - $mdToast, - ObjectProvider, - pipeline, - modificationMode, - TransitionService, - ShepherdService) { - this.RestApi = RestApi; - this.$mdToast = $mdToast; - this.$state = $state; - this.$mdDialog = $mdDialog; - this.pipelineCategories = []; - this.pipeline = pipeline; - this.ObjectProvider = ObjectProvider; - this.modificationMode = modificationMode; - this.updateMode = "update"; - this.TransitionService = TransitionService; - this.ShepherdService = ShepherdService; - } - - $onInit() { - this.getPipelineCategories(); - if (this.ShepherdService.isTourActive()) { - this.ShepherdService.trigger("enter-pipeline-name"); - } - this.loadAndPrepareEdgeNodes(); - - } - - loadAndPrepareEdgeNodes() { - this.RestApi.getAvailableEdgeNodes().then(response => { - //let edgeNodes = response.data as NodeInfo[]; - this.edgeNodes = response.data; - this.addAppIds(this.pipeline.sepas, this.edgeNodes); - this.addAppIds(this.pipeline.actions, this.edgeNodes); - }); - } - - addAppIds(pipelineElements, edgeNodes: Array<NodeInfo>) { - pipelineElements.forEach(pipelineElement => { - this.deploymentOptions[pipelineElement.appId] = []; - this.deploymentOptions[pipelineElement.appId].push(this.makeDefaultNodeInfo()); - edgeNodes.forEach(nodeInfo => { - if (nodeInfo.supportedPipelineElementAppIds.some(appId => appId === pipelineElement.appId)) { - this.deploymentOptions[pipelineElement.appId].push(nodeInfo); - } - }) - }); - } - - makeDefaultNodeInfo() { - let nodeInfo = {} as NodeInfo; - nodeInfo.nodeControllerId = "default"; - nodeInfo.nodeMetadata = {} as NodeMetadata; - nodeInfo.nodeMetadata.nodeAddress = "default"; - nodeInfo.nodeMetadata.nodeModel = "Default Node"; - return nodeInfo; - } - - triggerTutorial() { - if (this.ShepherdService.isTourActive()) { - this.ShepherdService.trigger("save-pipeline-dialog"); - } - } - - displayErrors(data) { - for (var i = 0, notification; notification = data.notifications[i]; i++) { - this.showToast("error", notification.title, notification.description); - } - } - - displaySuccess(data) { - if (data.notifications.length > 0) { - this.showToast("success", data.notifications[0].title, data.notifications[0].description); - } - } - - getPipelineCategories() { - this.RestApi.getPipelineCategories() - .then(pipelineCategories => { - this.pipelineCategories = pipelineCategories.data; - }); - - }; - - modifyPipelineElementsDeployments(pipelineElements) { - pipelineElements.forEach(p => { - let selectedTargetNodeId = p.deploymentTargetNodeId - if(selectedTargetNodeId != "default") { - let selectedNode = this.edgeNodes - .filter(node => node.nodeControllerId === selectedTargetNodeId) - - p.deploymentTargetNodeHostname = selectedNode - .map(node => node.nodeMetadata.nodeAddress)[0] - - p.deploymentTargetNodePort = selectedNode - .map(node => node.nodeControllerPort)[0] - } - else { - p.deploymentTargetNodeHostname = null - p.deploymentTargetNodePort = null - } - }) - } - - - savePipelineName(switchTab) { - if (this.pipeline.name == "") { - this.showToast("error", "Please enter a name for your pipeline"); - return false; - } - - let storageRequest; - - if (this.modificationMode && this.updateMode === 'update') { - this.modifyPipelineElementsDeployments(this.pipeline.sepas) - this.modifyPipelineElementsDeployments(this.pipeline.actions) - storageRequest = this.RestApi.updatePipeline(this.pipeline); - } else { - this.modifyPipelineElementsDeployments(this.pipeline.sepas) - this.modifyPipelineElementsDeployments(this.pipeline.actions) - storageRequest = this.RestApi.storePipeline(this.pipeline); - } - - storageRequest - .then(msg => { - let data = msg.data; - if (data.success) { - this.afterStorage(data, switchTab); - } else { - this.displayErrors(data); - } - }, data => { - this.showToast("error", "Connection Error", "Could not fulfill request"); - }); - }; - - afterStorage(data, switchTab) { - this.displaySuccess(data); - this.hide(); - this.TransitionService.makePipelineAssemblyEmpty(true); - this.RestApi.removePipelineFromCache(); - if (this.ShepherdService.isTourActive()) { - this.ShepherdService.hideCurrentStep(); - } - if (switchTab && !this.startPipelineAfterStorage) { - this.$state.go("streampipes.pipelines"); - } - if (this.startPipelineAfterStorage) { - this.$state.go("streampipes.pipelines", {pipeline: data.notifications[1].description}); - } - } - - hide() { - this.$mdDialog.hide(); - }; - - showToast(type, title, description?) { - this.$mdToast.show( - this.$mdToast.simple() - .textContent(title) - .position("top right") - .hideDelay(3000) - ); - } - - toggleAdvancedSettings() { - this.advancedSettings = ! (this.advancedSettings); - } -} - -SavePipelineController.$inject = ['$mdDialog', '$state', 'RestApi', '$mdToast', 'ObjectProvider', 'pipeline', 'modificationMode', 'TransitionService', 'ShepherdService']; diff --git a/ui/src/app/editor/dialog/save-pipeline/submitPipelineModal.tmpl.html b/ui/src/app/editor/dialog/save-pipeline/submitPipelineModal.tmpl.html deleted file mode 100644 index fae3929..0000000 --- a/ui/src/app/editor/dialog/save-pipeline/submitPipelineModal.tmpl.html +++ /dev/null @@ -1,115 +0,0 @@ -<!-- - ~ Licensed to the Apache Software Foundation (ASF) under one or more - ~ contributor license agreements. See the NOTICE file distributed with - ~ this work for additional information regarding copyright ownership. - ~ The ASF licenses this file to You under the Apache License, Version 2.0 - ~ (the "License"); you may not use this file except in compliance with - ~ the License. You may obtain a copy of the License at - ~ - ~ http://www.apache.org/licenses/LICENSE-2.0 - ~ - ~ Unless required by applicable law or agreed to in writing, software - ~ distributed under the License is distributed on an "AS IS" BASIS, - ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - ~ See the License for the specific language governing permissions and - ~ limitations under the License. - ~ - --> - -<md-dialog aria-label="Save" flex="70"> - <md-toolbar> - <div class="md-toolbar-tools"> - <h2>Save Pipeline</h2> - <span flex></span> - <md-button class="md-icon-button" ng-click="ctrl.hide()"> - <md-icon md-svg-icon="navigation:ic_close_24px" aria-label="Close dialog"></md-icon> - </md-button> - </div> - </md-toolbar> - - <md-dialog-content flex="100" class="md-dialog-content"> - <form name="ctrl.submitPipelineForm"> - <div flex="100" layout="column"> - <div id="overwriteCheckbox" class="checkbox" ng-show="ctrl.modificationMode"> - <md-radio-group class="md-accent" ng-model="ctrl.updateMode"> - <md-radio-button ng-value="'update'"> - Update pipeline <b>{{ctrl.pipeline.name}}</b> - </md-radio-button> - <md-radio-button ng-value="'clone'"> - Create new pipeline - </md-radio-button> - </md-radio-group> - </div> - <div flex="100" layout="column" ng-if="!ctrl.modificationMode || ctrl.updateMode=='clone'"> - <md-input-container flex><label>Pipeline Name</label> - <input name="pipelineName" ng-model="ctrl.pipeline.name" required ng-maxlength="40"/> - <span ng-show="ctrl.submitPipelineForm.pipelineName.$touched && ctrl.submitPipelineForm.pipelineName.$error.required">Please provide a pipeline name.</span> - <span ng-show="ctrl.submitPipelineForm.pipelineName.$error.maxlength">Please provide a shorter pipeline name.</span> - </md-input-container> - <md-input-container flex><label>Description</label> - <input name="pipelineDescription" ng-model="ctrl.pipeline.description" ng-maxlength="80"/> - <span ng-show="ctrl.submitPipelineForm.pipelineDescription.$error.maxlength">Please provide a shorter description.</span> - </md-input-container> - </div> - <md-checkbox ng-click="ctrl.triggerTutorial()" ng-disabled="false" aria-label="Start Pipeline" ng-model="ctrl.startPipelineAfterStorage"> - Start pipeline immediately - </md-checkbox> - <md-checkbox ng-disabled="false" aria-label="Start Pipeline" - ng-model="ctrl.advancedSettings"> - Deployment settings - </md-checkbox> - <div ng-if="ctrl.advancedSettings"> - <h4>Nodes</h4> - <div ng-repeat="pipelineElement in ctrl.pipeline.sepas"> - <div flex="100" layout="row"> - <div flex="50" layout="row" layout-align="start center"> - <span>{{pipelineElement.name}}</span> - </div> - <div flex="50" layout="row" layout-align="start center"> - <md-select ng-model="pipelineElement.deploymentTargetNodeId" style="margin: 0;"> - <md-option - ng-value = "node.nodeControllerId" ng-repeat="node in - ctrl.deploymentOptions[pipelineElement.appId]"> - <em>{{node.nodeMetadata.nodeAddress}} - </em> - </md-option> - </md-select> - </div> - </div> - </div> - <div ng-repeat="pipelineElement in ctrl.pipeline.actions"> - <div flex="100" layout="row"> - <div flex="50" layout="row" layout-align="start center"> - <span>{{pipelineElement.name}}</span> - </div> - <div flex="50" layout="row" layout-align="start center"> - <md-select ng-model="pipelineElement.deploymentTargetNodeId" style="margin: 0;"> - <md-option - ng-value = "node.nodeControllerId" ng-repeat="node in - ctrl.deploymentOptions[pipelineElement.appId]"> - <em>{{node.nodeMetadata.nodeAddress}} - </em> - </md-option> - </md-select> - </div> - </div> - </div> - </div> - </div> - </form> - </md-dialog-content> - <md-dialog-actions layout="row"> - <sp-button sp-button-gray ng-click="ctrl.hide()"> - Close - </sp-button> - <sp-button sp-button-blue ng-disabled="ctrl.submitPipelineForm.$invalid" - ng-click="ctrl.savePipelineName(false)" - ng-disabled="ctrl.startPipelineAfterStorage"> - Save - </sp-button> - <sp-button sp-button-blue ng-disabled="ctrl.submitPipelineForm.$invalid" - ng-click="ctrl.savePipelineName(true)"> - Save and go to pipeline view - </sp-button> - </md-dialog-actions> -</md-dialog> \ No newline at end of file
