http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/iteration/pom.xml ---------------------------------------------------------------------- diff --git a/examples/iteration/pom.xml b/examples/iteration/pom.xml new file mode 100644 index 0000000..471b173 --- /dev/null +++ b/examples/iteration/pom.xml @@ -0,0 +1,37 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <artifactId>malhar-examples-iteration</artifactId> + <packaging>jar</packaging> + + <name>Apache Apex Malhar Iteration Example</name> + <description>DataTorrent example applications that demonstrates the iteration feature.</description> + + <parent> + <groupId>org.apache.apex</groupId> + <artifactId>malhar-examples</artifactId> + <version>3.7.0-SNAPSHOT</version> + </parent> + +</project>
http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/iteration/src/assemble/appPackage.xml ---------------------------------------------------------------------- diff --git a/examples/iteration/src/assemble/appPackage.xml b/examples/iteration/src/assemble/appPackage.xml new file mode 100644 index 0000000..4138cf2 --- /dev/null +++ b/examples/iteration/src/assemble/appPackage.xml @@ -0,0 +1,59 @@ +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd";> + <id>appPackage</id> + <formats> + <format>jar</format> + </formats> + <includeBaseDirectory>false</includeBaseDirectory> + <fileSets> + <fileSet> + <directory>${basedir}/target/</directory> + <outputDirectory>/app</outputDirectory> + <includes> + <include>${project.artifactId}-${project.version}.jar</include> + </includes> + </fileSet> + <fileSet> + <directory>${basedir}/target/deps</directory> + <outputDirectory>/lib</outputDirectory> + </fileSet> + <fileSet> + <directory>${basedir}/src/site/conf</directory> + <outputDirectory>/conf</outputDirectory> + <includes> + <include>*.xml</include> + </includes> + </fileSet> + <fileSet> + <directory>${basedir}/src/main/resources/META-INF</directory> + <outputDirectory>/META-INF</outputDirectory> + </fileSet> + <fileSet> + <directory>${basedir}/src/main/resources/app</directory> + <outputDirectory>/app</outputDirectory> + </fileSet> + </fileSets> + +</assembly> + http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/iteration/src/main/java/org/apache/apex/examples/iteration/Application.java ---------------------------------------------------------------------- diff --git a/examples/iteration/src/main/java/org/apache/apex/examples/iteration/Application.java b/examples/iteration/src/main/java/org/apache/apex/examples/iteration/Application.java new file mode 100644 index 0000000..4b7bae9 --- /dev/null +++ b/examples/iteration/src/main/java/org/apache/apex/examples/iteration/Application.java @@ -0,0 +1,171 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.apex.examples.iteration; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.PrintStream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.conf.Configuration; + +import com.datatorrent.api.Context; +import com.datatorrent.api.DAG; +import com.datatorrent.api.DefaultInputPort; +import com.datatorrent.api.DefaultOutputPort; +import com.datatorrent.api.StreamingApplication; +import com.datatorrent.api.annotation.ApplicationAnnotation; +import com.datatorrent.common.util.BaseOperator; +import com.datatorrent.common.util.DefaultDelayOperator; +import com.datatorrent.lib.testbench.RandomEventGenerator; + +/** + * Iteration example : <br> + * + * <pre> + * LocalMode.runApp(new Application(), 600000); // 10 min run + * </pre> + * + * Run Success : <br> + * For successful deployment and run, user should see the Fibonacci sequence, something like the + * following output on the console: + * + * <pre> + * 1 + * 1 + * 2 + * 3 + * 5 + * 8 + * 13 + * 21 + * 34 + * 55 + * ... + * </pre> + * + * + * @since 3.4.0 + */ +@ApplicationAnnotation(name = "IterationExample") +public class Application implements StreamingApplication +{ + private static final Logger LOG = LoggerFactory.getLogger(Application.class); + private String extraOutputFileName; // for unit test + + public static class FibonacciOperator extends BaseOperator + { + public long currentNumber = 1; + private transient long tempNum; + public transient DefaultInputPort<Object> dummyInputPort = new DefaultInputPort<Object>() + { + @Override + public void process(Object tuple) + { + } + }; + public transient DefaultInputPort<Long> input = new DefaultInputPort<Long>() + { + @Override + public void process(Long tuple) + { + tempNum = (currentNumber == 1) ? 1 : tuple; + } + }; + public transient DefaultOutputPort<Long> output = new DefaultOutputPort<>(); + + + @Override + public void endWindow() + { + output.emit(currentNumber); + currentNumber += tempNum; + if (currentNumber <= 0) { + currentNumber = 1; + } + } + } + + public static class StdoutOperator extends BaseOperator + { + private String extraOutputFileName; // for unit test + private transient PrintStream extraOutputStream; + /** + * This is the input port which receives the tuples that will be written to stdout. + */ + public final transient DefaultInputPort<Object> input = new DefaultInputPort<Object>() + { + @Override + @SuppressWarnings("UseOfSystemOutOrSystemErr") + public void process(Object t) + { + String s = t.toString(); + LOG.info(s); + if (extraOutputStream != null) { + extraOutputStream.println(s); + } + } + }; + + @Override + public void setup(Context.OperatorContext context) + { + if (extraOutputFileName != null) { + try { + extraOutputStream = new PrintStream(new FileOutputStream(extraOutputFileName), true); + } catch (IOException ex) { + throw new RuntimeException(ex); + } + } + } + + @Override + public void teardown() + { + extraOutputStream.close(); + } + + public void setExtraOutputFileName(String fileName) + { + this.extraOutputFileName = fileName; + } + } + + public void setExtraOutputFileName(String fileName) + { + this.extraOutputFileName = fileName; + } + + @Override + public void populateDAG(DAG dag, Configuration conf) + { + RandomEventGenerator rand = dag.addOperator("rand", new RandomEventGenerator()); + FibonacciOperator fib = dag.addOperator("FIB", FibonacciOperator.class); + DefaultDelayOperator opDelay = dag.addOperator("opDelay", DefaultDelayOperator.class); + StdoutOperator console = new StdoutOperator(); + console.setExtraOutputFileName(extraOutputFileName); + dag.addOperator("console", console); + dag.addStream("dummy_to_operator", rand.integer_data, fib.dummyInputPort); + dag.addStream("operator_to_delay", fib.output, opDelay.input, console.input); + dag.addStream("delay_to_operator", opDelay.output, fib.input); + } + +} http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/iteration/src/main/java/org/apache/apex/examples/iteration/package-info.java ---------------------------------------------------------------------- diff --git a/examples/iteration/src/main/java/org/apache/apex/examples/iteration/package-info.java b/examples/iteration/src/main/java/org/apache/apex/examples/iteration/package-info.java new file mode 100644 index 0000000..e87da8b --- /dev/null +++ b/examples/iteration/src/main/java/org/apache/apex/examples/iteration/package-info.java @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * Iteration demonstration application. + */ +package org.apache.apex.examples.iteration; http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/iteration/src/main/resources/META-INF/properties.xml ---------------------------------------------------------------------- diff --git a/examples/iteration/src/main/resources/META-INF/properties.xml b/examples/iteration/src/main/resources/META-INF/properties.xml new file mode 100644 index 0000000..6b88913 --- /dev/null +++ b/examples/iteration/src/main/resources/META-INF/properties.xml @@ -0,0 +1,45 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<configuration> + <!-- Memory settings for all examples --> + <property> + <name>dt.attr.MASTER_MEMORY_MB</name> + <value>512</value> + </property> + <property> + <name>dt.attr.DEBUG</name> + <value>true</value> + </property> + <property> + <name>dt.application.*.operator.*.attr.MEMORY_MB</name> + <value>128</value> + </property> + <property> + <name>dt.application.*.operator.*.attr.JVM_OPTIONS</name> + <value>-Xmx128M</value> + </property> + <property> + <name>dt.application.*.operator.*.port.*.attr.BUFFER_MEMORY_MB</name> + <value>128</value> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/iteration/src/test/java/org/apache/apex/examples/iteration/ApplicationTest.java ---------------------------------------------------------------------- diff --git a/examples/iteration/src/test/java/org/apache/apex/examples/iteration/ApplicationTest.java b/examples/iteration/src/test/java/org/apache/apex/examples/iteration/ApplicationTest.java new file mode 100644 index 0000000..1a6c5e0 --- /dev/null +++ b/examples/iteration/src/test/java/org/apache/apex/examples/iteration/ApplicationTest.java @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.apex.examples.iteration; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; + +import org.junit.Assert; +import org.junit.Test; + +import org.apache.hadoop.conf.Configuration; + +import com.datatorrent.api.LocalMode; + +/** + * + */ +public class ApplicationTest +{ + @Test + public void testIterationApp() throws Exception + { + LocalMode lma = LocalMode.newInstance(); + Configuration conf = new Configuration(false); + Application app = new Application(); + String outputFileName = "target/output.txt"; + long timeout = 10 * 1000; // 10 seconds + + new File(outputFileName).delete(); + app.setExtraOutputFileName(outputFileName); + lma.prepareDAG(app, conf); + LocalMode.Controller lc = lma.getController(); + lc.runAsync(); + + long startTime = System.currentTimeMillis(); + do { + try { + Thread.sleep(500); + } catch (InterruptedException ex) { + break; + } + File file = new File(outputFileName); + if (file.length() > 50) { + break; + } + } + while (System.currentTimeMillis() - startTime < timeout); + + lc.shutdown(); + try (BufferedReader br = new BufferedReader(new FileReader(outputFileName))) { + Assert.assertEquals("1", br.readLine()); + Assert.assertEquals("1", br.readLine()); + Assert.assertEquals("2", br.readLine()); + Assert.assertEquals("3", br.readLine()); + Assert.assertEquals("5", br.readLine()); + Assert.assertEquals("8", br.readLine()); + Assert.assertEquals("13", br.readLine()); + Assert.assertEquals("21", br.readLine()); + Assert.assertEquals("34", br.readLine()); + Assert.assertEquals("55", br.readLine()); + Assert.assertEquals("89", br.readLine()); + Assert.assertEquals("144", br.readLine()); + Assert.assertEquals("233", br.readLine()); + Assert.assertEquals("377", br.readLine()); + Assert.assertEquals("610", br.readLine()); + Assert.assertEquals("987", br.readLine()); + } + } +} http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/iteration/src/test/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/examples/iteration/src/test/resources/log4j.properties b/examples/iteration/src/test/resources/log4j.properties new file mode 100644 index 0000000..cf0d19e --- /dev/null +++ b/examples/iteration/src/test/resources/log4j.properties @@ -0,0 +1,43 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +log4j.rootLogger=DEBUG,CONSOLE + +log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender +log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout +log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n +log4j.appender.CONSOLE.threshold=${test.log.console.threshold} +test.log.console.threshold=DEBUG + +log4j.appender.RFA=org.apache.log4j.RollingFileAppender +log4j.appender.RFA.layout=org.apache.log4j.PatternLayout +log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n +log4j.appender.RFA.File=/tmp/app.log + +# to enable, add SYSLOG to rootLogger +log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender +log4j.appender.SYSLOG.syslogHost=127.0.0.1 +log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout +log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n +log4j.appender.SYSLOG.Facility=LOCAL1 + +log4j.logger.org=info +#log4j.logger.org.apache.commons.beanutils=warn +log4j.logger.com.datatorrent=debug +log4j.logger.org.apache.apex=debug http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/machinedata/pom.xml ---------------------------------------------------------------------- diff --git a/examples/machinedata/pom.xml b/examples/machinedata/pom.xml new file mode 100644 index 0000000..cf17d9b --- /dev/null +++ b/examples/machinedata/pom.xml @@ -0,0 +1,66 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <artifactId>malhar-examples-machinedata</artifactId> + <packaging>jar</packaging> + + <name>Apache Apex Malhar MachineData Example</name> + <description></description> + + <parent> + <groupId>org.apache.apex</groupId> + <artifactId>malhar-examples</artifactId> + <version>3.7.0-SNAPSHOT</version> + </parent> + + <dependencies> + <dependency> + <groupId>org.apache.apex</groupId> + <artifactId>malhar-contrib</artifactId> + <version>${project.version}</version> + <exclusions> + <exclusion> + <groupId>*</groupId> + <artifactId>*</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>redis.clients</groupId> + <artifactId>jedis</artifactId> + <version>2.5.1</version> + </dependency> + <dependency> + <groupId>javax.mail</groupId> + <artifactId>javax.mail-api</artifactId> + <version>1.5.2</version> + </dependency> + <dependency> + <groupId>com.sun.mail</groupId> + <artifactId>javax.mail</artifactId> + <version>1.5.2</version> + </dependency> + </dependencies> + +</project> http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/machinedata/src/assemble/appPackage.xml ---------------------------------------------------------------------- diff --git a/examples/machinedata/src/assemble/appPackage.xml b/examples/machinedata/src/assemble/appPackage.xml new file mode 100644 index 0000000..4138cf2 --- /dev/null +++ b/examples/machinedata/src/assemble/appPackage.xml @@ -0,0 +1,59 @@ +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd";> + <id>appPackage</id> + <formats> + <format>jar</format> + </formats> + <includeBaseDirectory>false</includeBaseDirectory> + <fileSets> + <fileSet> + <directory>${basedir}/target/</directory> + <outputDirectory>/app</outputDirectory> + <includes> + <include>${project.artifactId}-${project.version}.jar</include> + </includes> + </fileSet> + <fileSet> + <directory>${basedir}/target/deps</directory> + <outputDirectory>/lib</outputDirectory> + </fileSet> + <fileSet> + <directory>${basedir}/src/site/conf</directory> + <outputDirectory>/conf</outputDirectory> + <includes> + <include>*.xml</include> + </includes> + </fileSet> + <fileSet> + <directory>${basedir}/src/main/resources/META-INF</directory> + <outputDirectory>/META-INF</outputDirectory> + </fileSet> + <fileSet> + <directory>${basedir}/src/main/resources/app</directory> + <outputDirectory>/app</outputDirectory> + </fileSet> + </fileSets> + +</assembly> + http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/machinedata/src/main/html/global.js ---------------------------------------------------------------------- diff --git a/examples/machinedata/src/main/html/global.js b/examples/machinedata/src/main/html/global.js new file mode 100644 index 0000000..753f58f --- /dev/null +++ b/examples/machinedata/src/main/html/global.js @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * Declaration and initialization for global variables. + */ + +// url parameters +var params; + +// Data Points +var aggrData; +var aggrDataPoints; +var contData; +var contDataPoints; + +// CPU data table +var cpuTable; +var cpuChart; +var cpuView; + +// ram data table +var ramTable; +var ramChart; +var ramView; + +// hdd data table +var hddTable; +var hddChart; +var hddView; + +// chart options +var chartOptions; + +// Date formatter +var dateFormatter; + +// window look back value +var lookback; +var aggrLookBack; +var contLookBack; +var contRefresh; + +// Get split query string +function QueryString() { + var query_string = {}; + var query = window.location.search.substring(1); + return query; +} +function SplitQuery(query) +{ + var params = {}; + var vars = query.split("&"); + for (var i=0;i<vars.length;i++) + { + var pair = vars[i].split("="); + if(pair.length == 2) + { + params[pair[0]] = pair[1]; + } + } + return params; +} + +// Initialize global variable(s) +function InitializeGlobal() +{ + // Initialize params + params = SplitQuery(QueryString()); + + // Initialize data points + aggrDataPoints = new Array(); + contDataPoints = new Array(); + + // Initialize cpu table + cpuTable = new google.visualization.DataTable(); + cpuTable.addColumn('datetime', 'Time'); + cpuTable.addColumn('number', 'CPU'); + chartOptions = { width: 600, height: 300, legend: 'none', pointSize: 0, lineWidth : 1 }; + cpuChart = new google.visualization.ScatterChart(document.getElementById('chart_div')); + cpuView = new google.visualization.DataView(cpuTable); + + // Initialize ram table + ramTable = new google.visualization.DataTable(); + ramTable.addColumn('datetime', 'Time'); + ramTable.addColumn('number', 'RAM');; + ramChart = new google.visualization.ScatterChart(document.getElementById('chart1_div')); + ramView = new google.visualization.DataView(ramTable); + + // Initialize hdd table + hddTable = new google.visualization.DataTable(); + hddTable.addColumn('datetime', 'Time'); + hddTable.addColumn('number', 'HDD');; + hddChart = new google.visualization.ScatterChart(document.getElementById('chart2_div')); + hddView = new google.visualization.DataView(hddTable); + + // get lookback value + lookback = (new Date().getTime()/1000) - 3600*6; + if (params['lookback'] && (params['lookback'].length > 0)) lookback = (new Date().getTime()/1000) - (3600*(parseInt(params['lookback']))); + aggrLookBack = lookback; + + // get continuos lookback + contLookBack = lookback; + contRefresh = 5; + + // get param lookback + paramLookBack = 6; + if (params['lookback'] && (params['lookback'].length > 0)) paramLookBack = parseInt(params['lookback']); + //if (params['refresh'] && (params['refresh'].length > 0)) contRefresh = parseInt(params['refresh']); +} + + +/** + * Function to create fetch urls from given parameters + */ +function DataUrl() +{ + var url = "json.php?bucket=m"; + url += "&customer="; + if (params['customer']) + { + url += params['customer']; + } + url += "&product="; + if (params['product']) + { + url += params['product']; + } + url += "&os="; + if (params['os']) + { + url += params['os']; + } + url += "&software1="; + if (params['software1']) + { + url += params['software1']; + } + url += "&software2="; + if (params['software2']) + { + url += params['software2']; + } + url += "&software3="; + if (params['software3']) + { + url += params['software3']; + } + url += "&from="; + url += Math.floor(lookback); + return url; +} + +/** + * Creates data table with time stamp and cpu values. + * Draw line chart for time vs cpu. + */ +function DrawCPUChart() +{ + // create/delete rows + if (cpuTable.getNumberOfRows() < aggrDataPoints.length) + { + var numRows = aggrDataPoints.length - cpuTable.getNumberOfRows(); + cpuTable.addRows(numRows); + } else { + for(var i=(cpuTable.getNumberOfRows()-1); i >= aggrDataPoints.length; i--) + { + cpuTable.removeRow(i); + } + } + + // Populate data table with time/cpu data points. + for(var i=0; i < cpuTable.getNumberOfRows(); i++) + { + //if(parseFloat(aggrDataPoints[i].cpu) < 500) continue; + cpuTable.setCell(i, 0, new Date(parseInt(aggrDataPoints[i].timestamp))); + cpuTable.setCell(i, 1, parseFloat(aggrDataPoints[i].cpu)); + } + + // Draw line chart. + chartOptions.title = 'CPU Usage (%)'; + cpuChart.draw(cpuView, chartOptions); +} + +/** + * Creates data table with time stamp and revenu values. + * Draw line chart for time vs ram. + */ +function DrawRAMChart() +{ + // create/delete rows + if (ramTable.getNumberOfRows() < aggrDataPoints.length) + { + var numRows = aggrDataPoints.length - ramTable.getNumberOfRows(); + ramTable.addRows(numRows); + } else { + for(var i=(ramTable.getNumberOfRows()-1); i >= aggrDataPoints.length; i--) + { + ramTable.removeRow(i); + } + } + + // Populate data table with time/ram data points. + for(var i=0; i < ramTable.getNumberOfRows(); i++) + { + ramTable.setCell(i, 0, new Date(parseInt(aggrDataPoints[i].timestamp))); + ramTable.setCell(i, 1, parseFloat(aggrDataPoints[i].ram)); + } + + // Draw line chart. + chartOptions.title = 'RAM Usage (%)'; + ramChart.draw(ramView, chartOptions); +} + +/** + * Creates data table with time stamp and hdd values. + * Draw line chart for time vs hdd. + */ +function DrawHDDChart() +{ + // create/delete rows + if (hddTable.getNumberOfRows() < aggrDataPoints.length) + { + var numRows = aggrDataPoints.length - hddTable.getNumberOfRows(); + hddTable.addRows(numRows); + } else { + for(var i=(hddTable.getNumberOfRows()-1); i >= aggrDataPoints.length; i--) + { + hddTable.removeRow(i); + } + } + + // Populate data table with time/hdd data points. + for(var i=0; i < hddTable.getNumberOfRows(); i++) + { + hddTable.setCell(i, 0, new Date(parseInt(aggrDataPoints[i].timestamp))); + hddTable.setCell(i, 1, parseInt(aggrDataPoints[i].hdd)); + } + + // Draw line chart. + chartOptions.title = 'HDD Usage (%)'; + hddChart.draw(hddView, chartOptions); +} + +/** + * Sort json array + */ +function sortByKey(array, key) { + return array.sort(function(a, b) { + var x = a[key]; var y = b[key]; + return ((x < y) ? -1 : ((x > y) ? 1 : 0)); + }); +} + http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/machinedata/src/main/html/index.php ---------------------------------------------------------------------- diff --git a/examples/machinedata/src/main/html/index.php b/examples/machinedata/src/main/html/index.php new file mode 100644 index 0000000..d52a960 --- /dev/null +++ b/examples/machinedata/src/main/html/index.php @@ -0,0 +1,263 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd";> +<html> +<head> +<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"> +<title>Machine Generated Data Example </title> + +<link rel="stylesheet" type="text/css" href="malhar.css"> + +<!-- Google charts include --> +<script type="text/javascript" src="https://www.google.com/jsapi";></script> +<script type="text/javascript"> +google.load('visualization', '1', {'packages':['corechart']}); +</script> + +<!-- Malhar charting utils --> +<script type="text/javascript" src="global.js"></script> + +<!-- window onload --> +<script type="text/javascript"> + +function DrawAggrCharts() +{ + // get refresh url + lookback = aggrLookBack; + var url = DataUrl(); + + // fetch data, draw charts + try + { + var connect = new XMLHttpRequest(); + connect.onreadystatechange = function() { + if(connect.readyState==4 && connect.status==200) { + +console.log(url); + aggrData = connect.response; + var pts = JSON.parse(aggrData); + aggrDataPoints = new Array(); + for(var i=0; i < pts.length; i++) aggrDataPoints.push(pts[i]); + DrawCPUChart(); + DrawRAMChart(); + DrawHDDChart(); + //DrawImpressionsChart(); + delete aggrData; + } + } + connect.open('GET', url, true); + connect.send(null); + } catch(e) { + } + aggrLookBack += 30; +} + +function DrawContCharts() +{ + // get refresh url + lookback = contLookBack; + var url = DataUrl(); + //document.getElementById('chart_div').innerHTML = url; + + // fetch data, draw charts + try + { + var connect = new XMLHttpRequest(); + connect.onreadystatechange = function() { + if(connect.readyState==4 && connect.status==200) { + contData = connect.response; + var newPts = JSON.parse(contData); + contDataPoints = new Array(); + for(var i=0; i < newPts.length; i++) contDataPoints.push(newPts[i]); + DrawCtrChart() ; + DrawMarginChart(); + delete contData; + delete newPts; + } + } + connect.open('GET', url, true); + connect.send(null); + } catch(e) { + } + contLookBack += contRefresh; +} + +window.onload = function() { + + // Initialize global + InitializeGlobal(); + + // Inituialize form fields + if (params['customer']) document.getElementById('customer').value = params['customer']; + if (params['product']) document.getElementById('product').value = params['product']; + if (params['os']) document.getElementById('os').value = params['os']; + if (params['software1']) document.getElementById('software1').value = params['software1']; + if (params['software2']) document.getElementById('software2').value = params['software2']; + if (params['software3']) document.getElementById('software3').value = params['software3']; + if (params['refresh']) + { + document.getElementById('refresh').value = params['refresh']; + } else { + document.getElementById('refresh').value = 5; + } + if (params['lookback']) + { + document.getElementById('lookback').value = params['lookback']; + } else { + document.getElementById('lookback').value = 6; + } + + // draw charts + DrawAggrCharts(); + //DrawContCharts(); + setInterval(DrawAggrCharts, 30000); + //setInterval(DrawContCharts, contRefresh * 1000); +}; + +</script> + +</head> +<body> + + <div id="header"> + <ul class="dashboard-modes"> + <li> + <a href="#" class="active">Machine Generated Data Example </a> + </li> + </ul> + + </div> + + <div id="main"> + <div id="pagecontent"> + <div class="dashboardMgr"> + <div class="inner" style=""> + <h2 class="title">View Real Time Data Charts</h2> + <form method="GET" action="index.php"> + + <label for="customer">Customer ID:</label> + <select name="customer" id="customer" style="width:200px;"> + <option value="">ALL</option> + <?php + for ($i = 1; $i <= 5; $i++) { + print "<option value=\"$i\">Customer $i</option>\n"; + } + ?> + </select> + + <label for="">Product ID:</label> + <select name="product" id="product" style="width:200px;"> + <option value="">ALL</option> + <?php + for ($i = 4; $i <= 6; $i++) { + print "<option value=\"$i\">Product $i</option>\n"; + } + ?> + </select> + + <label for="">Product OS:</label> + <select name="os" id="os" style="width:200px;"> + <option value="">ALL</option> + <?php + for ($i = 10; $i <= 12; $i++) { + print "<option value=\"$i\">OS $i</option>\n"; + } + ?> + </select> + + <label for="software1">Software1 Ver:</label> + <select name="software1" id="software1" style="width:200px;"> + <option value="">ALL</option> + <?php + for ($i = 10; $i <= 12; $i++) { + print "<option value=\"$i\">Software1 Version $i</option>\n"; + } + ?> + </select> + + <label for="software2">Software2 Ver:</label> + <select name="software2" id="software2" style="width:200px;"> + <option value="">ALL</option> + <?php + for ($i = 12; $i <= 14; $i++) { + print "<option value=\"$i\">Software2 Version $i</option>\n"; + } + ?> + </select> + + <label for="software3">Software3 Ver:</label> + <select name="software3" id="software3" style="width:200px;"> + <option value="">ALL</option> + <?php + for ($i = 4; $i <= 6; $i++) { + print "<option value=\"$i\">Software3 Version $i</option>\n"; + } + ?> + </select> + + <label for="">Refresh Interval:</label> + <div class="input-append"> + <input type="text" name="refresh" id="refresh" class="input-small"/> + <span class="add-on">Secs</span> + </div> + + + <label for="">Look Back:</label> + <div class="input-append"> + <input type="text" name="lookback" id="lookback" class="input-small"/> + <span class="add-on">Hours</span> + </div> + + <input type="submit" value="submit" class="btn btn-primary" /> + + </form> + </div> + <div class="collapser-container"> + <div class="collapser"> + <div class="collapse-dot"></div> + <div class="collapse-dot"></div> + <div class="collapse-dot"></div> + </div> + </div> + </div> + <div class="dashboardMain"> + + <!-- <table><tbody> + <tr> + <td><div id="chart_div"></div></td> + <td><div id="chart1_div" ></div></td> + </tr> + <tr> + <td><div id="chart2_div" ></div></td> + <td><div id="chart3_div" ></div></td> + </tr> + <tr> + <td><div id="chart4_div" ></div></td> + <td><div id="chart5_div" ></div></td> + </tr> + </tr></tbody></table> --> + <div class="chart-ctnr" id="chart_div"></div> + <div class="chart-ctnr" id="chart1_div" ></div> + <div class="chart-ctnr" id="chart2_div" ></div> +<!-- <div class="chart-ctnr" id="chart3_div" ></div> + <div class="chart-ctnr" id="chart4_div" ></div> + <div class="chart-ctnr" id="chart5_div" ></div> --> + </div> +</body> +</html> http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/d5bf96ca/examples/machinedata/src/main/html/json.php ---------------------------------------------------------------------- diff --git a/examples/machinedata/src/main/html/json.php b/examples/machinedata/src/main/html/json.php new file mode 100644 index 0000000..75a7117 --- /dev/null +++ b/examples/machinedata/src/main/html/json.php @@ -0,0 +1,96 @@ +<?php +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +header("Content-type: application/json"); +$redis = new Redis(); +$redis->connect('localhost'); +$redis->select(15); +$from = $_GET['from']; +$bucket = $_GET['bucket']; +$customer = $_GET['customer']; +$product = $_GET['product']; +$os = $_GET['os']; +$software1 = $_GET['software1']; +$software2 = $_GET['software2']; +$software3 = $_GET['software3']; + +switch ($bucket) { +case 'D': + $format = 'Ymd'; + $incr = 60 * 60 * 24; + break; +case 'h': + $format = 'YmdH'; + $incr = 60 * 60; + break; +case 'm': + $format = 'YmdHi'; + $incr = 60; + break; +default: + break; +} + +$arr = array(); +if ($customer != '') { + $arr[] = "0:".$customer; +} +if ($product != '') { + $arr[] = "1:".$product; +} +if ($os != '') { + $arr[] = "2:".$os; +} +if ($software1 != '') { + $arr[] = "3:".$software1; +} +if ($software2 != '') { + $arr[] = "4:".$software2; +} +if ($software3 != '') { + $arr[] = "5:".$software3; +} +$subpattern = ""; +if (count($arr) != 0) { + $subpattern = join("|", $arr); +} + +$result = array(); + +while ($from < time()) { + $date = gmdate($format, $from); + if ($subpattern != '') { + $key = $bucket . '|' . $date . '|' . $subpattern; + } else { + $key = $bucket . '|' . $date ; + } + $hash = $redis->hGetAll($key); + if ($hash) { + $cpu = $hash['cpu']; + $ram = $hash['ram']; + $hdd = $hash['hdd']; + $result[] = array('timestamp'=> $from * 1000, 'cpu'=>$cpu, 'ram'=>$ram, 'hdd'=>$hdd); + } + $from += $incr; +} + +array_pop($result); +print json_encode($result); + +?>
