[ 
https://issues.apache.org/jira/browse/METRON-1483?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16397158#comment-16397158
 ] 

ASF GitHub Bot commented on METRON-1483:
----------------------------------------

Github user cestella commented on a diff in the pull request:

    https://github.com/apache/metron/pull/958#discussion_r174186403
  
    --- Diff: 
metron-contrib/metron-performance/src/main/java/org/apache/metron/performance/load/LoadGenerator.java
 ---
    @@ -0,0 +1,165 @@
    +/**
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements.  See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership.  The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License.  You may obtain a copy of the License at
    + *
    + *     http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.metron.performance.load;
    +
    +
    +import com.google.common.base.Joiner;
    +import org.apache.commons.cli.CommandLine;
    +import org.apache.commons.cli.PosixParser;
    +import org.apache.kafka.clients.producer.KafkaProducer;
    +import org.apache.metron.common.utils.KafkaUtils;
    +import org.apache.metron.performance.load.monitor.AbstractMonitor;
    +import org.apache.metron.performance.load.monitor.EPSGeneratedMonitor;
    +import 
org.apache.metron.performance.load.monitor.EPSThroughputWrittenMonitor;
    +import org.apache.metron.performance.load.monitor.MonitorTask;
    +import org.apache.metron.performance.load.monitor.writers.CSVWriter;
    +import org.apache.metron.performance.load.monitor.writers.ConsoleWriter;
    +import org.apache.metron.performance.load.monitor.writers.Writable;
    +import org.apache.metron.performance.load.monitor.writers.Writer;
    +import org.apache.metron.performance.sampler.BiasedSampler;
    +import org.apache.metron.performance.sampler.Sampler;
    +import org.apache.metron.performance.sampler.UnbiasedSampler;
    +
    +import java.io.File;
    +import java.util.ArrayList;
    +import java.util.EnumMap;
    +import java.util.HashMap;
    +import java.util.List;
    +import java.util.Map;
    +import java.util.Optional;
    +import java.util.Timer;
    +import java.util.TimerTask;
    +import java.util.concurrent.ExecutorService;
    +import java.util.concurrent.Executors;
    +import java.util.concurrent.atomic.AtomicLong;
    +import java.util.function.Consumer;
    +
    +public class LoadGenerator
    +{
    +  public static String CONSUMER_GROUP = "load.group";
    +  public static long SEND_PERIOD_MS = 100;
    +  public static long MONITOR_PERIOD_MS = 1000*10;
    +  private static ExecutorService pool;
    +  private static ThreadLocal<KafkaProducer> kafkaProducer;
    +  public static AtomicLong numSent = new AtomicLong(0);
    +
    +  public static void main( String[] args ) throws Exception {
    +    CommandLine cli = LoadOptions.parse(new PosixParser(), args);
    +    EnumMap<LoadOptions, Optional<Object>> evaluatedArgs = 
LoadOptions.createConfig(cli);
    +    Map<String, Object> kafkaConfig = new HashMap<>();
    +    kafkaConfig.put("key.serializer", 
"org.apache.kafka.common.serialization.StringSerializer");
    +    kafkaConfig.put("value.serializer", 
"org.apache.kafka.common.serialization.StringSerializer");
    +    kafkaConfig.put("key.deserializer", 
"org.apache.kafka.common.serialization.StringDeserializer");
    +    kafkaConfig.put("value.deserializer", 
"org.apache.kafka.common.serialization.StringDeserializer");
    +    if(LoadOptions.ZK.has(cli)) {
    +      String zkQuorum = (String) evaluatedArgs.get(LoadOptions.ZK).get();
    +      kafkaConfig.put("bootstrap.servers", 
Joiner.on(",").join(KafkaUtils.INSTANCE.getBrokersFromZookeeper(zkQuorum)));
    +    }
    +    String groupId = 
evaluatedArgs.get(LoadOptions.CONSUMER_GROUP).get().toString();
    +    System.out.println("Consumer Group: " + groupId);
    +    kafkaConfig.put("group.id", groupId);
    --- End diff --
    
    done


> Create a tool to monitor performance of the topologies
> ------------------------------------------------------
>
>                 Key: METRON-1483
>                 URL: https://issues.apache.org/jira/browse/METRON-1483
>             Project: Metron
>          Issue Type: New Feature
>            Reporter: Casey Stella
>            Priority: Major
>
> In performance evaluation, generating synthetic load and monitoring the write 
> throughput of our kafka-to-kafka topologies has required a lot of custom 
> scripting.  We should have a tool that could do the following:
>  * Take a file representing a message template and generate synthetic load at 
> a given events per second
>  * Monitor the kafka offsets of a topic and report throughput numbers in 
> events per second
>  



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

Reply via email to