[ 
https://issues.apache.org/jira/browse/METRON-1483?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16395846#comment-16395846
 ] 

ASF GitHub Bot commented on METRON-1483:
----------------------------------------

Github user justinleet commented on a diff in the pull request:

    https://github.com/apache/metron/pull/958#discussion_r173921179
  
    --- Diff: 
metron-contrib/metron-performance/src/main/java/org/apache/metron/performance/load/LoadOptions.java
 ---
    @@ -0,0 +1,504 @@
    +/**
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements.  See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership.  The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License.  You may obtain a copy of the License at
    + *
    + *     http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.metron.performance.load;
    +
    +import com.google.common.base.Joiner;
    +import org.apache.commons.cli.*;
    +import org.apache.metron.common.utils.JSONUtils;
    +import org.apache.metron.common.utils.cli.CLIOptions;
    +import org.apache.metron.performance.sampler.BiasedSampler;
    +import org.apache.metron.stellar.common.utils.ConversionUtils;
    +import org.apache.metron.common.utils.cli.OptionHandler;
    +
    +import javax.annotation.Nullable;
    +import java.io.BufferedReader;
    +import java.io.File;
    +import java.io.FileNotFoundException;
    +import java.io.FileReader;
    +import java.io.IOException;
    +import java.util.ArrayList;
    +import java.util.EnumMap;
    +import java.util.List;
    +import java.util.Optional;
    +
    +public enum LoadOptions implements CLIOptions<LoadOptions> {
    +  HELP(new OptionHandler<LoadOptions>() {
    +
    +    @Override
    +    public String getShortCode() {
    +      return "h";
    +    }
    +
    +    @Nullable
    +    @Override
    +    public Option apply(@Nullable String s) {
    +      return new Option(s, "help", false, "Generate Help screen");
    +    }
    +  }),
    +  ZK(new OptionHandler<LoadOptions>() {
    +    @Nullable
    +    @Override
    +    public Option apply(@Nullable String s) {
    +      Option o = new Option(s, "zk_quorum", true, "zookeeper quorum");
    +      o.setArgName("QUORUM");
    +      o.setRequired(false);
    +      return o;
    +    }
    +
    +    @Override
    +    public Optional<Object> getValue(LoadOptions option, CommandLine cli) {
    +      if(option.has(cli)) {
    +        return Optional.ofNullable(option.get(cli));
    +      }
    +      else {
    +        return Optional.empty();
    +      }
    +    }
    +
    +    @Override
    +    public String getShortCode() {
    +      return "z";
    +    }
    +  }),
    +  CONSUMER_GROUP(new OptionHandler<LoadOptions>() {
    +    @Nullable
    +    @Override
    +    public Option apply(@Nullable String s) {
    +      Option o = new Option(s, "consumer_group", true, "Consumer Group.  
The default is " + LoadGenerator.CONSUMER_GROUP);
    +      o.setArgName("GROUP_ID");
    +      o.setRequired(false);
    +      return o;
    +    }
    +
    +    @Override
    +    public Optional<Object> getValue(LoadOptions option, CommandLine cli) {
    +      if(option.has(cli)) {
    +        return Optional.ofNullable(option.get(cli));
    +      }
    +      else {
    +        return Optional.of(LoadGenerator.CONSUMER_GROUP);
    +      }
    +    }
    +
    +    @Override
    +    public String getShortCode() {
    +      return "cg";
    +    }
    +  }),
    +  BIASED_SAMPLE(new OptionHandler<LoadOptions>() {
    +    @Nullable
    +    @Override
    +    public Option apply(@Nullable String s) {
    +      Option o = new Option(s, "sample_bias", true, "The discrete 
distribution to bias the sampling. " +
    +              "This is a CSV of 2 columns.  The first column is the % of 
the templates " +
    +              "and the 2nd column is the probability (0-100) that it's 
chosen.  For instance:\n" +
    +              "  20,80\n" +
    +              "  80,20\n" +
    +              "implies that 20% of the templates will comprise 80% of the 
output and the remaining 80% of the templates will comprise 20% of the 
output.");
    +      o.setArgName("BIAS_FILE");
    +      o.setRequired(false);
    +      return o;
    +    }
    +
    +    @Override
    +    public Optional<Object> getValue(LoadOptions option, CommandLine cli) {
    +      if(!option.has(cli)) {
    +        return Optional.empty();
    +      }
    +      File discreteDistributionFile  = new File(option.get(cli));
    +      if(discreteDistributionFile.exists()) {
    +        try {
    +
    +          return 
Optional.ofNullable(BiasedSampler.readDistribution(discreteDistributionFile));
    +        } catch (FileNotFoundException e) {
    +          throw new IllegalStateException("Unable to read distribution 
file: " + option.get(cli), e);
    +        } catch (IOException e) {
    +          throw new IllegalStateException("Unable to read distribution 
file: " + option.get(cli), e);
    +        }
    +      }
    +      else {
    +        throw new IllegalStateException("Unable to read distribution file: 
" + option.get(cli) + " file doesn't exist.");
    +      }
    +    }
    +
    +    @Override
    +    public String getShortCode() {
    +      return "bs";
    +    }
    +  })
    +  ,CSV(new OptionHandler<LoadOptions>() {
    +    @Nullable
    +    @Override
    +    public Option apply(@Nullable String s) {
    +      Option o = new Option(s, "csv", true, "A CSV file to emit monitoring 
data to.  " +
    +              "The format is a CSV with the following schema: timestamp, 
(name, eps, historical_mean, historical_stddev)+");
    +      o.setArgName("CSV_FILE");
    +      o.setRequired(false);
    +      return o;
    +    }
    +
    +    @Override
    +    public Optional<Object> getValue(LoadOptions option, CommandLine cli) {
    +      if(!option.has(cli)) {
    +        return Optional.empty();
    +      }
    +      return Optional.of(new File(option.get(cli)));
    +    }
    +
    +    @Override
    +    public String getShortCode() {
    +      return "c";
    +    }
    +  })
    +  ,TEMPLATE(new OptionHandler<LoadOptions>() {
    +    @Nullable
    +    @Override
    +    public Option apply(@Nullable String s) {
    +      Option o = new Option(s, "template", true, "The template file to use 
for generation.  This should be a file with a template per line with $METRON_TS 
and $METRON_GUID in the spots for timestamp and guid, if you so desire them.");
    +      o.setArgName("TEMPLATE_FILE");
    +      o.setRequired(false);
    +      return o;
    +    }
    +
    +    @Override
    +    public Optional<Object> getValue(LoadOptions option, CommandLine cli) {
    +      if(!option.has(cli)) {
    +        return Optional.empty();
    +      }
    +      File templateFile = new File(option.get(cli));
    +      if(templateFile.exists()) {
    +        List<String> templates = new ArrayList<>();
    +        try(BufferedReader br = new BufferedReader(new 
FileReader(templateFile))) {
    +          for(String line = null;(line = br.readLine()) != null;) {
    +            templates.add(line);
    +          }
    +          return Optional.of(templates);
    +        } catch (FileNotFoundException e) {
    --- End diff --
    
    Same deal on the identical catches.


> Create a tool to monitor performance of the topologies
> ------------------------------------------------------
>
>                 Key: METRON-1483
>                 URL: https://issues.apache.org/jira/browse/METRON-1483
>             Project: Metron
>          Issue Type: New Feature
>            Reporter: Casey Stella
>            Priority: Major
>
> In performance evaluation, generating synthetic load and monitoring the write 
> throughput of our kafka-to-kafka topologies has required a lot of custom 
> scripting.  We should have a tool that could do the following:
>  * Take a file representing a message template and generate synthetic load at 
> a given events per second
>  * Monitor the kafka offsets of a topic and report throughput numbers in 
> events per second
>  



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

Reply via email to