[
https://issues.apache.org/jira/browse/BEAM-11075?focusedWorklogId=506906&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-506906
]
ASF GitHub Bot logged work on BEAM-11075:
-----------------------------------------
Author: ASF GitHub Bot
Created on: 03/Nov/20 13:59
Start Date: 03/Nov/20 13:59
Worklog Time Spent: 10m
Work Description: tszerszen commented on a change in pull request #13245:
URL: https://github.com/apache/beam/pull/13245#discussion_r516502298
##########
File path: sdks/go/pkg/beam/io/synthetic/source.go
##########
@@ -165,10 +175,12 @@ type SourceConfigBuilder struct {
func DefaultSourceConfig() *SourceConfigBuilder {
return &SourceConfigBuilder{
cfg: SourceConfig{
- NumElements: 1, // 0 is invalid (drops elements).
- InitialSplits: 1, // 0 is invalid (drops elements).
- KeySize: 8, // 0 is invalid (drops elements).
- ValueSize: 8, // 0 is invalid (drops elements).
+ NumElements: 1, // 0 is invalid (drops elements).
+ InitialSplits: 1, // 0 is invalid (drops elements).
+ KeySize: 8, // 0 is invalid (drops elements).
+ ValueSize: 8, // 0 is invalid (drops elements).
+ HotKeys: 2, // 0 is invalid (drops elements).
Review comment:
```
NumElements: 1, // 0 is invalid (drops elements).
InitialSplits: 1, // 0 is invalid (drops elements).
KeySize: 8, // 0 is invalid (drops elements).
ValueSize: 8, // 0 is invalid (drops elements).
NumHotKeys: 0,
HotKeyFraction: 0,
```
##########
File path: sdks/go/pkg/beam/io/synthetic/source.go
##########
@@ -165,10 +175,12 @@ type SourceConfigBuilder struct {
func DefaultSourceConfig() *SourceConfigBuilder {
return &SourceConfigBuilder{
cfg: SourceConfig{
- NumElements: 1, // 0 is invalid (drops elements).
- InitialSplits: 1, // 0 is invalid (drops elements).
- KeySize: 8, // 0 is invalid (drops elements).
- ValueSize: 8, // 0 is invalid (drops elements).
+ NumElements: 1, // 0 is invalid (drops elements).
+ InitialSplits: 1, // 0 is invalid (drops elements).
+ KeySize: 8, // 0 is invalid (drops elements).
+ ValueSize: 8, // 0 is invalid (drops elements).
+ HotKeys: 2, // 0 is invalid (drops elements).
Review comment:
```
NumElements: 1, // 0 is invalid (drops elements).
InitialSplits: 1, // 0 is invalid (drops elements).
KeySize: 8, // 0 is invalid (drops elements).
ValueSize: 8, // 0 is invalid (drops elements).
NumHotKeys: 0,
HotKeyFraction: 0,
```
##########
File path: sdks/go/pkg/beam/io/synthetic/source.go
##########
@@ -262,8 +281,10 @@ func (b *SourceConfigBuilder) BuildFromJSON(jsonData
[]byte) SourceConfig {
// synthetic source. It should be created via a SourceConfigBuilder, not by
// directly initializing it (the fields are public to allow encoding).
type SourceConfig struct {
- NumElements int `json:"num_records"`
- InitialSplits int `json:"initial_splits"`
- KeySize int `json:"key_size"`
- ValueSize int `json:"value_size"`
+ NumElements int `json:"num_records"`
+ InitialSplits int `json:"initial_splits"`
+ HotKeys int `json:"hot_keys"`
Review comment:
```
NumHotKeys int `json:"num_hot_keys"`
HotKeyFraction float64 `json:"hot_key_fraction"`
```
##########
File path: sdks/go/pkg/beam/io/synthetic/source.go
##########
@@ -262,8 +281,10 @@ func (b *SourceConfigBuilder) BuildFromJSON(jsonData
[]byte) SourceConfig {
// synthetic source. It should be created via a SourceConfigBuilder, not by
// directly initializing it (the fields are public to allow encoding).
type SourceConfig struct {
- NumElements int `json:"num_records"`
- InitialSplits int `json:"initial_splits"`
- KeySize int `json:"key_size"`
- ValueSize int `json:"value_size"`
+ NumElements int `json:"num_records"`
+ InitialSplits int `json:"initial_splits"`
+ HotKeys int `json:"hot_keys"`
Review comment:
```
// NumHotKeys determines the number of keys with the same value among
// generated keys.
//
// Valid values are in the range of [0, ...] and the default value is 0.
func (b *SourceConfigBuilder) NumHotKeys(val int) *SourceConfigBuilder {
b.cfg.NumHotKeys = val
return b
}
// HotKeyFraction determines the value of hot key fraction.
//
// Valid values are floating point numbers from 0 to 1.
func (b *SourceConfigBuilder) HotKeyFraction(val float64)
*SourceConfigBuilder {
b.cfg.HotKeyFraction = val
return b
}
```
##########
File path: sdks/go/pkg/beam/io/synthetic/source.go
##########
@@ -235,6 +247,12 @@ func (b *SourceConfigBuilder) Build() SourceConfig {
if b.cfg.ValueSize <= 0 {
panic(fmt.Sprintf("SourceConfig.ValueSize must be >= 1. Got:
%v", b.cfg.ValueSize))
}
+ if b.cfg.HotKeys <= 0 {
+ panic(fmt.Sprintf("SourceConfig.HotKeys must be >= 1. Got: %v",
b.cfg.HotKeys))
+ }
+ if b.cfg.HotKeyFraction <= 0 {
+ panic(fmt.Sprintf("SourceConfig.HotKeyFraction must be >= 1.
Got: %v", b.cfg.HotKeyFraction))
Review comment:
```
if b.cfg.HotKeyFraction < 0 || b.cfg.HotKeyFraction > 1 {
panic(fmt.Sprintf("SourceConfig.HotKeyFraction must be a
floating point number from 0 and 1. Got: %v", b.cfg.NumHotKeys))
}
```
##########
File path: sdks/go/pkg/beam/io/synthetic/source.go
##########
@@ -246,7 +264,8 @@ func (b *SourceConfigBuilder) Build() SourceConfig {
// {
// "num_records": 5,
// "key_size": 5,
-// "value_size": 5
+// "value_size": 5,
+// "num_keys": 5,
Review comment:
```
// {
// "num_records": 5,
// "key_size": 5,
// "value_size": 5
// "value_size": 5,
// "num_hot_keys": 5,
// }
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 506906)
Time Spent: 1h 40m (was: 1.5h)
> Load Tests for Go SDK
> ---------------------
>
> Key: BEAM-11075
> URL: https://issues.apache.org/jira/browse/BEAM-11075
> Project: Beam
> Issue Type: Test
> Components: sdk-go, testing
> Reporter: Kamil Wasilewski
> Assignee: Kamil Wasilewski
> Priority: P3
> Time Spent: 1h 40m
> Remaining Estimate: 0h
>
> We have Load Tests for Python and Java SDKs[1], but we are missing the ones
> for Go SDK.
> Tests to be done:
> * ParDo
> * Combine
> * coGBK
> * GBK
> * Side Input
> The tests should run on Dataflow and Flink. The tests should be using
> synthetic source and be running in batch mode.
> [1]http://metrics.beam.apache.org/dashboards/f/OtXje1iGz/performance-tests-metrics
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
