[
https://issues.apache.org/jira/browse/BEAM-4726?focusedWorklogId=118809&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-118809
]
ASF GitHub Bot logged work on BEAM-4726:
----------------------------------------
Author: ASF GitHub Bot
Created on: 03/Jul/18 20:03
Start Date: 03/Jul/18 20:03
Worklog Time Spent: 10m
Work Description: herohde closed pull request #5881: [BEAM-4726] Add a
simple benchmark to the ParDo execution node.
URL: https://github.com/apache/beam/pull/5881
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/sdks/go/pkg/beam/core/runtime/exec/pardo_test.go
b/sdks/go/pkg/beam/core/runtime/exec/pardo_test.go
index 3b606819dc8..731a5cd7f65 100644
--- a/sdks/go/pkg/beam/core/runtime/exec/pardo_test.go
+++ b/sdks/go/pkg/beam/core/runtime/exec/pardo_test.go
@@ -20,6 +20,7 @@ import (
"testing"
"github.com/apache/beam/sdks/go/pkg/beam/core/graph"
+ "github.com/apache/beam/sdks/go/pkg/beam/core/graph/mtime"
"github.com/apache/beam/sdks/go/pkg/beam/core/graph/window"
"github.com/apache/beam/sdks/go/pkg/beam/core/typex"
"github.com/apache/beam/sdks/go/pkg/beam/core/util/reflectx"
@@ -95,3 +96,55 @@ func TestParDo(t *testing.T) {
t.Errorf("pardo(sumFn) side input = %v, want %v",
extractValues(sum.Elements...), extractValues(expectedSum...))
}
}
+
+func emitSumFn(n int, emit func(int)) {
+ emit(n + 1)
+}
+
+// BenchmarkParDo_EmitSumFn measures the overhead of invoking a ParDo in a
plan.
+//
+// On @lostluck's desktop:
+// BenchmarkParDo_EmitSumFn-12 1000000 1606 ns/op
585 B/op 7 allocs/op
+func BenchmarkParDo_EmitSumFn(b *testing.B) {
+ fn, err := graph.NewDoFn(emitSumFn)
+ if err != nil {
+ b.Fatalf("invalid function: %v", err)
+ }
+
+ g := graph.New()
+ nN := g.NewNode(typex.New(reflectx.Int),
window.DefaultWindowingStrategy(), true)
+ edge, err := graph.NewParDo(g, g.Root(), fn, []*graph.Node{nN}, nil)
+ if err != nil {
+ b.Fatalf("invalid pardo: %v", err)
+ }
+ var in []int
+ for i := 0; i < b.N; i++ {
+ in = append(in)
+ }
+
+ process := make(chan MainInput, 1)
+ out := &CaptureNode{UID: 1}
+ pardo := &ParDo{UID: 3, Fn: edge.DoFn, Inbound: edge.Input, Out:
[]Node{out}}
+ n := &BenchRoot{UID: 4, Elements: process, Out: pardo}
+ p, err := NewPlan("a", []Unit{n, pardo, out})
+ if err != nil {
+ b.Fatalf("failed to construct plan: %v", err)
+ }
+ go func() {
+ if err := p.Execute(context.Background(), "1", nil); err != nil
{
+ b.Fatalf("execute failed: %v", err)
+ }
+ if err := p.Down(context.Background()); err != nil {
+ b.Fatalf("down failed: %v", err)
+ }
+ }()
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ process <- MainInput{Key: FullValue{
+ Windows: window.SingleGlobalWindow,
+ Timestamp: mtime.ZeroTimestamp,
+ Elm: 1,
+ }}
+ }
+ close(process)
+}
diff --git a/sdks/go/pkg/beam/core/runtime/exec/unit_test.go
b/sdks/go/pkg/beam/core/runtime/exec/unit_test.go
index f65c6cd4c19..c05a94d907d 100644
--- a/sdks/go/pkg/beam/core/runtime/exec/unit_test.go
+++ b/sdks/go/pkg/beam/core/runtime/exec/unit_test.go
@@ -109,3 +109,39 @@ func (n *FixedRoot) FinishBundle(ctx context.Context)
error {
func (n *FixedRoot) Down(ctx context.Context) error {
return nil
}
+
+// BenchRoot is a test Root that emits elements through a channel for
benchmarking purposes.
+type BenchRoot struct {
+ UID UnitID
+ Elements <-chan MainInput
+ Out Node
+}
+
+func (n *BenchRoot) ID() UnitID {
+ return n.UID
+}
+
+func (n *BenchRoot) Up(ctx context.Context) error {
+ return nil
+}
+
+func (n *BenchRoot) StartBundle(ctx context.Context, id string, data
DataManager) error {
+ return n.Out.StartBundle(ctx, id, data)
+}
+
+func (n *BenchRoot) Process(ctx context.Context) error {
+ for elm := range n.Elements {
+ if err := n.Out.ProcessElement(ctx, elm.Key, elm.Values...);
err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (n *BenchRoot) FinishBundle(ctx context.Context) error {
+ return n.Out.FinishBundle(ctx)
+}
+
+func (n *BenchRoot) Down(ctx context.Context) error {
+ return nil
+}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 118809)
Time Spent: 0.5h (was: 20m)
> Reduce ParDo per element Invoke overhead
> ----------------------------------------
>
> Key: BEAM-4726
> URL: https://issues.apache.org/jira/browse/BEAM-4726
> Project: Beam
> Issue Type: Sub-task
> Components: sdk-go
> Reporter: Robert Burke
> Assignee: Robert Burke
> Priority: Major
> Time Spent: 0.5h
> Remaining Estimate: 0h
>
> Each call to invoke allocates a new args []interface{}, but the common case
> is to run the same ProcessElement function over and again. It should be
> possible to have a container struct to retain the args slice, and avoid
> recomputing the indices for where to assign parameters before calling the
> ProcessElementFn.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
