[
https://issues.apache.org/jira/browse/BEAM-9642?focusedWorklogId=425997&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-425997
]
ASF GitHub Bot logged work on BEAM-9642:
----------------------------------------
Author: ASF GitHub Bot
Created on: 22/Apr/20 02:38
Start Date: 22/Apr/20 02:38
Worklog Time Spent: 10m
Work Description: youngoli commented on a change in pull request #11474:
URL: https://github.com/apache/beam/pull/11474#discussion_r412590798
##########
File path: sdks/go/pkg/beam/core/runtime/exec/sdf.go
##########
@@ -263,20 +266,33 @@ func (n *ProcessSizedElementsAndRestrictions)
ProcessElement(ctx context.Context
return errors.Errorf("invalid status for ParDo %v: %v, want
Active", n.PDo.UID, n.PDo.status)
}
- userElm := elm.Elm.(*FullValue).Elm.(*FullValue)
rest := elm.Elm.(*FullValue).Elm2
rt := n.inv.Invoke(rest)
+ var mainIn = &MainInput{
Review comment:
Whoops, that was left behind in editing. Fixed
##########
File path: sdks/go/pkg/beam/core/runtime/exec/sdf.go
##########
@@ -263,20 +266,33 @@ func (n *ProcessSizedElementsAndRestrictions)
ProcessElement(ctx context.Context
return errors.Errorf("invalid status for ParDo %v: %v, want
Active", n.PDo.UID, n.PDo.status)
}
- userElm := elm.Elm.(*FullValue).Elm.(*FullValue)
rest := elm.Elm.(*FullValue).Elm2
rt := n.inv.Invoke(rest)
+ var mainIn = &MainInput{
+ Values: values,
+ RTracker: rt,
+ }
- return n.PDo.processMainInput(&MainInput{
- Key: FullValue{ // User userElm's values but the top-level
windows and timestamp.
+ // For the key, the way we fill it out depends on whether the input
element
+ // is a KV or single-element. Single-elements might have been lifted
out of
+ // their FullValue if they were decoded, so we need to have a case for
that.
+ // Also, we use the the top-level windows and timestamp.
+ if userElm, ok := elm.Elm.(*FullValue).Elm.(*FullValue); ok {
Review comment:
Ack. Added a TODO to remind me.
##########
File path: sdks/go/pkg/beam/core/runtime/exec/translate.go
##########
@@ -372,32 +386,43 @@ func (b *builder) makeLink(from string, id linkID) (Node,
error) {
switch op {
case graph.ParDo:
- n := &ParDo{UID: b.idgen.New(), Inbound: in,
Out: out}
- n.Fn, err = graph.AsDoFn(fn, graph.MainUnknown)
+ dofn, err := graph.AsDoFn(fn, graph.MainUnknown)
if err != nil {
return nil, err
}
- n.PID = transform.GetUniqueName()
-
- input :=
unmarshalKeyedValues(transform.GetInputs())
- for i := 1; i < len(input); i++ {
- // TODO(herohde) 8/8/2018: handle
different windows, view_fn and window_mapping_fn.
- // For now, assume we don't need any
information in the pardo payload.
-
- ec, wc, err :=
b.makeCoderForPCollection(input[i])
- if err != nil {
- return nil, err
+ switch urn {
+ case urnPairWithRestriction:
+ u = &PairWithRestriction{UID:
b.idgen.New(), Fn: dofn, Out: out[0]}
+ case urnSplitAndSizeRestrictions:
+ u = &SplitAndSizeRestrictions{UID:
b.idgen.New(), Fn: dofn, Out: out[0]}
+ default:
+ n := &ParDo{UID: b.idgen.New(), Fn:
dofn, Inbound: in, Out: out}
+ n.PID = transform.GetUniqueName()
+
+ input :=
unmarshalKeyedValues(transform.GetInputs())
+ for i := 1; i < len(input); i++ {
+ // TODO(herohde) 8/8/2018:
handle different windows, view_fn and window_mapping_fn.
Review comment:
Agreed. I was iffy on it myself since even 5 indentations deep was
already getting difficult to follow.
##########
File path: sdks/go/pkg/beam/core/runtime/exec/translate.go
##########
@@ -372,32 +386,43 @@ func (b *builder) makeLink(from string, id linkID) (Node,
error) {
switch op {
case graph.ParDo:
- n := &ParDo{UID: b.idgen.New(), Inbound: in,
Out: out}
- n.Fn, err = graph.AsDoFn(fn, graph.MainUnknown)
+ dofn, err := graph.AsDoFn(fn, graph.MainUnknown)
if err != nil {
return nil, err
}
- n.PID = transform.GetUniqueName()
-
- input :=
unmarshalKeyedValues(transform.GetInputs())
- for i := 1; i < len(input); i++ {
- // TODO(herohde) 8/8/2018: handle
different windows, view_fn and window_mapping_fn.
- // For now, assume we don't need any
information in the pardo payload.
-
- ec, wc, err :=
b.makeCoderForPCollection(input[i])
- if err != nil {
- return nil, err
+ switch urn {
+ case urnPairWithRestriction:
+ u = &PairWithRestriction{UID:
b.idgen.New(), Fn: dofn, Out: out[0]}
+ case urnSplitAndSizeRestrictions:
+ u = &SplitAndSizeRestrictions{UID:
b.idgen.New(), Fn: dofn, Out: out[0]}
+ default:
+ n := &ParDo{UID: b.idgen.New(), Fn:
dofn, Inbound: in, Out: out}
+ n.PID = transform.GetUniqueName()
+
+ input :=
unmarshalKeyedValues(transform.GetInputs())
+ for i := 1; i < len(input); i++ {
+ // TODO(herohde) 8/8/2018:
handle different windows, view_fn and window_mapping_fn.
+ // For now, assume we don't
need any information in the pardo payload.
+
+ ec, wc, err :=
b.makeCoderForPCollection(input[i])
+ if err != nil {
+ return nil, err
+ }
+
+ sid := StreamID{
+ Port: Port{URL:
b.desc.GetStateApiServiceDescriptor().GetUrl()},
+ PtransformID: id.to,
+ }
+ sideInputID :=
fmt.Sprintf("i%v", i) // SideInputID (= local id, "iN")
+ side :=
NewSideInputAdapter(sid, sideInputID, coder.NewW(ec, wc))
+ n.Side = append(n.Side, side)
}
-
- sid := StreamID{
- Port: Port{URL:
b.desc.GetStateApiServiceDescriptor().GetUrl()},
- PtransformID: id.to,
+ if urn ==
urnProcessSizedElementsAndRestrictions {
+ u =
&ProcessSizedElementsAndRestrictions{PDo: n}
+ } else {
+ u = n
}
Review comment:
Done.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 425997)
Time Spent: 6h (was: 5h 50m)
> Add SDF execution-time runners
> ------------------------------
>
> Key: BEAM-9642
> URL: https://issues.apache.org/jira/browse/BEAM-9642
> Project: Beam
> Issue Type: Sub-task
> Components: sdk-go
> Reporter: Daniel Oliveira
> Assignee: Daniel Oliveira
> Priority: Major
> Time Spent: 6h
> Remaining Estimate: 0h
>
> Adds execution-time SDF runner units to the exec package, and any unit tests
> + helpers required.
> This is needed to get the expanded SDF URNs to execute in the runner harness.
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
