damccorm commented on code in PR #26861: URL: https://github.com/apache/beam/pull/26861#discussion_r1240212830
########## learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/description.md: ########## @@ -0,0 +1,25 @@ +<!-- +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> +### Final challenge 1 + +You’re given a csv file with purchase transactions. Write a Beam pipeline to prepare a send report every 30 seconds. The report needs to be created only for transactions where quantity is more than 20. Review Comment: ```suggestion You’re given a csv file with purchase transactions. Write a Beam pipeline to prepare a report every 30 seconds. The report needs to be created only for transactions where quantity is more than 20. ``` ########## learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/description.md: ########## @@ -0,0 +1,20 @@ +<!-- +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> +### Final challenge 2 + +You are given a file analyzed.csv which contain analyzed words. Therefore, you need to analyze the shekspir/kinglear file. How many **negative words** and **positive words** , and negative words, positive words with strong or weak modal? Review Comment: ```suggestion You are given a file analyzed.csv which maps words to sentiments. Using this, analyze kinglear.txt. Output PCollections counting the number of **negative words** and **positive words** as well as PCollections counting the number of **positive words with strong modal** and **positive words with weak modal** ``` ########## learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/go-solution/main.go: ########## @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// beam-playground: +// name: FinalSolution2 +// description: Final challenge solution 2. +// multifile: true +// files: +// - name: analysis.csv +// context_line: 54 +// categories: +// - Quickstart +// complexity: ADVANCED +// tags: +// - hellobeam + +package main + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/window" + "github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/window/trigger" + "github.com/apache/beam/sdks/v2/go/pkg/beam" + "github.com/apache/beam/sdks/v2/go/pkg/beam/io/textio" + "github.com/apache/beam/sdks/v2/go/pkg/beam/log" + "github.com/apache/beam/sdks/v2/go/pkg/beam/transforms/filter" + "github.com/apache/beam/sdks/v2/go/pkg/beam/transforms/stats" + "github.com/apache/beam/sdks/v2/go/pkg/beam/x/beamx" + "github.com/apache/beam/sdks/v2/go/pkg/beam/x/debug" +) + +type Analysis struct { + Word string + Negative string + Positive string + Uncertainty string + Litigious string + Strong string + Weak string + Constraining string +} + +func (a Analysis) toString() string { + return fmt.Sprintf("Word: %s, Negative: %s, Positive: %s, Uncertainty: %s, Litigious: %s, Strong: %s, Weak: %s, Constraining: %s", + a.Word, a.Negative, a.Positive, a.Uncertainty, a.Litigious, a.Strong, a.Weak, a.Constraining) +} + +func main() { + + ctx := context.Background() + + beam.Init() + + p := beam.NewPipeline() + s := p.Root() + + shakespeare := textio.Read(s, "gs://apache-beam-samples/shakespeare/kinglear.txt") + shakespeareWords := getWords(s, shakespeare) + analysis := textio.Read(s, "analysis.csv") + analysisRecords := parseAnalysis(s, analysis) + + trigger := trigger.AfterEndOfWindow(). + EarlyFiring(trigger.AfterProcessingTime(). + PlusDelay(5 * time.Second)) + + fixedWindowedItems := beam.WindowInto(s, window.NewFixedWindows(30*time.Second), shakespeareWords, + beam.Trigger(trigger), + beam.AllowedLateness(30*time.Second), + beam.PanesDiscard(), + ) Review Comment: Why do we need windowing/triggering? ########## learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/description.md: ########## @@ -0,0 +1,25 @@ +<!-- +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> +### Final challenge 1 + +You’re given a csv file with purchase transactions. Write a Beam pipeline to prepare a send report every 30 seconds. The report needs to be created only for transactions where quantity is more than 20. + +Report should consist of two files named "**price more than 10**" and "**price less than 10**": Review Comment: ```suggestion Report should consist of two files named "**price_more_than_10.txt**" and "**price_less_than_10.txt**": ``` ########## learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/description.md: ########## @@ -0,0 +1,25 @@ +<!-- +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> +### Final challenge 1 + +You’re given a csv file with purchase transactions. Write a Beam pipeline to prepare a send report every 30 seconds. The report needs to be created only for transactions where quantity is more than 20. + +Report should consist of two files named "**price more than 10**" and "**price less than 10**": + +* Total transactions amount grouped by **ProductNo** for products with **price** greater than 10 +* Total transactions amount grouped by **ProductNo** for products with **price** less than 10 + +| TransactionNo | Date | ProductNo | ProductName | Price | Quantity | CustomerNo | Country | Review Comment: Could we add a description like: Example rows from input file ########## learning/tour-of-beam/learning-content/final-challenge/final-challenge-2/go-solution/main.go: ########## @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// beam-playground: +// name: FinalSolution2 +// description: Final challenge solution 2. +// multifile: true +// files: +// - name: analysis.csv +// context_line: 54 +// categories: +// - Quickstart +// complexity: ADVANCED +// tags: +// - hellobeam + +package main + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/window" + "github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/window/trigger" + "github.com/apache/beam/sdks/v2/go/pkg/beam" + "github.com/apache/beam/sdks/v2/go/pkg/beam/io/textio" + "github.com/apache/beam/sdks/v2/go/pkg/beam/log" + "github.com/apache/beam/sdks/v2/go/pkg/beam/transforms/filter" + "github.com/apache/beam/sdks/v2/go/pkg/beam/transforms/stats" + "github.com/apache/beam/sdks/v2/go/pkg/beam/x/beamx" + "github.com/apache/beam/sdks/v2/go/pkg/beam/x/debug" +) + +type Analysis struct { + Word string + Negative string + Positive string + Uncertainty string + Litigious string + Strong string + Weak string + Constraining string +} + +func (a Analysis) toString() string { + return fmt.Sprintf("Word: %s, Negative: %s, Positive: %s, Uncertainty: %s, Litigious: %s, Strong: %s, Weak: %s, Constraining: %s", + a.Word, a.Negative, a.Positive, a.Uncertainty, a.Litigious, a.Strong, a.Weak, a.Constraining) +} + +func main() { + + ctx := context.Background() + + beam.Init() + + p := beam.NewPipeline() + s := p.Root() + + shakespeare := textio.Read(s, "gs://apache-beam-samples/shakespeare/kinglear.txt") + shakespeareWords := getWords(s, shakespeare) + analysis := textio.Read(s, "analysis.csv") + analysisRecords := parseAnalysis(s, analysis) + + trigger := trigger.AfterEndOfWindow(). + EarlyFiring(trigger.AfterProcessingTime(). + PlusDelay(5 * time.Second)) + + fixedWindowedItems := beam.WindowInto(s, window.NewFixedWindows(30*time.Second), shakespeareWords, + beam.Trigger(trigger), + beam.AllowedLateness(30*time.Second), + beam.PanesDiscard(), + ) Review Comment: Same comment applies to other solutions ########## learning/tour-of-beam/learning-content/final-challenge/final-challenge-1/go-challenge/main.go: ########## @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// beam-playground: +// name: FinalChallenge1 +// description: Final challenge 1. +// multifile: true +// files: +// - name: input.csv +// context_line: 54 +// categories: +// - Quickstart +// complexity: ADVANCED +// tags: +// - hellobeam + +package main + +import ( + "context" + "github.com/apache/beam/sdks/v2/go/pkg/beam" + "github.com/apache/beam/sdks/v2/go/pkg/beam/io/textio" + "github.com/apache/beam/sdks/v2/go/pkg/beam/x/beamx" + "log" +) + +type Transaction struct { + ID int64 + Date string + ProductID string + ProductName string + Price float64 + Quantity int64 + CustomerID int64 + Country string +} + +func main() { + beam.Init() + p := beam.NewPipeline() + s := p.Root() + + file := textio.Read(s, "input.csv") + + textio.Write(s, "smallerThan10.txt", file) Review Comment: ```suggestion textio.Write(s, "price_less_than_10.txt", file) ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
