This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-5789-d5133244344c2a309cfd586c22babab6e6694bf1 in repository https://gitbox.apache.org/repos/asf/texera.git
commit ba32e2bb8dbb7de72a3fb76729ce2e36247275f2 Author: EmilySun621 <[email protected]> AuthorDate: Fri Jun 19 14:20:10 2026 -0700 test(workflow-operator): add spec for TextGenCodegen (#5789) <!-- Thanks for sending a pull request (PR)! Here are some tips for you: 1. If this is your first time, please read our contributor guidelines: [Contributing to Texera](https://github.com/apache/texera/blob/main/CONTRIBUTING.md) 2. Ensure you have added or run the appropriate tests for your PR 3. If the PR is work in progress, mark it a draft on GitHub. 4. Please write your PR title to summarize what this PR proposes, we are following Conventional Commits style for PR titles as well. 5. Be sure to keep the PR description updated to reflect all changes. --> ### What changes were proposed in this PR? <!-- Please clarify what changes you are proposing. The purpose of this section is to outline the changes. Here are some tips for you: 1. If you propose a new API, clarify the use case for a new API. 2. If you fix a bug, you can clarify why it is a bug. 3. If it is a refactoring, clarify what has been changed. 3. It would be helpful to include a before-and-after comparison using screenshots or GIFs. 4. Please consider writing useful notes for better and faster reviews. --> Adds a unit test spec for `TextGenCodegen`. Tests cover: - `task` equals `"text-generation"` - `payloadPython(ctx)` opens with the `if task == "text-generation":` branch - `payloadPython(ctx)` emits the chat-completions shape (`messages`, `max_tokens`, `temperature`) - `payloadPython(ctx)` includes the `else` fallback with `{"inputs": prompt_value}` - `parsePython(ctx)` reads from `choices` / `message` / `content` - No raw-string leakage: snippets reference `self.*` attributes, never literal context values - Context-independence: two different `CodegenContext` inputs produce identical output ### Any related issues, documentation, discussions? <!-- Please use this section to link other resources if not mentioned already. 1. If this PR fixes an issue, please include `Fixes #1234`, `Resolves #1234` or `Closes #1234`. If it is only related, simply mention the issue number. 2. If there is design documentation, please add the link. 3. If there is a discussion in the mailing list, please add the link. --> Closes #5778 ### How was this PR tested? <!-- If tests were added, say they were added here. Or simply mention that if the PR is tested with existing test cases. Make sure to include/update test cases that check the changes thoroughly including negative and positive cases if possible. If it was tested in a way different from regular unit tests, please clarify how you tested step by step, ideally copy and paste-able, so that other reviewers can test and check, and descendants can verify in the future. If tests were not added, please describe why they were not added and/or why it was difficult to add. --> Spec verified with `sbt "WorkflowOperator/testOnly *TextGenCodegenSpec"`. 7 tests passing. ### Was this PR authored or co-authored using generative AI tooling? <!-- If generative AI tooling has been used in the process of authoring this PR, please include the phrase: 'Generated-by: ' followed by the name of the tool and its version. If no, write 'No'. Please refer to the [ASF Generative Tooling Guidance](https://www.apache.org/legal/generative-tooling.html) for details. --> Coauthored-by: Claude Code (Anthropic) --- .../huggingFace/codegen/TextGenCodegenSpec.scala | 133 +++++++++++++++++++++ 1 file changed, 133 insertions(+) diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/codegen/TextGenCodegenSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/codegen/TextGenCodegenSpec.scala new file mode 100644 index 0000000000..d7714af067 --- /dev/null +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/huggingFace/codegen/TextGenCodegenSpec.scala @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.operator.huggingFace.codegen + +import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +class TextGenCodegenSpec extends AnyFlatSpec with Matchers { + + private def makeCtx( + hfApiToken: EncodableString = "token", + modelId: EncodableString = "Qwen/Qwen2.5-72B-Instruct", + promptColumn: EncodableString = "prompt", + resultColumn: EncodableString = "hf_response", + task: EncodableString = "text-generation", + systemPrompt: EncodableString = "You are a helpful assistant.", + safeMaxTokens: Int = 256, + safeTemp: Double = 0.7 + ): CodegenContext = + CodegenContext( + hfApiToken = hfApiToken, + modelId = modelId, + promptColumn = promptColumn, + resultColumn = resultColumn, + task = task, + systemPrompt = systemPrompt, + safeMaxTokens = safeMaxTokens, + safeTemp = safeTemp + ) + + "TextGenCodegen.task" should "be the canonical text-generation string" in { + TextGenCodegen.task shouldBe "text-generation" + } + + "TextGenCodegen.payloadPython" should "open with the text-generation branch" in { + val out = TextGenCodegen.payloadPython(makeCtx()) + out should include("""if task == "text-generation":""") + } + + it should "emit the OpenAI chat-completions payload shape (messages, max_tokens, temperature)" in { + val out = TextGenCodegen.payloadPython(makeCtx()) + out should include("messages") + out should include("max_tokens") + out should include("temperature") + } + + it should "include the else fallback that ships the raw prompt as inputs" in { + val out = TextGenCodegen.payloadPython(makeCtx()) + out should include("""payload = {"inputs": prompt_value}""") + } + + "TextGenCodegen.parsePython" should "pull text out of choices[0].message.content" in { + val out = TextGenCodegen.parsePython(makeCtx()) + out should include("choices") + out should include("message") + out should include("content") + } + + "TextGenCodegen snippets" should "never inline raw CodegenContext string values" in { + // The snippets must reference self.* attributes — the base class decodes + // user-supplied strings safely at runtime. Sentinel values chosen to be + // distinctive and non-overlapping with the static template text. + val ctx = makeCtx( + hfApiToken = "MARKER_TOKEN_zXyq42", + modelId = "MARKER_MODEL_zXyq42", + promptColumn = "MARKER_PROMPT_zXyq42", + resultColumn = "MARKER_RESULT_zXyq42", + task = "MARKER_TASK_zXyq42", + systemPrompt = "MARKER_SYSTEM_zXyq42" + ) + val payload = TextGenCodegen.payloadPython(ctx) + val parse = TextGenCodegen.parsePython(ctx) + + payload should not include "MARKER_TOKEN_zXyq42" + payload should not include "MARKER_MODEL_zXyq42" + payload should not include "MARKER_PROMPT_zXyq42" + payload should not include "MARKER_RESULT_zXyq42" + payload should not include "MARKER_TASK_zXyq42" + payload should not include "MARKER_SYSTEM_zXyq42" + parse should not include "MARKER_TOKEN_zXyq42" + parse should not include "MARKER_MODEL_zXyq42" + parse should not include "MARKER_PROMPT_zXyq42" + parse should not include "MARKER_RESULT_zXyq42" + parse should not include "MARKER_TASK_zXyq42" + parse should not include "MARKER_SYSTEM_zXyq42" + } + + it should "produce identical output regardless of the CodegenContext contents" in { + // text-generation's payload/parse are static — they reference only + // self.* attributes, never ctx fields. Two unrelated contexts must + // serialise to byte-identical Python. A future refactor that + // accidentally consumes a ctx field will regress here. + val ctxA = makeCtx( + hfApiToken = "token-A", + modelId = "model-A", + promptColumn = "col-A", + resultColumn = "result-A", + systemPrompt = "system-A", + safeMaxTokens = 1, + safeTemp = 0.0 + ) + val ctxB = makeCtx( + hfApiToken = "token-B", + modelId = "model-B", + promptColumn = "col-B", + resultColumn = "result-B", + systemPrompt = "system-B", + safeMaxTokens = 4096, + safeTemp = 2.0 + ) + + TextGenCodegen.payloadPython(ctxA) shouldBe TextGenCodegen.payloadPython(ctxB) + TextGenCodegen.parsePython(ctxA) shouldBe TextGenCodegen.parsePython(ctxB) + } +}
