zyratlo commented on code in PR #8767: URL: https://github.com/apache/texera/pull/8767#discussion_r4159778541
########## frontend/src/app/workspace/service/notebook-migration/folder-assembly.ts: ########## @@ -0,0 +1,359 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { ScriptSegmentation, splitScriptLines } from "./script-segmentation"; + +/** + * Assembles a folder of Python sources into the one numbered document the migration LLM sees. + * + * Converted as a single input, so the model reports line ranges as it does for a script and + * nothing downstream learns folders exist; a banner line before each file carries provenance. + * Lines split through the segmenter's own splitScriptLines, so assembly, numbering and slicing + * agree on what a line is. + */ + +// Never project code, matched per segment so a cache or vendored dependency is caught at any +// depth. Hidden segments are covered separately by the leading-dot rule. +const EXCLUDED_SEGMENTS = new Set(["__pycache__", "venv", "site-packages", "node_modules"]); + +// Build output at the root, but ordinary package names deeper in a tree (src/build/ can be real +// code). Dropping real code silently is worse than letting a build tree hit the caps. +const EXCLUDED_ROOT_DIRECTORIES = new Set(["build", "dist"]); + +// Non-Python files are named in the layout, never read. Capped because a folder can hold +// thousands of data files and the layout is orientation, not an inventory. +export const MAX_LISTED_OTHER_FILES = 20; + +/** + * Caps on one conversion, refused before any request is sent. Sized by the reply, not the prompt: + * the conversion asks for every line of code back and sets no output budget, so the model's output + * ceiling binds long before its context does. Raise once a larger folder has been measured. + */ +export const MAX_FOLDER_FILES = 100; +export const MAX_FOLDER_CHARACTERS = 60_000; Review Comment: Fixed in b52beddcb. sendPrompt now passes an explicit 48,000 token budget, and a `finishReason: "length"` reply throws instead of reaching the parser, so an overrun says the reply was cut off rather than that the JSON was malformed. 48,000 is still reasoning rather than measurement, which is why the truncation check matters. Note it sits in sendPrompt, so the notebook and script paths get the budget too; they had the same latent default. ########## frontend/src/app/workspace/service/notebook-migration/folder-assembly.ts: ########## @@ -0,0 +1,359 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { ScriptSegmentation, splitScriptLines } from "./script-segmentation"; + +/** + * Assembles a folder of Python sources into the one numbered document the migration LLM sees. + * + * Converted as a single input, so the model reports line ranges as it does for a script and + * nothing downstream learns folders exist; a banner line before each file carries provenance. + * Lines split through the segmenter's own splitScriptLines, so assembly, numbering and slicing + * agree on what a line is. + */ + +// Never project code, matched per segment so a cache or vendored dependency is caught at any +// depth. Hidden segments are covered separately by the leading-dot rule. +const EXCLUDED_SEGMENTS = new Set(["__pycache__", "venv", "site-packages", "node_modules"]); + +// Build output at the root, but ordinary package names deeper in a tree (src/build/ can be real +// code). Dropping real code silently is worse than letting a build tree hit the caps. +const EXCLUDED_ROOT_DIRECTORIES = new Set(["build", "dist"]); + +// Non-Python files are named in the layout, never read. Capped because a folder can hold +// thousands of data files and the layout is orientation, not an inventory. +export const MAX_LISTED_OTHER_FILES = 20; + +/** + * Caps on one conversion, refused before any request is sent. Sized by the reply, not the prompt: + * the conversion asks for every line of code back and sets no output budget, so the model's output + * ceiling binds long before its context does. Raise once a larger folder has been measured. + */ +export const MAX_FOLDER_FILES = 100; +export const MAX_FOLDER_CHARACTERS = 60_000; + +// Pre-read guard for the character cap: UTF-8 never uses fewer bytes than characters, so a +// selection this far over cannot come in under it, and one huge file would otherwise hang the tab +// before the character cap, which needs the files read, could run. +export const MAX_FOLDER_BYTES = MAX_FOLDER_CHARACTERS * 4; Review Comment: Fixed in c2f01e578. Now reads "UTF-8 uses at most four bytes per character", which is the direction that makes bytes > 4 * CAP imply chars > CAP. The bound is unchanged. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
