This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris-website.git
The following commit(s) were added to refs/heads/master by this push:
new 119862e0da3 [fix] Truncate large query profiles after MergedProfile
(#4034)
119862e0da3 is described below
commit 119862e0da34fc3c97f9d58dd469a29ed8b1d640
Author: Mairui Li <[email protected]>
AuthorDate: Wed Aug 5 10:46:53 2026 +0800
[fix] Truncate large query profiles after MergedProfile (#4034)
## Summary
- accept local Query Profile files up to 100 MiB
- reduce files over 10 MiB before upload by retaining Summary and
MergedProfile
- remove per-instance DetailProfile/Execution Profile sections
- keep the final upload within the existing 10 MiB backend limit
- prevent stale asynchronous file preparation from replacing a newer
selection
- add focused tests for truncation boundaries, UTF-8 byte size, and file
limits
## Why
Large Apache Doris Query Profiles can exceed the backend's 10 MiB upload
limit because they contain extensive per-instance DetailProfile data.
Summary and MergedProfile retain the aggregated execution metrics needed
for the initial diagnosis. Removing the following per-instance sections
allows large Profiles to use the existing analysis API without
increasing the backend request limit.
Truncation is only performed when a MergedProfile header is followed by
a recognized per-instance profile header. Files that cannot be reduced
safely to 10 MiB are rejected.
## Validation
- `node --test src/components/profile-analysis/*.test.js`
- 51 tests passed
- targeted TypeScript compilation passed
- verified with a 41,070,035-byte real Profile
- output: 123,362 bytes
- MergedProfile retained
- DetailProfile removed
---
.../profile-analysis/ProfileUploader.tsx | 42 +++++--
.../profile-analysis/profile-analysis.api.ts | 3 +-
.../profile-analysis.components.test.js | 23 +++-
.../profile-analysis/profile-analysis.file.test.js | 132 +++++++++++++++++++++
.../profile-analysis/profile-analysis.file.ts | 44 +++++++
5 files changed, 230 insertions(+), 14 deletions(-)
diff --git a/src/components/profile-analysis/ProfileUploader.tsx
b/src/components/profile-analysis/ProfileUploader.tsx
index eef4cf89e87..9b3bb5fefb4 100644
--- a/src/components/profile-analysis/ProfileUploader.tsx
+++ b/src/components/profile-analysis/ProfileUploader.tsx
@@ -1,9 +1,8 @@
import HCaptcha from '@hcaptcha/react-hcaptcha';
import React, { ChangeEvent, DragEvent, JSX, useCallback, useRef, useState }
from 'react';
+import { MAX_RAW_BYTES, prepareProfileFile } from './profile-analysis.file';
import type { ResponseLanguage } from './profile-analysis.types';
-export const MAX_PROFILE_FILE_SIZE_BYTES = 10 * 1024 * 1024;
-
interface ProfileUploaderProps {
file: File | null;
language: ResponseLanguage;
@@ -18,8 +17,8 @@ export function validateProfileFile(file: File): string |
null {
if (!file.name.toLowerCase().endsWith('.txt')) {
return 'Select a .txt file. Other file types are not supported.';
}
- if (file.size > MAX_PROFILE_FILE_SIZE_BYTES) {
- return 'The selected file is larger than 10 MiB.';
+ if (file.size > MAX_RAW_BYTES) {
+ return 'The selected file is larger than 100 MiB. Please provide a
merged Profile.';
}
return null;
}
@@ -48,6 +47,7 @@ export function ProfileUploader({
const [hcaptchaToken, setHCaptchaToken] = useState<string | null>(null);
const [hcaptchaError, setHCaptchaError] = useState<string | null>(null);
const hcaptchaRef = useRef<HCaptcha>(null);
+ const filePreparationIdRef = useRef(0);
const resetCaptcha = useCallback(() => {
hcaptchaRef.current?.resetCaptcha();
@@ -55,7 +55,8 @@ export function ProfileUploader({
setHCaptchaError(null);
}, []);
- const acceptFile = (nextFile: File | null) => {
+ const acceptFile = async (nextFile: File | null) => {
+ const preparationId = ++filePreparationIdRef.current;
if (!nextFile) {
setValidationError(null);
onFileChange(null);
@@ -63,12 +64,31 @@ export function ProfileUploader({
}
const nextError = validateProfileFile(nextFile);
- setValidationError(nextError);
- onFileChange(nextError ? null : nextFile);
+ if (nextError) {
+ setValidationError(nextError);
+ onFileChange(null);
+ return;
+ }
+
+ setValidationError(null);
+ onFileChange(null);
+ try {
+ const preparedFile = await prepareProfileFile(nextFile);
+ if (filePreparationIdRef.current !== preparationId) return;
+ onFileChange(preparedFile);
+ } catch (reason) {
+ if (filePreparationIdRef.current !== preparationId) return;
+ setValidationError(
+ reason instanceof Error
+ ? reason.message
+ : 'The Profile could not be prepared for upload.',
+ );
+ onFileChange(null);
+ }
};
const handleInputChange = (event: ChangeEvent<HTMLInputElement>) => {
- acceptFile(event.currentTarget.files?.item(0) ?? null);
+ void acceptFile(event.currentTarget.files?.item(0) ?? null);
event.currentTarget.value = '';
};
@@ -82,14 +102,15 @@ export function ProfileUploader({
onFileChange(null);
return;
}
- acceptFile(event.dataTransfer.files.item(0));
+ void acceptFile(event.dataTransfer.files.item(0));
};
return (
<section className="profile-analysis__uploader"
aria-labelledby="profile-analysis-upload-title">
<h2 id="profile-analysis-upload-title">Upload a Query Profile</h2>
<p id="profile-analysis-file-help"
className="profile-analysis__help">
- Choose one UTF-8 .txt file up to 10 MiB after reviewing and
accepting the notice below.
+ Choose one UTF-8 .txt file up to 100 MiB after reviewing and
accepting the notice below. Files
+ over 10 MiB are reduced to their aggregated Profile sections
before upload.
</p>
<fieldset className="profile-analysis__language"
disabled={disabled}>
@@ -254,6 +275,7 @@ export function ProfileUploader({
const accepted = event.currentTarget.checked;
setConsentAccepted(accepted);
if (!accepted) {
+ filePreparationIdRef.current += 1;
resetCaptcha();
onFileChange(null);
}
diff --git a/src/components/profile-analysis/profile-analysis.api.ts
b/src/components/profile-analysis/profile-analysis.api.ts
index bf6eb7b3913..cbc3619f591 100644
--- a/src/components/profile-analysis/profile-analysis.api.ts
+++ b/src/components/profile-analysis/profile-analysis.api.ts
@@ -8,6 +8,7 @@ import type {
ResponseLanguage,
} from './profile-analysis.types';
import { isUuid } from './profile-analysis.storage';
+import { prepareProfileFile } from './profile-analysis.file';
const ANALYSIS_JOBS_PATH = '/api/profile/analysis-jobs';
const ANALYSIS_JOB_REQUESTS_PATH = '/api/profile/analysis-job-requests';
@@ -165,7 +166,7 @@ export async function createAnalysisJob(
}
const formData = new FormData();
- formData.append('file', file);
+ formData.append('file', await prepareProfileFile(file));
formData.append('language', language);
formData.append('consent', 'true');
formData.append('privacyNoticeVersion', PRIVACY_NOTICE_VERSION);
diff --git
a/src/components/profile-analysis/profile-analysis.components.test.js
b/src/components/profile-analysis/profile-analysis.components.test.js
index 9e38a6d9438..02b43f5cf34 100644
--- a/src/components/profile-analysis/profile-analysis.components.test.js
+++ b/src/components/profile-analysis/profile-analysis.components.test.js
@@ -27,11 +27,11 @@ require.extensions['.tsx'] = compileTypeScript;
const { AnalysisResult } = require('./AnalysisResult.tsx');
const { AnalysisStatus } = require('./AnalysisStatus.tsx');
const {
- MAX_PROFILE_FILE_SIZE_BYTES,
ProfileUploader,
formatProfileFileSize,
validateProfileFile,
} = require('./ProfileUploader.tsx');
+const { MAX_RAW_BYTES } = require('./profile-analysis.file.ts');
require.extensions['.ts'] = previousTypeScriptLoader;
require.extensions['.tsx'] = previousTsxLoader;
@@ -42,9 +42,9 @@ test('accepts case-insensitive txt files and rejects other
types or oversized fi
const oversizedFile = {
name: 'query.txt',
- size: MAX_PROFILE_FILE_SIZE_BYTES + 1,
+ size: MAX_RAW_BYTES + 1,
};
- assert.match(validateProfileFile(oversizedFile), /10 MiB/);
+ assert.match(validateProfileFile(oversizedFile), /100 MiB/);
});
test('formats file sizes for display', () => {
@@ -53,6 +53,23 @@ test('formats file sizes for display', () => {
assert.equal(formatProfileFileSize(2 * 1024 * 1024), '2.0 MiB');
});
+test('explains the raw file limit and large-profile reduction in English', ()
=> {
+ const markup = renderToStaticMarkup(
+ React.createElement(ProfileUploader, {
+ file: null,
+ language: 'en',
+ disabled: false,
+ hcaptchaSiteKey,
+ onFileChange() {},
+ onLanguageChange() {},
+ onAnalyze() {},
+ }),
+ );
+
+ assert.match(markup, /UTF-8 \.txt file up to 100 MiB/);
+ assert.match(markup, /Files over 10 MiB are reduced to their aggregated
Profile sections/);
+});
+
test('disables Analyze until a file exists and while analysis is running', ()
=> {
const withoutFile = renderToStaticMarkup(
React.createElement(ProfileUploader, {
diff --git a/src/components/profile-analysis/profile-analysis.file.test.js
b/src/components/profile-analysis/profile-analysis.file.test.js
new file mode 100644
index 00000000000..0fb24fc0217
--- /dev/null
+++ b/src/components/profile-analysis/profile-analysis.file.test.js
@@ -0,0 +1,132 @@
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const path = require('node:path');
+const test = require('node:test');
+const { File } = require('node:buffer');
+const typescript = require('typescript');
+
+const fileModulePath = path.join(__dirname, 'profile-analysis.file.ts');
+const compiledFileModule =
typescript.transpileModule(fs.readFileSync(fileModulePath, 'utf8'), {
+ compilerOptions: {
+ module: typescript.ModuleKind.CommonJS,
+ target: typescript.ScriptTarget.ES2020,
+ },
+}).outputText;
+const fileModule = { exports: {} };
+new Function('module', 'exports', 'require', '__filename', '__dirname',
compiledFileModule)(
+ fileModule,
+ fileModule.exports,
+ require,
+ fileModulePath,
+ path.dirname(fileModulePath),
+);
+
+const {
+ byteLen,
+ ensureTxt,
+ MAX_RAW_BYTES,
+ MAX_UPLOAD_BYTES,
+ prepareProfileFile,
+ truncateProfile,
+} = fileModule.exports;
+
+test('counts profile text in UTF-8 bytes instead of JavaScript characters', ()
=> {
+ assert.equal('é中'.length, 2);
+ assert.equal(byteLen('é中'), 5);
+});
+
+test('ensures prepared profile names have a txt extension', () => {
+ assert.equal(ensureTxt('query-profile'), 'query-profile.txt');
+ assert.equal(ensureTxt('query-profile.TXT'), 'query-profile.TXT');
+});
+
+test('truncates at the first per-instance header after MergedProfile', () => {
+ const source = [
+ 'Summary:',
+ 'Execution Profile summary text that is not a top-level detail header',
+ 'Execution Summary:',
+ 'MergedProfile:',
+ ' Fragment 0:',
+ ' Pipeline 0:',
+ 'DetailProfile(query-id-1):',
+ ' Instance detail that must not be uploaded',
+ 'Execution Profile query-id-2',
+ ' More instance detail',
+ ].join('\n');
+
+ const truncated = truncateProfile(source);
+
+ assert.ok(truncated.startsWith('# [truncated: per-instance execution
profiles removed]\n'));
+ assert.match(truncated, /Summary:/);
+ assert.match(truncated, /Execution Summary:/);
+ assert.match(truncated, /MergedProfile:/);
+ assert.match(truncated, /Fragment 0:/);
+ assert.match(truncated, /Pipeline 0:/);
+ assert.doesNotMatch(truncated, /^DetailProfile\b/m);
+ assert.doesNotMatch(truncated, /Instance detail/);
+});
+
+test('does not claim truncation without a MergedProfile followed by a detail
boundary', () => {
+ const mergedOnly = 'Summary:\nExecution Summary:\nMergedProfile:\n
Fragment 0:\n Pipeline 0:';
+ const detailOnly = 'Summary:\nDetailProfile(query-id-1):\n Instance
detail';
+ assert.equal(truncateProfile(mergedOnly), mergedOnly);
+ assert.equal(truncateProfile(detailOnly), detailOnly);
+});
+
+test('returns an upload-sized profile file without reading or repackaging it',
async () => {
+ const file = new File(['Summary:\nMergedProfile:'], 'query-profile.txt', {
type: 'text/plain' });
+ assert.equal(await prepareProfileFile(file), file);
+});
+
+test('rejects a raw profile over 100 MiB without reading it', async () => {
+ let textCalls = 0;
+ const file = {
+ name: 'huge-profile.txt',
+ size: MAX_RAW_BYTES + 1,
+ async text() {
+ textCalls += 1;
+ return 'must not be read';
+ },
+ };
+
+ await assert.rejects(prepareProfileFile(file), /larger than 100
MiB.*merged Profile/);
+ assert.equal(textCalls, 0);
+});
+
+test('removes per-instance detail from a large profile and returns a plain txt
file', async () => {
+ const aggregate = 'Summary:\nExecution Summary:\nMergedProfile:\n
Fragment 0:\n Pipeline 0:';
+ const source =
`${aggregate}\nDetailProfile(query-id-1):\n${'x'.repeat(MAX_UPLOAD_BYTES)}`;
+ const file = {
+ name: 'query-profile',
+ size: MAX_UPLOAD_BYTES + 1,
+ async text() {
+ return source;
+ },
+ };
+
+ const prepared = await prepareProfileFile(file);
+
+ assert.equal(prepared.type, 'text/plain');
+ assert.equal(prepared.name, 'query-profile.txt');
+ assert.ok(prepared.size <= MAX_UPLOAD_BYTES);
+ const preparedText = await prepared.text();
+ assert.match(preparedText, /^# \[truncated: per-instance execution
profiles removed\]/);
+ assert.match(preparedText, /MergedProfile:/);
+ assert.doesNotMatch(preparedText, /^DetailProfile\b/m);
+});
+
+test('rejects a merged-only profile that remains over the upload limit', async
() => {
+ const source = `Summary:\nMergedProfile:\n${'x'.repeat(MAX_UPLOAD_BYTES)}`;
+ const file = {
+ name: 'merged-profile.txt',
+ size: MAX_UPLOAD_BYTES + 1,
+ async text() {
+ return source;
+ },
+ };
+
+ await assert.rejects(
+ prepareProfileFile(file),
+ /still larger than 10 MiB.*merged Profile/,
+ );
+});
diff --git a/src/components/profile-analysis/profile-analysis.file.ts
b/src/components/profile-analysis/profile-analysis.file.ts
new file mode 100644
index 00000000000..bbad9bd1d5d
--- /dev/null
+++ b/src/components/profile-analysis/profile-analysis.file.ts
@@ -0,0 +1,44 @@
+export const MAX_UPLOAD_BYTES = 10 * 1024 * 1024;
+export const MAX_RAW_BYTES = 100 * 1024 * 1024;
+
+const TRUNCATION_MARKER = '# [truncated: per-instance execution profiles
removed]';
+const MERGED_PROFILE_HEADER = /^MergedProfile:\s*$/m;
+const DETAIL_PROFILE_HEADER = /^(?:DetailProfile(?:\([^\r\n]*\))?|Execution
Profile\b[^\r\n]*):?\s*$/gm;
+
+export function byteLen(value: string): number {
+ return new Blob([value]).size;
+}
+
+export function ensureTxt(name: string): string {
+ return name.toLowerCase().endsWith('.txt') ? name : `${name}.txt`;
+}
+
+export function truncateProfile(text: string): string {
+ const mergedProfile = MERGED_PROFILE_HEADER.exec(text);
+ if (!mergedProfile) return text;
+
+ DETAIL_PROFILE_HEADER.lastIndex = mergedProfile.index +
mergedProfile[0].length;
+ const detailProfile = DETAIL_PROFILE_HEADER.exec(text);
+ DETAIL_PROFILE_HEADER.lastIndex = 0;
+ if (!detailProfile) return text;
+
+ const retainedText = text.slice(0, detailProfile.index).trimEnd();
+ return `${TRUNCATION_MARKER}\n${retainedText}\n`;
+}
+
+export async function prepareProfileFile(file: File): Promise<File> {
+ if (file.size <= MAX_UPLOAD_BYTES) {
+ return file;
+ }
+ if (file.size > MAX_RAW_BYTES) {
+ throw new Error('The selected file is larger than 100 MiB. Please
provide a merged Profile.');
+ }
+
+ const output = truncateProfile(await file.text());
+ if (byteLen(output) > MAX_UPLOAD_BYTES) {
+ throw new Error(
+ 'The Profile is still larger than 10 MiB after removing
per-instance execution profiles. Please provide a merged Profile.',
+ );
+ }
+ return new File([output], ensureTxt(file.name), { type: 'text/plain' });
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
