This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris-website.git
The following commit(s) were added to refs/heads/master by this push:
new f8f3bcc26a3 fix(i18n): support page state shards in translation
updates (#4329)
f8f3bcc26a3 is described below
commit f8f3bcc26a30a6de16c701f2d8fe68e14f4660f7
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Fri Oct 16 14:44:36 2026 +0800
fix(i18n): support page state shards in translation updates (#4329)
Existing Japanese translations stored in page state files are rejected
as untracked because the translation engine reads only the legacy
ledger. Merge the same effective state as the site, honor human
ownership, and write each candidate to its page state file when shards
are enabled. Legacy repositories keep their current ledger format.
Generation now records the exact state file in each candidate manifest.
Publishing validates that path against the candidate identity and stages
only the matching translation and state. Snapshot checks preserve
concurrent edits; a failed batch write restores earlier translations and
state files.
Validation:
- Node 20.20.2: all 103 i18n tests pass, including 10 new regression
cases for existing shards, legacy migration, ownership, concurrent
changes, rollback and publication paths.
- All six current stale pages prepare successfully with offline fixture
translations and retain their protected-content signatures; no source,
translation or state file is applied.
- Full local bare-remote simulation of the actual bitmap-union-int
update passes: an invalid response exits 1; a valid fixture generates
one candidate, passes the real translation gate, and pushes only the
Japanese page plus its page state file before recording a mocked draft
PR.
- Dev build of en, zh-CN and ja passes with Node 20.20.2. Japanese
static verification passes for 1,778 pages, with 0 skipped. Existing
technical-stale pages retain their English fallback.
No live model call or GitHub write-credential validation is claimed. The
repaired workflow must be merged to trusted master before the one-page
live validation is rerun. Automatic generation remains disabled.
## Versions
- [x] dev
- [x] 4.x
- [ ] 3.x
- [ ] 2.1 or older (not covered by version/language sync gate)
The managed translation scope remains Dev, 4.x and community. No product
documentation, route, sidebar or locale scope changes.
## Languages
- [ ] Chinese
- [x] English
Only the tooling README changes; there are no localized product-doc
counterparts to synchronize.
## Docs Checklist
- [x] Checked by AI
- [x] Test Cases Built
- [x] Updated required version and language counterparts, or explained
why not
- [x] If only one language changed, confirmed whether source/translation
counterparts need sync
Co-authored-by: morningman <[email protected]>
---
scripts/i18n/__tests__/workflow.test.cjs | 188 ++++++++++++++++++++++++++++++-
scripts/i18n/simulate-workflow.cjs | 13 ++-
scripts/i18n/state.cjs | 2 +-
scripts/i18n/translate-docs.cjs | 87 ++++++++++----
scripts/i18n/workflow-generate.cjs | 3 +-
scripts/i18n/workflow-publish.cjs | 18 +--
translations/README.md | 12 +-
7 files changed, 286 insertions(+), 37 deletions(-)
diff --git a/scripts/i18n/__tests__/workflow.test.cjs
b/scripts/i18n/__tests__/workflow.test.cjs
index 9c3f1942317..3b5bf8fc995 100644
--- a/scripts/i18n/__tests__/workflow.test.cjs
+++ b/scripts/i18n/__tests__/workflow.test.cjs
@@ -4,7 +4,9 @@ const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const {execFileSync} = require('node:child_process');
-const {hash, loadConfig, stateFileFor} = require('../state.cjs');
+const {hash, loadConfig, stateFileFor, loadRecords, identity} =
require('../state.cjs');
+const {structure} = require('../structure.cjs');
+const {readState, applyCandidates} = require('../translate-docs.cjs');
const {reconcileOpenPrs} = require('../workflow-prs.cjs');
const {relevantPush, replaceableBotPr} = require('../workflow-scan.cjs');
const {pairFor, machineOwned, generate, allowGeneration} =
require('../workflow-generate.cjs');
@@ -227,3 +229,187 @@ test('one model result creates separate Dev and 4.x
targets and state records',
assert(state.pages.every(page => page.reviewStatus === 'machine' &&
page.sourceHash === hash(source)));
} finally {fs.rmSync(dir, {recursive: true, force: true});}
});
+
+async function updateFixture(t, {sharded = true, human = false} = {}) {
+ const root = fs.mkdtempSync(path.join(os.tmpdir(),
'translation-state-update-'));
+ t.after(() => fs.rmSync(root, {recursive: true, force: true}));
+ const git = (...args) => execFileSync('git', args, {cwd: root, encoding:
'utf8'}).trim();
+ const put = (file, text) => {
+ fs.mkdirSync(path.dirname(path.join(root, file)), {recursive: true});
+ fs.writeFileSync(path.join(root, file), text);
+ };
+ const previousSource = '# Hello\n\nOld.\n';
+ const source = '# Hello\n\nNew.\n';
+ const previousTarget = '# こんにちは {#hello}\n\n古い。\n';
+ put('docs/guide.md', previousSource);
+ put('translations/scope/ja.json', JSON.stringify(scope));
+ git('init', '-q'); git('add', '.');
+ git('-c', 'user.name=Test', '-c', '[email protected]', 'commit',
'-qm', 'old source');
+ const previousCommit = git('rev-parse', 'HEAD');
+ const oldRecord = {...record, sourceLocale: 'en', sourceCommit:
previousCommit,
+ sourceHash: hash(previousSource), sourceTechnicalHash: (await
structure(previousSource, '.md')).technicalHash,
+ targetHash: hash(previousTarget), ownership: human ? 'human' :
'machine', reviewStatus: 'machine',
+ translatedAt: '2026-10-15T00:00:00.000Z', glossaryVersion: '1'};
+ put(record.targetPath, previousTarget);
+ // A stale legacy copy must not take precedence over the page shard.
+ const legacy = sharded ? {...oldRecord, ownership: 'human', targetHash:
hash('obsolete legacy target')} : oldRecord;
+ const legacyPath = 'translations/state/ja/docs.json';
+ put(legacyPath, JSON.stringify({schemaVersion: 1, pages: [legacy]}, null,
2) + '\n');
+ const shardPath = stateFileFor(oldRecord);
+ if (sharded) {
+ put('translations/state/ja/shards.schema.json',
JSON.stringify({schemaVersion: 1, format: 'page-record-v1'}));
+ put(shardPath, JSON.stringify({schemaVersion: 1, page: oldRecord},
null, 2) + '\n');
+ }
+ put('docs/guide.md', source);
+ git('add', '.');
+ git('-c', 'user.name=Test', '-c', '[email protected]', 'commit',
'-qm', 'updated source');
+ const sourceCommit = git('rev-parse', 'HEAD');
+ const task = {...record, id: 'T06-DEV-000000000000001', locale: 'ja',
groupId: 'DEV',
+ kind: 'update', sourceCommit, sourceHash: hash(source), targetHash:
hash(previousTarget),
+ requiresHumanReview: false, activePr: null};
+ const plan = {schemaVersion: 1, locale: 'ja', sourceCommit, selected:
[task.id], tasks: [task], exceptions: []};
+ let calls = 0;
+ const adapter = {model: 'fixture', usedTokens: 0, translate: async
({units}) => {
+ calls++;
+ return {translations: units.map(unit => ({id: unit.id,
+ text: ({Hello: 'こんにちは', 'New.': '新しい。'})[unit.text] ??
unit.text}))};
+ }};
+ const run = extra => generate({rootDir: root, target, output:
path.join(root, 'output'),
+ ensureFresh: sha => assert.equal(sha, sourceCommit), plan,
+ reconciliation: {botPrs: [], botDetails: [], blockedTargets: []},
maxPages: 1,
+ adapter, glossary: require('../../../translations/glossary/ja.json'),
...extra});
+ return {root, git, put, oldRecord, previousTarget, legacyPath, shardPath,
plan, task, adapter, run,
+ calls: () => calls};
+}
+
+test('an existing page shard is updated without rewriting or trusting its
obsolete legacy copy', async t => {
+ const fixture = await updateFixture(t);
+ const {root, legacyPath, shardPath, task} = fixture;
+ const before = fs.readFileSync(path.join(root, legacyPath), 'utf8');
+ const manifest = await fixture.run();
+ assert.equal(manifest.candidates.length, 1);
+ assert.equal(manifest.failures.length, 0);
+ assert.equal(manifest.candidates[0].statePath, shardPath);
+ const effective = loadRecords(root, 'ja').get(identity(task));
+ assert.equal(effective.sourceHash, task.sourceHash);
+ assert.equal(effective.targetHash, hash(fs.readFileSync(path.join(root,
task.targetPath), 'utf8')));
+ assert.match(fs.readFileSync(path.join(root, task.targetPath), 'utf8'),
/新しい。/);
+ assert.equal(fs.readFileSync(path.join(root, legacyPath), 'utf8'), before);
+ assert.deepEqual(expectedFiles(manifest, scope), [task.targetPath,
shardPath].sort());
+});
+
+test('a repository without shard support keeps updating its legacy ledger',
async t => {
+ const fixture = await updateFixture(t, {sharded: false});
+ const manifest = await fixture.run();
+ assert.equal(manifest.candidates.length, 1);
+ assert.equal(manifest.candidates[0].statePath, fixture.legacyPath);
+ assert.equal(loadRecords(fixture.root,
'ja').get(identity(fixture.task)).sourceHash, fixture.task.sourceHash);
+ assert(!fs.existsSync(path.join(fixture.root, fixture.shardPath)));
+});
+
+test('enabling shards migrates only the updated legacy page into its own state
file', async t => {
+ const fixture = await updateFixture(t, {sharded: false});
+ fixture.put('translations/state/ja/shards.schema.json',
JSON.stringify({schemaVersion: 1, format: 'page-record-v1'}));
+ const before = fs.readFileSync(path.join(fixture.root,
fixture.legacyPath), 'utf8');
+ const manifest = await fixture.run();
+ assert.equal(manifest.candidates.length, 1);
+ assert.equal(manifest.candidates[0].statePath, fixture.shardPath);
+ assert.equal(loadRecords(fixture.root,
'ja').get(identity(fixture.task)).sourceHash, fixture.task.sourceHash);
+ assert.equal(fs.readFileSync(path.join(fixture.root, fixture.legacyPath),
'utf8'), before);
+});
+
+test('a human-owned shard is protected even when the legacy ledger says
machine-owned', async t => {
+ const fixture = await updateFixture(t, {human: true});
+ fixture.put(fixture.legacyPath, JSON.stringify({schemaVersion: 1,
+ pages: [{...fixture.oldRecord, ownership: 'machine'}]}));
+ const manifest = await fixture.run();
+ assert.equal(manifest.candidates.length, 0);
+ assert.equal(manifest.failures[0].error, 'Human maintained target requires
manual review');
+ assert.equal(fixture.calls(), 0);
+ assert.equal(fs.readFileSync(path.join(fixture.root, record.targetPath),
'utf8'), fixture.previousTarget);
+});
+
+test('a shard edited during generation is preserved and the target is not
applied', async t => {
+ const fixture = await updateFixture(t);
+ const translate = fixture.adapter.translate;
+ fixture.adapter.translate = async input => {
+ fixture.put(fixture.shardPath, JSON.stringify({schemaVersion: 1,
+ page: {...fixture.oldRecord, ownership: 'human'}}));
+ return translate(input);
+ };
+ await assert.rejects(fixture.run(), /State changed during generation/);
+ assert.equal(loadRecords(fixture.root,
'ja').get(identity(fixture.task)).ownership, 'human');
+ assert.equal(fs.readFileSync(path.join(fixture.root, record.targetPath),
'utf8'), fixture.previousTarget);
+});
+
+test('a human shard created during a legacy-page update is not overwritten',
async t => {
+ const fixture = await updateFixture(t, {sharded: false});
+ fixture.put('translations/state/ja/shards.schema.json',
JSON.stringify({schemaVersion: 1, format: 'page-record-v1'}));
+ const translate = fixture.adapter.translate;
+ fixture.adapter.translate = async input => {
+ fixture.put(fixture.shardPath, JSON.stringify({schemaVersion: 1,
+ page: {...fixture.oldRecord, ownership: 'human'}}));
+ return translate(input);
+ };
+ await assert.rejects(fixture.run(), /State changed during generation/);
+ assert.equal(loadRecords(fixture.root,
'ja').get(identity(fixture.task)).ownership, 'human');
+ assert.equal(fs.readFileSync(path.join(fixture.root, record.targetPath),
'utf8'), fixture.previousTarget);
+});
+
+test('a failed shard write rolls back the translated target', async t => {
+ const fixture = await updateFixture(t);
+ const before = fs.readFileSync(path.join(fixture.root, fixture.shardPath),
'utf8');
+ const rename = fs.renameSync;
+ fs.renameSync = (from, to) => {
+ if (to === path.join(fixture.root, fixture.shardPath)) throw new
Error('fixture state write failure');
+ return rename(from, to);
+ };
+ try {await assert.rejects(fixture.run(), /fixture state write failure/);}
+ finally {fs.renameSync = rename;}
+ assert.equal(fs.readFileSync(path.join(fixture.root, record.targetPath),
'utf8'), fixture.previousTarget);
+ assert.equal(fs.readFileSync(path.join(fixture.root, fixture.shardPath),
'utf8'), before);
+});
+
+test('a later state write failure restores every earlier target and page shard
in the batch', async t => {
+ const fixture = await updateFixture(t);
+ const other = {...fixture.oldRecord, docId: 'other', sourcePath:
'docs/other.md',
+ targetPath: 'i18n/ja/docusaurus-plugin-content-docs/current/other.md'};
+ fixture.put(other.targetPath, fixture.previousTarget);
+ fixture.put(stateFileFor(other), JSON.stringify({schemaVersion: 1, page:
other}));
+ const records = [fixture.oldRecord, other];
+ const state = readState(fixture.root, 'ja');
+ const before = records.map(item => fs.readFileSync(path.join(fixture.root,
stateFileFor(item)), 'utf8'));
+ const candidates = records.map(item => ({task: item, record: {...item,
targetHash: hash('# 新しい。\n')},
+ targetFile: path.join(fixture.root, item.targetPath), previousTarget:
fixture.previousTarget, target: '# 新しい。\n'}));
+ const rename = fs.renameSync;
+ fs.renameSync = (from, to) => {
+ if (to === path.join(fixture.root, stateFileFor(other))) throw new
Error('fixture second state write failure');
+ return rename(from, to);
+ };
+ try {assert.throws(() => applyCandidates(state, candidates), /fixture
second state write failure/);}
+ finally {fs.renameSync = rename;}
+ records.forEach((item, index) => {
+ assert.equal(fs.readFileSync(path.join(fixture.root, item.targetPath),
'utf8'), fixture.previousTarget);
+ assert.equal(fs.readFileSync(path.join(fixture.root,
stateFileFor(item)), 'utf8'), before[index]);
+ });
+});
+
+test('applying a staged candidate cannot replace a human-owned effective
record', async t => {
+ const fixture = await updateFixture(t, {human: true});
+ assert.throws(() => applyCandidates(readState(fixture.root, 'ja'), [{task:
fixture.task,
+ targetFile: path.join(fixture.root, record.targetPath),
previousTarget: fixture.previousTarget,
+ target: '# 新しい。\n', record: {...fixture.oldRecord, ownership:
'machine'}}]), /Human maintained/);
+ assert.equal(loadRecords(fixture.root,
'ja').get(identity(fixture.task)).ownership, 'human');
+});
+
+test('publication deduplicates shared state and rejects an unrelated page
shard', () => {
+ const first = {...record, statePath: stateFileFor({...record, locale:
'ja'})};
+ assert.deepEqual(expectedFiles({schemaVersion: 1, provider: 'codex',
candidates: [first]}, scope),
+ [first.targetPath, first.statePath].sort());
+ assert.throws(() => expectedFiles({schemaVersion: 1, provider: 'codex',
+ candidates: [{...first, statePath:
'translations/state/ja/pages/default/current/other.json'}]}, scope), /state
path/);
+ const second = {...record, docId: 'other', sourcePath: 'docs/other.md',
+ targetPath: 'i18n/ja/docusaurus-plugin-content-docs/current/other.md'};
+ assert.deepEqual(expectedFiles({schemaVersion: 1, provider: 'codex',
candidates: [record, second]}, scope),
+ [record.targetPath, second.targetPath,
'translations/state/ja/docs.json'].sort());
+});
diff --git a/scripts/i18n/simulate-workflow.cjs
b/scripts/i18n/simulate-workflow.cjs
index 0a173a19dbd..dec7944379a 100644
--- a/scripts/i18n/simulate-workflow.cjs
+++ b/scripts/i18n/simulate-workflow.cjs
@@ -5,6 +5,7 @@ const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const {execFileSync, spawnSync} = require('node:child_process');
+const {loadRecords} = require('./state.cjs');
const sourceRoot = path.resolve(__dirname, '../..');
const sourcePath = process.env.I18N_SIM_SOURCE_PATH
@@ -133,14 +134,18 @@ function main() {
const branch = command('git', ['rev-parse', `refs/heads/${pr.head}`],
{cwd: remote});
const target = command('git', ['show', `${branch}:${targetPath}`], {cwd:
remote});
assert.match(target, /[\u3040-\u30ff\u3400-\u9fff]/);
- const state = JSON.parse(command('git', ['show',
`${branch}:translations/state/ja/docs.json`], {cwd: remote}));
- assert(state.pages.some(page => page.sourcePath === sourcePath &&
page.reviewStatus === 'machine'));
+ const state = [...loadRecords(repo, 'ja').values()];
+ const candidate = generation.candidates[0];
+ assert(state.some(page => page.sourcePath === sourcePath &&
page.reviewStatus === 'machine'
+ && page.sourceHash === candidate.sourceHash && page.targetHash ===
candidate.targetHash));
+ const expectedFiles = [targetPath, candidate.statePath].sort();
+ const committedFiles = command('git', ['diff', '--name-only',
plan.sourceCommit, branch], {cwd: repo}).split('\n').sort();
+ assert.deepEqual(committedFiles, expectedFiles);
const report = {kind: 'local simulation; no external model or GitHub
writes', output,
sourceCommit: plan.sourceCommit, selected: plan.selected, sourcePath,
invalidModel: {exitCode: 1, error: failure.failures[0].error,
usedTokens: failure.usedTokens},
generation: {candidates: generation.candidates.length, usedTokens:
generation.usedTokens},
- draftPr: {url: pr.url, branch: pr.head, commit: branch, files:
[targetPath,
- 'translations/state/ja/docs.json']}};
+ draftPr: {url: pr.url, branch: pr.head, commit: branch, files:
committedFiles}};
fs.writeFileSync(path.join(output, 'report.json'), JSON.stringify(report,
null, 2) + '\n');
process.stdout.write(JSON.stringify(report, null, 2) + '\n');
}
diff --git a/scripts/i18n/state.cjs b/scripts/i18n/state.cjs
index c93e7606487..29191c457e4 100644
--- a/scripts/i18n/state.cjs
+++ b/scripts/i18n/state.cjs
@@ -197,4 +197,4 @@ function decide({page, source, target, record,
technicalHash}) {
}
module.exports = {hash, identity, relative, loadConfig, managedLocales,
pipelineLocale, languageName, inventoryBaseline, loadRecords, loadRecordsAt,
- stateFileFor, listShardFiles, sourceLocaleFor, decide};
+ stateFileFor, listShardFiles, collectRecords, sourceLocaleFor, decide};
diff --git a/scripts/i18n/translate-docs.cjs b/scripts/i18n/translate-docs.cjs
index b85086385cd..be137c86960 100644
--- a/scripts/i18n/translate-docs.cjs
+++ b/scripts/i18n/translate-docs.cjs
@@ -4,7 +4,7 @@ const fs = require('node:fs');
const path = require('node:path');
const {randomUUID} = require('node:crypto');
const {execFileSync} = require('node:child_process');
-const {hash, identity, loadConfig, pipelineLocale} = require('./state.cjs');
+const {hash, identity, relative, loadConfig, pipelineLocale, stateFileFor,
listShardFiles, collectRecords} = require('./state.cjs');
const {structure} = require('./structure.cjs');
const {protectionSignature} = require('./protection.cjs');
const {checkedTask, safeFile, openAIAdapter, mockAdapter, translateDocument} =
require('./translation-engine.cjs');
@@ -47,11 +47,32 @@ function scopeFile(root, locale) { return path.join(root,
`translations/scope/${
function glossaryFile(root, locale) { return path.join(root,
`translations/glossary/${locale}.json`); }
function readState(root, locale) {
- const file = path.join(root, `translations/state/${locale}/docs.json`);
- const raw = fs.readFileSync(file, 'utf8');
- const data = JSON.parse(raw);
- assert(data.schemaVersion === 1 && Array.isArray(data.pages), 'Invalid
translation state');
- return {file, raw, data};
+ root = path.resolve(root);
+ const file = safeFile(root, `translations/state/${locale}/docs.json`);
+ const raw = readOptional(file);
+ const legacy = raw === null ? {schemaVersion: 1, pages: []} :
JSON.parse(raw);
+ const shards = new Map(listShardFiles(root, locale)
+ .map(name => [name, fs.readFileSync(safeFile(root, name), 'utf8')]));
+ const schemaFile = safeFile(root,
`translations/state/${locale}/shards.schema.json`);
+ const schemaRaw = readOptional(schemaFile);
+ if (schemaRaw !== null) {
+ const schema = JSON.parse(schemaRaw);
+ assert(schema.schemaVersion === 1 && schema.format ===
'page-record-v1', 'Unsupported translation shard schema');
+ }
+ // Build the effective view from the same bytes we retain for write guards.
+ const data = {...legacy, pages: [...collectRecords(locale, legacy,
shards).values()]};
+ return {root, locale, file, raw, legacy, shards, schemaFile, schemaRaw,
sharded: schemaRaw !== null, data};
+}
+
+function readOptional(file) {
+ try {return fs.readFileSync(file, 'utf8');}
+ catch (error) {if (error.code === 'ENOENT') return null; throw error;}
+}
+
+function statePathForRecord(state, record) {
+ assert.equal(record.locale, state.locale, 'Candidate state locale
changed');
+ const shard = stateFileFor(record);
+ return state.sharded || state.shards.has(shard) ? shard :
relative(state.root, state.file);
}
function writeAtomic(file, value) {
@@ -65,6 +86,7 @@ function writeAtomic(file, value) {
async function prepare({root, plan, taskId, scope, state, adapter, glossary,
target, maxDocumentChars, maxChunkChars}) {
const {task, source} = checkedTask(plan, taskId, scope, root);
const existing = state.data.pages.find(item => identity(item) ===
identity(task));
+ assert(!existing || existing.ownership === 'machine', 'Human maintained
target requires manual review');
const targetFile = safeFile(root, task.targetPath);
const previousTarget = fs.existsSync(targetFile) ?
fs.readFileSync(targetFile, 'utf8') : null;
assert(!previousTarget || existing, 'Untracked target needs human review');
@@ -99,29 +121,50 @@ async function prepare({root, plan, taskId, scope, state,
adapter, glossary, tar
}
function applyCandidates(state, candidates) {
- assert.equal(fs.readFileSync(state.file, 'utf8'), state.raw, 'State
changed during generation');
+ assert.equal(readOptional(state.file), state.raw, 'State changed during
generation');
+ if (state.schemaFile) assert.equal(readOptional(state.schemaFile),
state.schemaRaw, 'State changed during generation');
+ assert.equal(new Set(candidates.map(item => identity(item.record))).size,
candidates.length, 'Duplicate candidate identity');
+ assert.equal(new Set(candidates.map(item => item.targetFile)).size,
candidates.length, 'Duplicate candidate target');
for (const candidate of candidates) {
- const actual = fs.existsSync(candidate.targetFile) ?
fs.readFileSync(candidate.targetFile, 'utf8') : null;
+ const actual = readOptional(candidate.targetFile);
assert.equal(actual, candidate.previousTarget, `Target changed during
generation: ${candidate.task.targetPath}`);
+ const existing = state.data.pages.find(item => identity(item) ===
identity(candidate.record));
+ assert(!existing || existing.ownership === 'machine', 'Human
maintained target requires manual review');
+ }
+ const writes = candidates.map(item => ({file: item.targetFile, before:
item.previousTarget, after: item.target}));
+ const legacyCandidates = [];
+ for (const candidate of candidates) {
+ const name = statePathForRecord(state, candidate.record);
+ if (name === relative(state.root, state.file)) {
+ legacyCandidates.push(candidate);
+ // A new shard must not silently supersede the legacy record we
read.
+ assert.equal(readOptional(safeFile(state.root,
stateFileFor(candidate.record))), null,
+ 'State changed during generation');
+ continue;
+ }
+ const file = safeFile(state.root, name);
+ const before = state.shards.get(name) ?? null;
+ assert.equal(readOptional(file), before, 'State changed during
generation');
+ writes.push({file, before, after: JSON.stringify({schemaVersion: 1,
page: candidate.record}, null, 2) + '\n'});
+ }
+ if (legacyCandidates.length) {
+ const pages = state.legacy.pages.filter(page =>
!legacyCandidates.some(item => identity(item.record) === identity(page)));
+ pages.push(...legacyCandidates.map(item => item.record));
+ pages.sort((a, b) => identity(a).localeCompare(identity(b), 'en'));
+ writes.push({file: state.file, before: state.raw, after:
JSON.stringify({...state.legacy, pages}, null, 2) + '\n'});
}
- const pages = state.data.pages.filter(page => !candidates.some(item =>
identity(item.record) === identity(page)));
- pages.push(...candidates.map(item => item.record));
- pages.sort((a, b) => identity(a).localeCompare(identity(b), 'en'));
- const nextState = JSON.stringify({...state.data, pages}, null, 2) + '\n';
const written = [];
try {
- for (const candidate of candidates) {
- fs.mkdirSync(path.dirname(candidate.targetFile), {recursive:
true});
- writeAtomic(candidate.targetFile, candidate.target);
- written.push(candidate);
+ for (const write of writes) {
+ fs.mkdirSync(path.dirname(write.file), {recursive: true});
+ writeAtomic(write.file, write.after);
+ written.push(write);
}
- writeAtomic(state.file, nextState);
} catch (error) {
- for (const candidate of written.reverse()) {
- if (candidate.previousTarget === null)
fs.rmSync(candidate.targetFile, {force: true});
- else writeAtomic(candidate.targetFile, candidate.previousTarget);
+ for (const write of written.reverse()) {
+ if (write.before === null) fs.rmSync(write.file, {force: true});
+ else writeAtomic(write.file, write.before);
}
- writeAtomic(state.file, state.raw);
throw error;
}
}
@@ -242,4 +285,4 @@ async function main() {
}
if (require.main === module) main().catch(error =>
{console.error(error.message); process.exitCode = 1;});
-module.exports = {optionsFrom, prepare, applyCandidates, applyStaged,
readState, targetFor, scopeFile, glossaryFile};
+module.exports = {optionsFrom, prepare, applyCandidates, applyStaged,
readState, statePathForRecord, targetFor, scopeFile, glossaryFile};
diff --git a/scripts/i18n/workflow-generate.cjs
b/scripts/i18n/workflow-generate.cjs
index b774f96529d..fbf319e8a3d 100644
--- a/scripts/i18n/workflow-generate.cjs
+++ b/scripts/i18n/workflow-generate.cjs
@@ -7,7 +7,7 @@ const {execFileSync} = require('node:child_process');
const {hash, identity, languageName, pipelineLocale} = require('./state.cjs');
const {checkedTask, managedPath, safeFile} =
require('./translation-engine.cjs');
const {codexAdapter} = require('./codex-adapter.cjs');
-const {prepare, applyCandidates, readState, targetFor} =
require('./translate-docs.cjs');
+const {prepare, applyCandidates, readState, statePathForRecord, targetFor} =
require('./translate-docs.cjs');
const {replaceableBotPr} = require('./workflow-scan.cjs');
const {markerPrefixFor} = require('./automation-naming.cjs');
@@ -130,6 +130,7 @@ async function generate({plan, reconciliation, maxPages =
2, target,
pluginId: item.task.pluginId, version: item.task.version, docId:
item.task.docId,
sourcePath: item.task.sourcePath, sourceHash: item.task.sourceHash,
targetPath: item.task.targetPath, targetHash:
item.record.targetHash,
+ statePath: statePathForRecord(state, item.record),
reusedFrom: item.reusedFrom || null, requests: item.requests})),
failures};
fs.mkdirSync(output, {recursive: true});
fs.writeFileSync(path.join(output, 'generation.json'),
JSON.stringify(manifest, null, 2) + '\n');
diff --git a/scripts/i18n/workflow-publish.cjs
b/scripts/i18n/workflow-publish.cjs
index 48398fdd51f..3584ee178d1 100644
--- a/scripts/i18n/workflow-publish.cjs
+++ b/scripts/i18n/workflow-publish.cjs
@@ -3,7 +3,7 @@ const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const {execFileSync} = require('node:child_process');
-const {languageName, pipelineLocale} = require('./state.cjs');
+const {languageName, pipelineLocale, stateFileFor} = require('./state.cjs');
const {managedPath} = require('./translation-engine.cjs');
const {statePathFor} = require('./workflow-prs.cjs');
const {botBranchFor, botBranchPattern} = require('./automation-naming.cjs');
@@ -15,12 +15,16 @@ function expectedFiles(manifest, scope) {
assert(manifest.schemaVersion === 1 && manifest.provider === 'codex'
&& manifest.candidates.length > 0 && manifest.candidates.length <=
scope.maxBatchPages,
'Invalid live candidate manifest');
- const files = manifest.candidates.map(candidate => {
+ const files = manifest.candidates.flatMap(candidate => {
managedPath(scope, {...candidate, locale: scope.locale});
- return candidate.targetPath;
+ // Older manifests use the shared ledger; new ones name one exact
shard.
+ const statePath = candidate.statePath ?? statePathFor(scope.locale);
+ assert(statePath === statePathFor(scope.locale)
+ || statePath === stateFileFor({...candidate, locale:
scope.locale}), 'Invalid candidate state path');
+ return [candidate.targetPath, statePath];
});
- assert.equal(new Set(files).size, files.length, 'Duplicate target path');
- return [statePathFor(scope.locale), ...files].sort();
+ assert.equal(new Set(manifest.candidates.map(item =>
item.targetPath)).size, manifest.candidates.length, 'Duplicate target path');
+ return [...new Set(files)].sort();
}
function checkStaged(manifest, scope) {
@@ -54,7 +58,7 @@ function main() {
const language = languageName(root, locale);
const scope = JSON.parse(fs.readFileSync(path.join(root,
`translations/scope/${locale}.json`), 'utf8'));
assert.equal(git('rev-parse', 'HEAD'), manifest.sourceCommit, 'Source
checkout changed before publishing');
- const expected = checkStaged(manifest, scope);
+ checkStaged(manifest, scope);
const branch = `${botBranchFor(locale)}${process.env.GITHUB_RUN_ID}`;
assert(botBranchPattern(locale).test(branch), 'Unsafe run ID');
const user = kind === 'pat' ? JSON.parse(execFileSync('gh', ['api',
'user', '--jq', '{login,id}'],
@@ -90,7 +94,7 @@ function main() {
'--head', branch, '--draft', '--title', `docs(i18n): machine
${language} translation candidates`,
'--body-file', bodyFile], {cwd: root, encoding: 'utf8'}).trim();
if (process.env.GITHUB_STEP_SUMMARY)
fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY,
- `\nOpened draft translation PR: ${url}; ${expected.length - 1}
candidate pages.\n`);
+ `\nOpened draft translation PR: ${url}; ${manifest.candidates.length}
candidate pages.\n`);
process.stdout.write(`${url}\n`);
}
diff --git a/translations/README.md b/translations/README.md
index 579f9a1c36a..a87bbb8f768 100644
--- a/translations/README.md
+++ b/translations/README.md
@@ -26,7 +26,17 @@ Each file contains `{"schemaVersion":1,"page":<record>}`.
The loader combines
both locations; a page file supersedes a legacy entry with the same identity.
The path must match the record's native identity, so independent translation
PRs
modify distinct state files. `state/ja/shards.schema.json` marks the repository
-as ready for page files. Each record contains these fields:
+as ready for page files. The manual and automated translation tools read the
+same merged state as the site, with page files taking precedence over legacy
+entries. Once the marker is present, updates and new translations write their
+own page files and leave the legacy ledger unchanged. Repositories without the
+marker retain legacy writes, while existing page files are always updated in
+place. Generation records the exact state path for each candidate so
publication
+can stage only its translation and corresponding state. Before applying a
batch,
+the tools check the original target and state bytes and reject concurrent
edits;
+a failed write rolls back all files already written by that batch.
+
+Each record contains these fields:
| Field | Meaning |
| --- | --- |
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
