This is an automated email from the ASF dual-hosted git repository. urfree pushed a commit to branch refactor/migrate-scripts in repository https://gitbox.apache.org/repos/asf/pulsar-site.git
commit 68929d4342943b7f3dd19ca01b0cd32c44854ec0 Author: LiLi <[email protected]> AuthorDate: Fri Feb 11 11:07:48 2022 +0800 feat: docs migration scripts Signed-off-by: LiLi <[email protected]> --- site2/website-next/.gitignore | 1 - site2/website-next/migration-scripts/.gitignore | 2 + site2/website-next/migration-scripts/auto.js | 98 ++++++++++++ site2/website-next/migration-scripts/fix-code.js | 41 +++++ site2/website-next/migration-scripts/fix-jsx.js | 7 + site2/website-next/migration-scripts/fix-kop.js | 44 +++++ site2/website-next/migration-scripts/fix-md.js | 54 +++++++ site2/website-next/migration-scripts/fix-miss.js | 59 +++++++ site2/website-next/migration-scripts/fix-space.js | 54 +++++++ site2/website-next/migration-scripts/fix-tab.js | 97 +++++++++++ site2/website-next/migration-scripts/fix-table.js | 72 +++++++++ .../migration-scripts/fix-tip-note-in-list.js | 27 ++++ .../website-next/migration-scripts/fix-tip-note.js | 81 ++++++++++ .../migration-scripts/migrate-chapter.js | 178 +++++++++++++++++++++ site2/website-next/migration-scripts/migrate-md.js | 45 ++++++ .../migration-scripts/migrate-missing.js | 3 + .../migration-scripts/migrate-override.js | 3 + site2/website-next/migration-scripts/migration.js | 8 + site2/website-next/migration-scripts/migration.sh | 5 + site2/website-next/migration-scripts/override.js | 44 +++++ 20 files changed, 922 insertions(+), 1 deletion(-) diff --git a/site2/website-next/.gitignore b/site2/website-next/.gitignore index 2f55ed0..25890cc 100644 --- a/site2/website-next/.gitignore +++ b/site2/website-next/.gitignore @@ -20,6 +20,5 @@ yarn-debug.log* yarn-error.log* bak/ -migration-scripts/ .latest .versions \ No newline at end of file diff --git a/site2/website-next/migration-scripts/.gitignore b/site2/website-next/migration-scripts/.gitignore new file mode 100644 index 0000000..89db65e --- /dev/null +++ b/site2/website-next/migration-scripts/.gitignore @@ -0,0 +1,2 @@ +.cache +/node_modules \ No newline at end of file diff --git a/site2/website-next/migration-scripts/auto.js b/site2/website-next/migration-scripts/auto.js new file mode 100644 index 0000000..84bce81 --- /dev/null +++ b/site2/website-next/migration-scripts/auto.js @@ -0,0 +1,98 @@ +const fs = require("fs"); +const path = require("path"); +const lodash = require("lodash"); +const child_process = require("child_process"); +const migrate = require("./migrate-chapter"); +const override = require("./override"); + +const args = process.argv.slice(2); +const version = args[0]; +let rerun = version == "rerun"; + +function run(rerun) { + if (rerun) { + let cache = fs.readFileSync(path.join(__dirname, ".cache"), "utf8"); + cache = JSON.parse(cache); + console.log( + "auto migrate docs for version: ", + cache.version, + "chapter: ", + cache.chapter + ); + migrate(cache.version, cache.chapter, ""); + return; + } + + let version_full = "version-" + version; + let src = "../../website/versioned_docs/" + version_full; + let dest = "../../website-next/versioned_docs/" + version_full; + if (version == "next") { + src = "../../docs"; + dest = "../../website-next/docs"; + } + src = path.join(__dirname, src); + dest = path.join(__dirname, dest); + + let sidebar_file = path.join( + __dirname, + "../../website/versioned_sidebars/" + version_full + "-sidebars.json" + ); + if (version == "next") { + sidebar_file = path.join(__dirname, "../../website/sidebars.json"); + } + let sidebar = fs.readFileSync(sidebar_file, "utf8"); + sidebar = JSON.parse(sidebar); + + sidebar = sidebar[version == "next" ? "docs" : version_full + "-docs"]; + + let new_sidebar_file = path.join( + __dirname, + "../../website-next/versioned_sidebars/" + version_full + "-sidebars.json" + ); + if (version == "next") { + new_sidebar_file = path.join(__dirname, "../../website-next/sidebars.json"); + } + + let chapters = Object.keys(sidebar); + + function migrateChapter() { + let new_sidebar = []; + let new_sidebar_map = {}; + if (fs.existsSync(new_sidebar_file)) { + new_sidebar = JSON.parse(fs.readFileSync(new_sidebar_file, "utf8")); + let key = + version == "next" ? "docsSidebar" : version_full + "/docsSidebar"; + new_sidebar = new_sidebar[key]; + new_sidebar_map = lodash.keyBy(new_sidebar, "label"); + } + + let migrated_chapters = Object.keys(new_sidebar_map); + + let need_migration_chapers = lodash.difference(chapters, migrated_chapters); + + let current_chapter = + need_migration_chapers.length > 0 ? need_migration_chapers[0] : null; + if (current_chapter) { + console.log( + "auto migrate docs for version: ", + version, + "chapter: ", + current_chapter + ); + migrate(version, current_chapter, ""); + fs.writeFileSync( + path.join(__dirname, ".cache"), + JSON.stringify({ version: version, chapter: current_chapter }) + ); + } else { + console.log( + "所有章节已迁移完成，现在执行一次完整的【" + version + "】copy fix" + ); + override(version); + } + } + + migrateChapter(); +} + +run(rerun); diff --git a/site2/website-next/migration-scripts/fix-code.js b/site2/website-next/migration-scripts/fix-code.js new file mode 100644 index 0000000..b297473 --- /dev/null +++ b/site2/website-next/migration-scripts/fix-code.js @@ -0,0 +1,41 @@ +module.exports = (data) => { + let nData = data; + + let reg = /^([\t ]*)(>*)([\t ]*)(```.*)((((?!```).)*\n*)+)```/gm; //代码块 + while ((codeGroup = reg.exec(data))) { + let _match = codeGroup[0]; + let space1 = codeGroup[1]; + let arrow = codeGroup[2]; + let space2 = codeGroup[3]; + let begin = codeGroup[4]; + let code = codeGroup[5]; + let prefix = space1 + arrow + space2; + if ((code = /[\t ]*(.+\n*)*\S/.exec(code))) { + code = code[0]; //剔除纯空白的行首和行尾, 得到纯代码 + if (code.substr(0, prefix.length) != prefix) { + code = prefix + code.replace(/\n(.*)/g, "\n" + prefix + "$1"); //行对齐 + } + } + let newCodeBlock = + "\n" + + prefix + + begin + + "\n" + + prefix + + "\n" + + code + + "\n" + + prefix + + "\n" + + prefix + + "```" + + "\n"; + nData = nData.replace(_match, newCodeBlock); + } + return nData + .replace( + /(.*)(\s*\n){3,}(\s*```.*)((((?!```).)*\n*)+)```.*(\s*\n)+/g, + "$1\n\n$3$4```\n\n" + ) //解决多个空行 + .replace(/^(\s*```.*(((?!```).)*\n*)+)```.*\n+/gm, "$1```\n\n"); //解决多个空行 //解决代码块结束标识符后无空行 +}; diff --git a/site2/website-next/migration-scripts/fix-jsx.js b/site2/website-next/migration-scripts/fix-jsx.js new file mode 100644 index 0000000..48bce1c --- /dev/null +++ b/site2/website-next/migration-scripts/fix-jsx.js @@ -0,0 +1,7 @@ +module.exports = (data) => { + return data + .replace(/(import Tabs from '@theme\/Tabs';)/g, "````mdx-code-block\n$1") + .replace(/(import TabItem from '@theme\/TabItem';)/g, "$1\n````") + .replace(/(<Tabs>?)/g, "````mdx-code-block\n$1") + .replace(/(<\/Tabs>)/g, "$1\n````"); +}; diff --git a/site2/website-next/migration-scripts/fix-kop.js b/site2/website-next/migration-scripts/fix-kop.js new file mode 100644 index 0000000..cad71c1 --- /dev/null +++ b/site2/website-next/migration-scripts/fix-kop.js @@ -0,0 +1,44 @@ +const fs = require("fs"); +const path = require("path"); +const axios = require("axios"); + +const versionReg = /^v(\d+\.?){3,4}$/gm; +const dir = "/Users/leo/space/sn/pulsar-hub/protocol-handlers/kop"; +const tags_url = + "https://urfreespace:[email protected]/repos/streamnative/kop/tags";; +const kop_md_url = + "https://urfreespace:ghp_yramcnmbemdopczt1v9pdwv5psjz4n3m1...@raw.githubusercontent.com/streamnative/kop/{{version}}/docs/kop.md";; + +async function fix() { + let { data } = await axios({ + method: "get", + url: tags_url, + proxy: { + host: "127.0.0.1", + port: 7890, + }, + }); + let tags = data + .map((item) => item.name) + .filter((item) => versionReg.test(item)); + // console.log(tags); + for (let tag of tags) { + let url = kop_md_url.replace(/{{version}}/, tag); + let { data } = await axios({ + method: "get", + url: url, + proxy: { + host: "127.0.0.1", + port: 7890, + }, + }); + let _path = path.join(dir, tag.substr(1)); + if (!fs.existsSync(_path)) { + fs.mkdirSync(_path); + } + data = data.replace(/kop\/blob\/master/g, "kop/blob/" + tag); + fs.writeFileSync(path.join(_path, "kop.md"), data); + } +} + +fix(); diff --git a/site2/website-next/migration-scripts/fix-md.js b/site2/website-next/migration-scripts/fix-md.js new file mode 100644 index 0000000..14f15bc --- /dev/null +++ b/site2/website-next/migration-scripts/fix-md.js @@ -0,0 +1,54 @@ +const fixTab = require("./fix-tab"); +const fixTipNote = require("./fix-tip-note"); +const fixTable = require("./fix-table"); +const fixCode = require("./fix-code"); +const fixTipNoteInList = require("./fix-tip-note-in-list"); +const fixJSX = require("./fix-jsx"); + +function fix(data, version) { + let reg = new RegExp("id:\\s*version-" + version + "-(incubating-)?"); + data = fixTable(data); + data = fixTab(data); + data = data.replace(/(<TabItem.*)\n(.*{@inject:\s*.+?})/g, "$1\n\n$2"); + data = data.replace(/^(>\s*```\w*)(\s*\n){2,}>/gm, "$1\n>\n>"); + data = fixTipNote(data); + data = fixTipNoteInList(data); + data = fixCode(data); + + data = data + .replace(reg, "id: ") + .replace("id: deploy-docs", "id: deploy-dcos") + .replace(/`{4,}/g, "```") + .replace(/^(id:\sstandalone)\s*$/gm, "slug: /\n$1") + .replace(/<style[\s\S]*?<\/style>/g, "") + .replace(/\[(.*)\]\s*\((.*)\.md\)/g, "[$1]($2)") + .replace(/\*\*(((?!\*).)+)\*/g, "**$1**") //**macOS* => **macOS** + .replace(/\*\*(((?!\*).)+)\*{3}/g, "**$1**") //**macOS*** => **macOS** + .replace(/(\w)[\t ]*\n\[/g, "$1 [") + .replace(/sidebar_label:\s*(.*)/, 'sidebar_label: "$1"') + .replace(/""(.*)""/, '"$1"') + .replace(/\[\[pulsar:version_number\]\]/g, "@pulsar:version_number@") + .replace(/{{(pulsar:.*?)}}/g, "@$1@") + .replace(/(@inject:\s*\w+:)`(\w+)`/g, "$1$2") + .replace(/<empty string>/g, "|") + .replace(/<li>(((?!<\/?li>|\|).)+)/g, "<li>$1</li>") + .replace(/<\/li>\s*<\/li>/g, "</li>") + .replace(/\|(((?!<\/?li>|\|).)+)<\/li>/g, "|<li>$1</li>") + .replace(/<\/li>\s*<\/li>/g, "</li>") //sometimes needed, no idea for now... + .replace(/<\/br>/g, "@AAA@") + .replace(/<br>/g, "@AAA@") + .replace(/<br\/>/g, "@AAA@") + .replace(/<>/g, "") + .replace(/@AAA@/g, "<br />") + .replace( + /<span style="(((?!:).)+):(((?!>).)+);"/g, + '<span style={{color: "$3"}}' + ) + .replace(/\s?style=".*?"/g, "") + .replace(/\]\(assets\//g, "](/assets/"); + + return data; + // return fixJSX(data); +} + +module.exports = fix; diff --git a/site2/website-next/migration-scripts/fix-miss.js b/site2/website-next/migration-scripts/fix-miss.js new file mode 100644 index 0000000..f20574b --- /dev/null +++ b/site2/website-next/migration-scripts/fix-miss.js @@ -0,0 +1,59 @@ +const fs = require("fs"); +const path = require("path"); + +function fix(docId, versions) { + console.log("Fix missing for: ", docId, versions); + + let fixed = false; + + function travel(dir, callback) { + fs.readdirSync(dir) + .sort(() => -1) + .forEach((file) => { + var pathname = path.join(dir, file); + if (fs.statSync(pathname).isDirectory()) { + if (!fixed) { + travel(pathname, callback); + } + } else { + if (!fixed) { + fixed = callback(pathname); + } + } + }); + } + + const vdocs = path.join(__dirname, "../../website/versioned_docs"); + travel(vdocs, (pathname) => { + let data = fs.readFileSync(pathname, "utf8"); + let _match = /^id:\s*version-[\d\.]+-(incubating-)?(.*)/gm.exec(data); + // if (_match[1].indexOf(docId) > -1) { + if (_match && _match[2] == docId) { + console.log(pathname, _match[0]); + + let vs = versions.split(","); + vs.forEach((v) => { + if (v == "next") { + fs.writeFileSync( + path.join(vdocs, "../../docs/", docId + ".md"), + data.replace(/^id:\s*version-[\d\.]+-(.*)/gm, "id: " + docId), + "utf8" + ); + } else { + fs.writeFileSync( + path.join(vdocs, "version-" + v, docId + ".md"), + data.replace( + /^id:\s*version-[\d\.]+-(.*)/gm, + "id: version-" + v + "-" + docId + ), + "utf8" + ); + } + }); + return true; + } + return false; + }); +} + +module.exports = fix; diff --git a/site2/website-next/migration-scripts/fix-space.js b/site2/website-next/migration-scripts/fix-space.js new file mode 100644 index 0000000..56dfd99 --- /dev/null +++ b/site2/website-next/migration-scripts/fix-space.js @@ -0,0 +1,54 @@ +const fs = require("fs"); +const path = require("path"); + +const code_map = {}; +let index = 0; + +function fixSpace(data) { + const code_reg = /(```\w*)((((?!```).)*\n*)+)```/; + const _match = code_reg.exec(data); + if (_match) { + const code = _match[0]; + const TMP_KEY = "_TMP_CODE_KEY_" + index + "_"; + code_map[TMP_KEY] = code; + index++; + data = data.replace(code_reg, TMP_KEY); + return fixSpace(data); + } else { + return data; + } +} + +function fixTmp(data) { + const tmp_reg = /_TMP_CODE_KEY_(\d+)_/; + const _match = tmp_reg.exec(data); + if (_match) { + data = data.replace(tmp_reg, code_map[_match[0]]); + return fixTmp(data); + } else { + return data; + } +} + +function fix(data) { + data = fixSpace(data); + data = data.replace(/^[\t ]+/gm, ""); + data = fixTmp(data); + return data; +} + +function test() { + let data = fs.readFileSync( + path.join(__dirname, "../bak/io-quickstart.md"), + "utf8" + ); + data = fixSpace(data); + data = data.replace(/^[\t ]+/gm, ""); + data = fixTmp(data); + // console.log(data); + fs.writeFileSync(path.join(__dirname, "../bak/io-quickstart-fixed.md"), data); +} + +// test(); + +module.exports = fix; diff --git a/site2/website-next/migration-scripts/fix-tab.js b/site2/website-next/migration-scripts/fix-tab.js new file mode 100644 index 0000000..3dd751a --- /dev/null +++ b/site2/website-next/migration-scripts/fix-tab.js @@ -0,0 +1,97 @@ +const fs = require("fs"); +const path = require("path"); +const lodash = require("lodash"); +const TAB_REG = + /([\t ]*)<!--DOCUSAURUS_CODE_TABS-->(((?!<!--END_DOCUSAURUS_CODE_TABS-->).)*\n*)*<!--END_DOCUSAURUS_CODE_TABS-->/; + +function removeTabTag(tab) { + return tab + .replace(/<!--DOCUSAURUS_CODE_TABS-->/, "") + .replace(/<!--END_DOCUSAURUS_CODE_TABS-->/, ""); +} + +function findTabItemNameList(tab) { + const _match = tab.match(/<!--(.*)-->/g); + return _match.map((item) => { + return item.replace("<!--", "").replace("-->", ""); + }); +} + +function replaceTabItemTag(tab) { + const tab_item_reg = /([\t ]*)<!--(.*)-->((((?![\t ]*<!--).)*\n*)*)/; + const _match = tab_item_reg.exec(tab); + if (_match) { + const space = _match[1]; + const tab_item_name = _match[2]; + const tab_item_content = lodash.trimEnd(_match[3]); + let tab_item_str = space + '<TabItem value="'; + tab_item_str += + tab_item_name + '">' + tab_item_content + "\n\n" + space + "</TabItem>\n"; + tab = tab.replace(tab_item_reg, tab_item_str); + return replaceTabItemTag(tab); + } else { + return tab; + } +} + +function importTabComponent(data) { + if (!/import Tabs from '@theme\/Tabs';/g.exec(data)) { + return data.replace( + /---((((?!---).)*\n*)*)---/, + "---$1---" + + "\n\nimport Tabs from '@theme/Tabs';\nimport TabItem from '@theme/TabItem';\n" + ); + } + return data; +} + +function fixTab(data) { + const _match = TAB_REG.exec(data); + if (_match) { + data = importTabComponent(data); + let tab = _match[0]; + tab = removeTabTag(tab); + const names = findTabItemNameList(tab); + tab = replaceTabItemTag(tab); + const names_map = names.map((item) => { + return { + label: item, + value: item, + }; + }); + + const space = _match[1]; + const tab_tag_begin = + space + + "<Tabs \n" + + space + + ' defaultValue="' + + names[0] + + '"\n' + + space + + " values={" + + JSON.stringify(names_map) + + "}>"; + + const tab_tag_end = "\n" + space + "</Tabs>"; + tab = tab_tag_begin + tab + tab_tag_end; + return fixTab(data.replace(TAB_REG, tab)); + } else { + return data + .replace(/(<TabItem.*)\n[\t ]*\n/g, "$1\n") + .replace(/(<TabItem.*)/g, "$1\n"); + } +} + +function test() { + let data = fs.readFileSync( + path.join(__dirname, "../bak/schema-manage.md"), + "utf8" + ); + data = fixTab(data); + console.log(data); +} + +// test() + +module.exports = fixTab; diff --git a/site2/website-next/migration-scripts/fix-table.js b/site2/website-next/migration-scripts/fix-table.js new file mode 100644 index 0000000..ef629f8 --- /dev/null +++ b/site2/website-next/migration-scripts/fix-table.js @@ -0,0 +1,72 @@ +const fs = require("fs"); +const path = require("path"); +const NL = "\n"; + +//Convert HTML table to markdown table +function convertTableElementToMarkdown(table) { + let rows = []; + let trEls = table.match(/[<]tr[\s\S]*?[/]tr[>]/g); + for (let i = 0; i < trEls.length; i++) { + let tableRow = trEls[i]; + let markdownRow = convertTableRowElementToMarkdown(tableRow, i); + rows.push(markdownRow); + } + return rows.join(NL); +} + +//Convert the rows of the HTML table to the rows of the markdown table +function convertTableRowElementToMarkdown(tableRow, rowNumber) { + let cells = []; + let cellEls = []; + cellEls = tableRow.match(/[<]th[\s\S]*?[/]th[>]/g); + if (!cellEls || cellEls.length == 0) { + cellEls = tableRow.match(/[<]td[\s\S]*?[/]td[>]/g); + } + let dividerCells = []; + if (rowNumber == 0) { + for (i = 0; i < cellEls.length; i++) { + dividerCells.push("---" + " |"); + } + dividerCells = "| " + dividerCells.join(" "); + } + for (let i = 0; i < cellEls.length; i++) { + let cell = cellEls[i]; + cell.indexOf(">") + 1; + + cells.push( + cell + .substring(cell.indexOf(">") + 1, cell.indexOf("<", cell.indexOf(">"))) + .replace(/\*\s+([^\|\*]*)/g, "<li>$1</li>") + " | " + ); + } + let row = "| " + cells.join(" "); + if (rowNumber == 0) { + row = row + NL + dividerCells; + } + + return row; +} + +function fix(data) { + let content = data; + let patt1 = /[<]table[\s\S]*?[/]table[>]/g; + let tables = content.match(patt1); + if (tables) { + for (let e of tables) { + let et = e.replace(/\s+/g, " "); + let markdownTable = convertTableElementToMarkdown(et); + content = content.replace(e, markdownTable); + } + } + return content; +} + +function test() { + let data = fs.readFileSync(path.join(__dirname, "../bak/txn-how.md"), "utf8"); + data = fix(data); + fs.writeFileSync(path.join(__dirname, "../bak/txn-how-fixed.md"), data); +} + +// test(); + +module.exports = fix; diff --git a/site2/website-next/migration-scripts/fix-tip-note-in-list.js b/site2/website-next/migration-scripts/fix-tip-note-in-list.js new file mode 100644 index 0000000..81a3cba --- /dev/null +++ b/site2/website-next/migration-scripts/fix-tip-note-in-list.js @@ -0,0 +1,27 @@ +module.exports = (data) => { + let nData = data; + let block = null; + + let uListReg = /^([\*-][\t ]+.*)\n+((\n*[\t ]+.*)*)*/gm; + + while ((block = uListReg.exec(data))) { + let content = block[2]; + if (!content) { + continue; + } + let nContent = content.replace(/^ (?= *)/gm, " "); + nData = nData.replace(content, nContent); + } + + let listReg = /^(\d+\.[\t ]+.*)\n+((\n*[\t ]+.*)*)*/gm; + while ((block = listReg.exec(data))) { + let content = block[2]; + if (!content) { + continue; + } + let nContent = content.replace(/^ (?= *)/gm, " "); + nData = nData.replace(content, nContent); + } + + return nData; +}; diff --git a/site2/website-next/migration-scripts/fix-tip-note.js b/site2/website-next/migration-scripts/fix-tip-note.js new file mode 100644 index 0000000..6becf1a --- /dev/null +++ b/site2/website-next/migration-scripts/fix-tip-note.js @@ -0,0 +1,81 @@ +const fs = require("fs"); +const path = require("path"); + +function fixTip(data) { + const _reg = + /^([\t ]*)>[\t ]*#*[\t ]*\**(Note|Tip)+s?:?\**[\t ]*\n(([\t ]*)>[\t ]*[^><].*\n)+/gm; + const _type_reg = /#*[\t ]*\**(Note|Tip)+s?:?\**[\t ]*/i; + + let type = "note"; + let _tips = data.match(_reg); + if (!_tips) { + return data; + } + for (let _tip of _tips) { + let tip = _tip.replace(/^[ ]>/gm, ">").replace(/^([\t ]*>)[\t ]*/gm, "$1"); + let type_match = _type_reg.exec(tip); + if (type_match) { + type = type_match[1] + ""; + type = type.toLowerCase(); + } + let space = /^([\t ]*)>/.exec(tip)[1]; + // if (space.length % 2 == 1) { + // space = space.substr(1); + // } + tip = tip + .replace(_type_reg, "") + .replace(/^([\t ]*)>[\t ]*/gm, space) + .replace(/^\s*\n/gm, ""); + tip = space + ":::" + type + "\n\n" + tip + "\n" + space + ":::\n"; + data = data.replace(_tip, tip); + } + return data + .replace( + /(.*)(\s*\n){3,}(\s*:::.*)((((?!:::).)*\n*)+):::.*(\s*\n)+/g, + "$1\n\n$3$4:::\n\n" + ) //解决多个空行 + .replace(/(\s*:::.*(((?!:::).)*\n*)+):::.*\n+/g, "$1:::\n\n"); //解决多个空行 //解决代码块结束标识符后无空行 +} + +// function fixListWithTip(data) { +// const _reg = /^([*-]\s+.*)([\s\S]*?)(:::[\s\S]*?:::)/gm; +// let _match = data.match(_reg); +// if (!_match) { +// return data; +// } +// for (let _m of _match) { +// let __m = _m.match(/^([*-]\s+.*)([\s\S]*?)(:::[\s\S]*?:::)/m); +// let top = __m[1]; +// let seconds = __m[2]; +// let tip = __m[3]; +// // seconds = seconds.replace(/^([\t ]*)(\w+)/gm, "$1* $2"); +// tip = tip.replace(/^[\t ]+/gm, ""); +// data = data +// .replace(_m, top + seconds + "\n" + tip) +// .replace(/(\s*\n){3,}:::/g, "\n\n:::") +// .replace(/:::(\s*\n){3,}/g, ":::\n\n"); +// } +// return data; +// } + +function fix(data) { + data = fixTip(data); + // data = fixListWithTip(data); + return data; +} + +function test() { + let data = fs.readFileSync( + path.join(__dirname, "../bak/tiered-storage-overview.md"), + "utf8" + ); + data = fix(data); + fs.writeFileSync( + path.join(__dirname, "../bak/tiered-storage-overview-fixed.md"), + data + ); +} + +// test(); + +module.exports = fix; diff --git a/site2/website-next/migration-scripts/migrate-chapter.js b/site2/website-next/migration-scripts/migrate-chapter.js new file mode 100644 index 0000000..554f1d5 --- /dev/null +++ b/site2/website-next/migration-scripts/migrate-chapter.js @@ -0,0 +1,178 @@ +//Usage: +//Example +//node scripts/migration.js 2.7.3 "Concepts and Architecture" concepts +//node scripts/migration.js 2.7.3 "Concepts and Architecture" concepts fix + +const fs = require("fs"); +const path = require("path"); +const nextSidebar = require("../sidebars"); +const _ = require("lodash"); +const fixMd = require("./fix-md"); +const fixMissing = require("./fix-miss"); + +function travel(dir, callback) { + fs.readdirSync(dir).forEach((file) => { + var pathname = path.join(dir, file); + if (fs.statSync(pathname).isDirectory()) { + // travel(pathname, callback); + } else { + callback(pathname); + } + }); +} + +module.exports = function migrate(version, category, prefix) { + try { + console.log( + "migration docs for ", + "version: ", + version, + "category: ", + category, + "prefix: ", + prefix + ); + + let version_full = "version-" + version; + let src = "../../website/versioned_docs/" + version_full; + let dest = "../../website-next/versioned_docs/" + version_full; + if (version == "next") { + src = "../../docs"; + dest = "../../website-next/docs"; + } + src = path.join(__dirname, src); + dest = path.join(__dirname, dest); + + let sidebar_file = path.join( + __dirname, + "../../website/versioned_sidebars/" + version_full + "-sidebars.json" + ); + if (version == "next") { + sidebar_file = path.join(__dirname, "../../website/sidebars.json"); + } + let sidebar = fs.readFileSync(sidebar_file, "utf8"); + sidebar = JSON.parse(sidebar); + + const _key = version == "next" ? "docs" : version_full + "-docs"; + sidebar = sidebar[_key][category]; + if (version != "next") { + sidebar = sidebar.map((item) => { + return item.substr(version_full.length + 1); + }); + } + if (!sidebar) { + return; + } + let new_sidebar_file = path.join( + __dirname, + "../../website-next/versioned_sidebars/" + version_full + "-sidebars.json" + ); + let new_sidebar = nextSidebar; + if (version == "next") { + new_sidebar_file = path.join( + __dirname, + "../../website-next/sidebars.json" + ); + if (!_.keyBy(new_sidebar.docsSidebar, "label")[category]) { + new_sidebar.docsSidebar.push({ + type: "category", + label: category, + items: sidebar, + }); + } + } else { + try { + new_sidebar = fs.readFileSync(new_sidebar_file, "utf8"); + new_sidebar = JSON.parse(new_sidebar); + } catch { + new_sidebar = { + [version_full + "/docsSidebar"]: [], + }; + } + if ( + !_.keyBy(new_sidebar[version_full + "/docsSidebar"], "label")[category] + ) { + new_sidebar[version_full + "/docsSidebar"].push({ + type: "category", + label: category, + items: sidebar.map((item) => { + return { + type: "doc", + id: + version_full + + "/" + + (item == "deploy-docs" ? "deploy-dcos" : item), + }; + }), + // collapsible: true, + // collapsed: true, + }); + } + } + fs.writeFileSync(new_sidebar_file, JSON.stringify(new_sidebar, null, 2)); + + // console.log("path: ", src, dest); + let migrated_docs = []; + travel(src, function (pathname) { + let filename = path.basename(pathname); + try { + let stat = fs.statSync(dest); + if (!stat.isDirectory()) { + fs.mkdirSync(dest); + } + } catch { + fs.mkdirSync(dest); + } + if ( + sidebar.includes(filename.substr(0, filename.length - 3)) || + (prefix && filename.startsWith(prefix)) + ) { + console.log(filename); + migrated_docs.push(filename.substr(0, filename.length - 3)); + // if (fix) { + let data = fs.readFileSync(pathname, "utf8"); + data = fixMd(data, version); + fs.writeFileSync( + path.join( + dest, + filename + // filename == "deploy-dcos.md" ? "deploy-docs.md" : filename + ), + data + ); + // } else { + // fs.copyFileSync(pathname, path.join(dest, filename)); + // } + } else { + let data = fs.readFileSync(pathname, "utf8"); + let _match = /^id:\s*version-[\d\.]+-(.*)/gm.exec(data); + if (_match && sidebar.includes(_match[1])) { + // filename = _match[1] + '.md'; + // if (fix) { + console.log(filename); + migrated_docs.push(_match[1]); + data = fixMd(data, version); + fs.writeFileSync(path.join(dest, filename), data); + // } else { + // fs.copyFileSync(pathname, path.join(dest, filename)); + // } + } + } + }); + let missing_docs = _.difference(sidebar, migrated_docs); + if (missing_docs.length > 0) { + console.log("fix missing docs:", missing_docs); + missing_docs.forEach((docId) => { + fixMissing(docId, version); + let filename = docId + ".md"; + let _src = path.join(src, filename); + let data = fs.readFileSync(_src, "utf8"); + data = fixMd(data, version); + fs.writeFileSync(path.join(dest, filename), data); + fs.unlinkSync(_src); + }); + } + } catch (e) { + console.error(e); + } +}; diff --git a/site2/website-next/migration-scripts/migrate-md.js b/site2/website-next/migration-scripts/migrate-md.js new file mode 100644 index 0000000..de1d977 --- /dev/null +++ b/site2/website-next/migration-scripts/migrate-md.js @@ -0,0 +1,45 @@ +const fs = require("fs"); +const path = require("path"); +const _ = require("lodash"); +const fixMd = require("./fix-md"); + +const args = process.argv.slice(2); +const src = args[0]; +const dest = args[1]; + +function travel(dir, callback) { + fs.readdirSync(dir).forEach((file) => { + var pathname = path.join(dir, file); + if (fs.statSync(pathname).isDirectory()) { + // travel(pathname, callback); + } else { + callback(pathname); + } + }); +} + +if (!fs.existsSync(src)) { + console.log(src, "is not exists."); + return; +} + +let stat = fs.statSync(src); +if (stat.isDirectory()) { + stat = fs.statSync(dest); + if (!stat.isDirectory()) { + fs.mkdirSync(dest); + } + travel(src, (pathname) => { + let data = fs.readFileSync(pathname, "utf8"); + data = fixMd(data, "blog"); + data = data.replace(/!\[\]\(\.\.\/img\//g, "![](/img/"); + fs.writeFileSync(path.join(dest, path.basename(pathname)), data); + }); +} else { + let data = fs.readFileSync(src, "utf8"); + data = fixMd(data, "blog"); + data = data.replace(/!\[\]\(\.\.\/img\//g, "![](/img/"); + fs.writeFileSync(dest, data); +} + +console.log("Done."); diff --git a/site2/website-next/migration-scripts/migrate-missing.js b/site2/website-next/migration-scripts/migrate-missing.js new file mode 100644 index 0000000..9db85a6 --- /dev/null +++ b/site2/website-next/migration-scripts/migrate-missing.js @@ -0,0 +1,3 @@ +const fix = require("./fix-missing"); +const args = process.argv.slice(2); +fix(args[0], args[1]); diff --git a/site2/website-next/migration-scripts/migrate-override.js b/site2/website-next/migration-scripts/migrate-override.js new file mode 100644 index 0000000..cb13964 --- /dev/null +++ b/site2/website-next/migration-scripts/migrate-override.js @@ -0,0 +1,3 @@ +const fix = require("./override"); +const args = process.argv.slice(2); +fix(args[0]); diff --git a/site2/website-next/migration-scripts/migration.js b/site2/website-next/migration-scripts/migration.js new file mode 100644 index 0000000..39302e2 --- /dev/null +++ b/site2/website-next/migration-scripts/migration.js @@ -0,0 +1,8 @@ +const migrate = require("./migrate-chapter"); + +const args = process.argv.slice(2); +const version = args[0]; +const category = args[1]; +const prefix = args[2]; + +migrate(version, category, prefix); diff --git a/site2/website-next/migration-scripts/migration.sh b/site2/website-next/migration-scripts/migration.sh new file mode 100644 index 0000000..69b4f3a --- /dev/null +++ b/site2/website-next/migration-scripts/migration.sh @@ -0,0 +1,5 @@ +version=2.7.3 +chapter="Cookbooks" +prefix="cookbooks" + +node bak/migration.js $version "$chapter" $prefix fix \ No newline at end of file diff --git a/site2/website-next/migration-scripts/override.js b/site2/website-next/migration-scripts/override.js new file mode 100644 index 0000000..b75d118 --- /dev/null +++ b/site2/website-next/migration-scripts/override.js @@ -0,0 +1,44 @@ +const fs = require("fs"); +const path = require("path"); +const _ = require("lodash"); +const fixMd = require("./fix-md"); + +function travel(dir, callback) { + fs.readdirSync(dir).forEach((file) => { + var pathname = path.join(dir, file); + if (fs.statSync(pathname).isDirectory()) { + // travel(pathname, callback); + // } else if (file == "tiered-storage-filesystem.md") { + } else { + callback(pathname); + } + }); +} + +function fix(version) { + let version_full = "version-" + version; + let src = "../../website/versioned_docs/" + version_full; + let dest = "../../website-next/versioned_docs/" + version_full; + if (version == "next") { + src = "../../docs"; + dest = "../../website-next/docs"; + } + src = path.join(__dirname, src); + dest = path.join(__dirname, dest); + + travel(src, function (pathname) { + let filename = path.basename(pathname); + try { + fs.statSync(dest); + } catch { + fs.mkdirSync(dest); + } + console.log(filename); + let data = fs.readFileSync(pathname, "utf8"); + data = fixMd(data, version); + // filename = filename == "deploy-dcos.md" ? "deploy-docs.md" : filename; + fs.writeFileSync(path.join(dest, filename), data); + }); +} + +module.exports = fix;
