This is an automated email from the ASF dual-hosted git repository. maximebeauchemin pushed a commit to branch supersetbot-docker in repository https://gitbox.apache.org/repos/asf/superset.git
commit 29cf485e7bdfd157931047cc1e0b37be48cdd26f Author: Maxime Beauchemin <[email protected]> AuthorDate: Wed Feb 14 16:35:59 2024 -0800 YUP --- .github/supersetbot/src/cli.js | 18 ++ .github/supersetbot/src/docker.js | 141 ++++++++++++ .github/supersetbot/src/docker.test.js | 244 +++++++++++++++++++++ .github/supersetbot/src/index.js | 16 +- .../src/{utils.test.js => utils.index.js} | 0 .github/supersetbot/src/utils.js | 42 +++- 6 files changed, 449 insertions(+), 12 deletions(-) diff --git a/.github/supersetbot/src/cli.js b/.github/supersetbot/src/cli.js index a784a98f60..3d7c6b8f66 100755 --- a/.github/supersetbot/src/cli.js +++ b/.github/supersetbot/src/cli.js @@ -18,6 +18,8 @@ */ import { Command } from 'commander'; import * as commands from './commands.js'; +import * as docker from './docker.js'; +import * as utils from './utils.js'; export default function getCLI(envContext) { const program = new Command(); @@ -58,6 +60,22 @@ export default function getCLI(envContext) { }); await wrapped(opts.repo, opts.issue, envContext); }); + program.command('docker') + .option('-p, --preset <preset>', 'Build preset', /^(lean|dev|dockerize|websocket|py310|ci)$/i, 'lean') + .option('-c, --context <context>', 'Build context', /^(push|pull_request|release)$/i, 'local') + .option('-b, --context-ref <ref>', 'Reference to the PR, release, or branch') + .option('-l, --platform <platform...>', 'Platforms (multiple values allowed)', /^(linux\/arm64|linux\/amd64)$/i, ['linux/amd64']) + .option('-d, --dry-run', 'Run the command in dry-run mode') + .option('-f, --force-latest', 'Force the "latest" tag on the release') + .option('-v, --verbose', 'Print more info') + .action(function () { + const opts = envContext.processOptions(this, ['repo']); + const cmd = docker.getDockerCommand(opts); + console.log(cmd); + if (!opts.dryRun) { + utils.runShellCommand(cmd); + } + }); return program; } diff --git a/.github/supersetbot/src/docker.js b/.github/supersetbot/src/docker.js new file mode 100644 index 0000000000..028698e0d7 --- /dev/null +++ b/.github/supersetbot/src/docker.js @@ -0,0 +1,141 @@ +import { spawnSync } from 'child_process'; + +const REPO = 'apache/superset'; +const CACHE_REPO = `${REPO}-cache`; +const BASE_PY_IMAGE = '3.9-slim-bookworm'; + +export function runCmd(command, raiseOnFailure = true) { + const { stdout, stderr } = spawnSync(command, { shell: true, encoding: 'utf-8', env: process.env }); + + if (stderr && raiseOnFailure) { + throw new Error(stderr); + } + return stdout; +} + +function getGitSha() { + return runCmd('git rev-parse HEAD').trim(); +} + +function getBuildContextRef(buildContext) { + const event = buildContext || process.env.GITHUB_EVENT_NAME; + const githubRef = process.env.GITHUB_REF || ''; + + if (event === 'pull_request') { + const githubHeadRef = process.env.GITHUB_HEAD_REF || ''; + return githubHeadRef.replace(/[^a-zA-Z0-9]/g, '-').slice(0, 40); + } if (event === 'release') { + return githubRef.replace('refs/tags/', '').slice(0, 40); + } if (event === 'push') { + return githubRef.replace('refs/heads/', '').replace(/[^a-zA-Z0-9]/g, '-').slice(0, 40); + } + return ''; +} + +export function isLatestRelease(release) { + const output = runCmd(`../../scripts/tag_latest_release.sh ${release} --dry-run`, false) || ''; + return output.includes('SKIP_TAG::false'); +} + +function makeDockerTag(parts) { + return `${REPO}:${parts.filter((part) => part).join('-')}`; +} + +export function getDockerTags({ + preset, platforms, sha, buildContext, buildContextRef, forceLatest = false, +}) { + const tags = new Set(); + const tagChunks = []; + + const isLatest = isLatestRelease(buildContextRef); + + if (preset !== 'lean') { + tagChunks.push(preset); + } + + if (platforms.length === 1) { + const platform = platforms[0]; + const shortBuildPlatform = platform.replace('linux/', '').replace('64', ''); + if (shortBuildPlatform !== 'amd') { + tagChunks.push(shortBuildPlatform); + } + } + + tags.add(makeDockerTag([sha, ...tagChunks])); + tags.add(makeDockerTag([sha.slice(0, 7), ...tagChunks])); + + if (buildContext === 'release') { + tags.add(makeDockerTag([buildContextRef, ...tagChunks])); + if (isLatest || forceLatest) { + tags.add(makeDockerTag(['latest', ...tagChunks])); + } + } else if (buildContext === 'push' && buildContextRef === 'master') { + tags.add(makeDockerTag(['master', ...tagChunks])); + } else if (buildContext === 'pull_request') { + tags.add(makeDockerTag([`pr-${buildContextRef}`, ...tagChunks])); + } + + return [...tags]; +} + +export function getDockerCommand({ + preset, platform, isAuthenticated, buildContext, buildContextRef, forceLatest = false, +}) { + const platforms = platform; + + let buildTarget = ''; + let pyVer = BASE_PY_IMAGE; + let dockerContext = '.'; + + if (preset === 'dev') { + buildTarget = 'dev'; + } else if (preset === 'lean') { + buildTarget = 'lean'; + } else if (preset === 'py310') { + buildTarget = 'lean'; + pyVer = '3.10-slim-bookworm'; + } else if (preset === 'websocket') { + dockerContext = 'superset-websocket'; + } else if (preset === 'ci') { + buildTarget = 'ci'; + } else if (preset === 'dockerize') { + dockerContext = '-f dockerize.Dockerfile .'; + } else { + console.error(`Invalid build preset: ${preset}`); + process.exit(1); + } + + let ref = buildContextRef; + if (!ref) { + ref = getBuildContextRef(buildContext); + } + const sha = getGitSha(); + const tags = getDockerTags({ + preset, platforms, sha, buildContext, buildContextRef: ref, forceLatest, + }).map((tag) => `-t ${tag}`).join(' \\\n '); + const dockerArgs = isAuthenticated ? '--push' : '--load'; + const targetArgument = buildTarget ? `--target ${buildTarget}` : ''; + const cacheRef = `${CACHE_REPO}:${pyVer}${platforms.length === 1 ? `-${platforms[0].replace('linux/', '').replace('64', '')}` : ''}`; + const platformArg = `--platform ${platforms.join(',')}`; + const cacheFromArg = `--cache-from=type=registry,ref=${cacheRef}`; + const cacheToArg = isAuthenticated ? `--cache-to=type=registry,mode=max,ref=${cacheRef}` : ''; + const buildArg = pyVer ? `--build-arg PY_VER=${pyVer}` : ''; + const actor = process.env.GITHUB_ACTOR; + + return ` + docker buildx build \\ + ${dockerArgs} \\ + ${tags} \\ + ${cacheFromArg} \\ + ${cacheToArg} \\ + ${targetArgument} \\ + ${buildArg} \\ + ${platformArg} \\ + --label sha=${sha} \\ + --label target=${buildTarget} \\ + --label build_trigger=${ref} \\ + --label base=${pyVer} \\ + --label build_actor=${actor} \\ + ${dockerContext} + `; +} diff --git a/.github/supersetbot/src/docker.test.js b/.github/supersetbot/src/docker.test.js new file mode 100644 index 0000000000..7b3c8ee3e7 --- /dev/null +++ b/.github/supersetbot/src/docker.test.js @@ -0,0 +1,244 @@ +import * as dockerUtils from './docker.js'; + +const SHA = '22e7c602b9aa321ec7e0df4bb0033048664dcdf0'; +const PR_ID = '666'; +const OLD_REL = '2.1.0'; +const NEW_REL = '2.1.1'; +const REPO = 'apache/superset'; + +beforeEach(() => { + process.env.TEST_ENV = 'true'; +}); + +afterEach(() => { + delete process.env.TEST_ENV; +}); + +describe('isLatestRelease', () => { + test.each([ + ['2.1.0', false], + ['2.1.1', true], + ['1.0.0', false], + ['3.0.0', true], + ])('returns %s for release %s', (release, expectedBool) => { + expect(dockerUtils.isLatestRelease(release)).toBe(expectedBool); + }); +}); + +describe('getDockerTags', () => { + test.each([ + // PRs + [ + 'lean', + ['linux/arm64'], + SHA, + 'pull_request', + PR_ID, + [`${REPO}:22e7c60-arm`, `${REPO}:${SHA}-arm`, `${REPO}:pr-${PR_ID}-arm`], + ], + [ + 'ci', + ['linux/amd64'], + SHA, + 'pull_request', + PR_ID, + [`${REPO}:22e7c60-ci`, `${REPO}:${SHA}-ci`, `${REPO}:pr-${PR_ID}-ci`], + ], + [ + 'lean', + ['linux/amd64'], + SHA, + 'pull_request', + PR_ID, + [`${REPO}:22e7c60`, `${REPO}:${SHA}`, `${REPO}:pr-${PR_ID}`], + ], + [ + 'dev', + ['linux/arm64'], + SHA, + 'pull_request', + PR_ID, + [ + `${REPO}:22e7c60-dev-arm`, + `${REPO}:${SHA}-dev-arm`, + `${REPO}:pr-${PR_ID}-dev-arm`, + ], + ], + [ + 'dev', + ['linux/amd64'], + SHA, + 'pull_request', + PR_ID, + [`${REPO}:22e7c60-dev`, `${REPO}:${SHA}-dev`, `${REPO}:pr-${PR_ID}-dev`], + ], + // old releases + [ + 'lean', + ['linux/arm64'], + SHA, + 'release', + OLD_REL, + [`${REPO}:22e7c60-arm`, `${REPO}:${SHA}-arm`, `${REPO}:${OLD_REL}-arm`], + ], + [ + 'lean', + ['linux/amd64'], + SHA, + 'release', + OLD_REL, + [`${REPO}:22e7c60`, `${REPO}:${SHA}`, `${REPO}:${OLD_REL}`], + ], + [ + 'dev', + ['linux/arm64'], + SHA, + 'release', + OLD_REL, + [ + `${REPO}:22e7c60-dev-arm`, + `${REPO}:${SHA}-dev-arm`, + `${REPO}:${OLD_REL}-dev-arm`, + ], + ], + [ + 'dev', + ['linux/amd64'], + SHA, + 'release', + OLD_REL, + [`${REPO}:22e7c60-dev`, `${REPO}:${SHA}-dev`, `${REPO}:${OLD_REL}-dev`], + ], + // new releases + [ + 'lean', + ['linux/arm64'], + SHA, + 'release', + NEW_REL, + [ + `${REPO}:22e7c60-arm`, + `${REPO}:${SHA}-arm`, + `${REPO}:${NEW_REL}-arm`, + `${REPO}:latest-arm`, + ], + ], + [ + 'lean', + ['linux/amd64'], + SHA, + 'release', + NEW_REL, + [`${REPO}:22e7c60`, `${REPO}:${SHA}`, `${REPO}:${NEW_REL}`, `${REPO}:latest`], + ], + [ + 'dev', + ['linux/arm64'], + SHA, + 'release', + NEW_REL, + [ + `${REPO}:22e7c60-dev-arm`, + `${REPO}:${SHA}-dev-arm`, + `${REPO}:${NEW_REL}-dev-arm`, + `${REPO}:latest-dev-arm`, + ], + ], + [ + 'dev', + ['linux/amd64'], + SHA, + 'release', + NEW_REL, + [ + `${REPO}:22e7c60-dev`, + `${REPO}:${SHA}-dev`, + `${REPO}:${NEW_REL}-dev`, + `${REPO}:latest-dev`, + ], + ], + // merge on master + [ + 'lean', + ['linux/arm64'], + SHA, + 'push', + 'master', + [`${REPO}:22e7c60-arm`, `${REPO}:${SHA}-arm`, `${REPO}:master-arm`], + ], + [ + 'lean', + ['linux/amd64'], + SHA, + 'push', + 'master', + [`${REPO}:22e7c60`, `${REPO}:${SHA}`, `${REPO}:master`], + ], + [ + 'dev', + ['linux/arm64'], + SHA, + 'push', + 'master', + [ + `${REPO}:22e7c60-dev-arm`, + `${REPO}:${SHA}-dev-arm`, + `${REPO}:master-dev-arm`, + ], + ], + [ + 'dev', + ['linux/amd64'], + SHA, + 'push', + 'master', + [`${REPO}:22e7c60-dev`, `${REPO}:${SHA}-dev`, `${REPO}:master-dev`], + ], + + ])('returns expected tags', (preset, platforms, sha, buildContext, buildContextRef, expectedTags) => { + const tags = dockerUtils.getDockerTags({ + preset, platforms, sha, buildContext, buildContextRef, + }); + expect(tags).toEqual(expect.arrayContaining(expectedTags)); + }); +}); + +describe('getDockerCommand', () => { + test.each([ + [ + 'lean', + ['linux/amd64'], + true, + SHA, + 'push', + 'master', + ['--push', `-t ${REPO}:master `], + ], + [ + 'dev', + ['linux/amd64'], + false, + SHA, + 'push', + 'master', + ['--load', `-t ${REPO}:master-dev `], + ], + // multi-platform + [ + 'lean', + ['linux/arm64', 'linux/amd64'], + true, + SHA, + 'push', + 'master', + ['--platform linux/arm64,linux/amd64'], + ], + ])('returns expected docker command', (preset, platform, isAuthenticated, sha, buildContext, buildContextRef, contains) => { + const cmd = dockerUtils.getDockerCommand({ + preset, platform, isAuthenticated, sha, buildContext, buildContextRef, + }); + contains.forEach((expectedSubstring) => { + expect(cmd).toContain(expectedSubstring); + }); + }); +}); diff --git a/.github/supersetbot/src/index.js b/.github/supersetbot/src/index.js index b518c35ae1..2246e4d5bc 100644 --- a/.github/supersetbot/src/index.js +++ b/.github/supersetbot/src/index.js @@ -16,6 +16,20 @@ * specific language governing permissions and limitations * under the License. */ -import { runCommandFromGithubAction } from './utils.js'; +import { parseArgsStringToArgv } from 'string-argv'; + +import getCLI from './cli.js'; +import Context from './context.js'; + +async function runCommandFromGithubAction(rawCommand) { + const envContext = new Context('GHA'); + const cli = getCLI(envContext); + + // Make rawCommand look like argv + const cmd = rawCommand.trim().replace('@supersetbot', 'supersetbot'); + const args = parseArgsStringToArgv(cmd); + await cli.parseAsync(['node', ...args]); + await envContext.onDone(); +} export { runCommandFromGithubAction }; diff --git a/.github/supersetbot/src/utils.test.js b/.github/supersetbot/src/utils.index.js similarity index 100% rename from .github/supersetbot/src/utils.test.js rename to .github/supersetbot/src/utils.index.js diff --git a/.github/supersetbot/src/utils.js b/.github/supersetbot/src/utils.js index e0cc6a2d62..f0d44fca52 100644 --- a/.github/supersetbot/src/utils.js +++ b/.github/supersetbot/src/utils.js @@ -17,17 +17,37 @@ * under the License. */ -import { parseArgsStringToArgv } from 'string-argv'; -import Context from './context.js'; -import getCLI from './cli.js'; +import { spawn } from 'child_process'; -export async function runCommandFromGithubAction(rawCommand) { - const envContext = new Context('GHA'); - const cli = getCLI(envContext); +export function runShellCommand(command) { + return new Promise((resolve, reject) => { + // Split the command string into an array of arguments + const args = command.split(/\s+/); + const childProcess = spawn(args.shift(), args); - // Make rawCommand look like argv - const cmd = rawCommand.trim().replace('@supersetbot', 'supersetbot'); - const args = parseArgsStringToArgv(cmd); - await cli.parseAsync(['node', ...args]); - await envContext.onDone(); + let stdoutData = ''; + let stderrData = ''; + + // Capture stdout data + childProcess.stdout.on('data', (data) => { + stdoutData += data; + console.log(`stdout: ${data}`); + }); + + // Capture stderr data + childProcess.stderr.on('data', (data) => { + stderrData += data; + console.error(`stderr: ${data}`); + }); + + // Handle process exit + childProcess.on('close', (code) => { + console.log(`child process exited with code ${code}`); + if (code === 0) { + resolve(stdoutData); + } else { + reject(new Error(`Command failed with code ${code}: ${stderrData}`)); + } + }); + }); }
