This is an automated email from the ASF dual-hosted git repository. klesh pushed a commit to branch feat/single-binary in repository https://gitbox.apache.org/repos/asf/devlake.git
commit 2997094bd212833ba003804af54ee2747ee4cfaa Author: Klesh Wong <[email protected]> AuthorDate: Fri Oct 30 15:23:12 2026 +0800 feat(build): link plugins into the server binary via a generated registry Plugins were only discoverable at runtime, through the .so files in PLUGIN_DIR. Add a compile-time path so the whole plugin set can be linked into one binary: - core/plugin.MustRegister registers a plugin under the name it reports. - tools/pluginsgen scans plugins/*/impl for the type implementing plugin.PluginMeta and generates plugins/all, which imports each plugin and registers it from an init function. It honours DEVLAKE_PLUGINS so subset builds keep working, and emits an empty package for "none". - server/main.go blank-imports the registry. Building with -tags devlake_external_plugins swaps in an empty package for the shared-object-only build. - runner.LoadGoPlugins treats a missing PLUGIN_DIR as "no shared-object plugins" instead of failing startup, so a single-binary deployment does not need to ship an empty directory. - CI regenerates the registry and fails when the checked-in file drifts. Build scripts and Dockerfiles are deliberately untouched: `make build` still produces the .so files, and they register over the compiled-in ones harmlessly. Repackaging follows separately. --- .github/workflows/test.yml | 5 + Makefile | 3 + backend/Makefile | 8 +- backend/core/plugin/hub.go | 21 ++ backend/core/runner/loader.go | 10 + backend/plugins/all/all.go | 113 +++++++++ .../{server/main.go => plugins/all/external.go} | 23 +- backend/server/main.go | 4 + backend/tools/pluginsgen/main.go | 260 +++++++++++++++++++++ 9 files changed, 430 insertions(+), 17 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 21dfc9653..86e2ea95e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -56,6 +56,11 @@ jobs: cd backend go mod tidy git diff --exit-code -- go.mod go.sum + - name: Check plugins/all is up to date + run: | + cd backend + go run ./tools/pluginsgen + git diff --exit-code -- plugins/all - name: Install mockery 3.8.1 run: backend/scripts/install-mockery.sh - name: Build Python diff --git a/Makefile b/Makefile index 8639ce2dc..99cd0d95e 100644 --- a/Makefile +++ b/Makefile @@ -71,6 +71,9 @@ swag: build-plugin: make build-plugin -C backend +plugins-registry: + make plugins-registry -C backend + build-server: make build-server -C backend diff --git a/backend/Makefile b/backend/Makefile index 3e40d4c00..a09f198a2 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -54,13 +54,19 @@ dep: go-dep python-dep swag: mock scripts/swag.sh +# Regenerate plugins/all, the package that links the Go plugins into the server +# binary. Run it after adding, removing or renaming a plugin; CI checks that the +# checked-in file matches the tree. +plugins-registry: + go run ./tools/pluginsgen + build-plugin: scripts/build-plugins.sh build-plugin-debug: DEVLAKE_DEBUG=1 scripts/build-plugins.sh -build-server: swag +build-server: plugins-registry swag VERSION=$(VERSION) scripts/build-server.sh build-python: #don't mix this with the other build commands diff --git a/backend/core/plugin/hub.go b/backend/core/plugin/hub.go index d043600c1..12d89ab17 100644 --- a/backend/core/plugin/hub.go +++ b/backend/core/plugin/hub.go @@ -43,6 +43,27 @@ func RegisterPlugin(name string, plugin PluginMeta) errors.Error { return nil } +// MustRegister registers p under the name it reports, panicking if the plugin +// cannot be identified. +// +// The generated plugins/all package calls it for every plugin that is compiled +// into the binary, which makes the plugin set a build-time decision. Registering +// a name twice overwrites the previous entry, so a plugin that is both compiled +// in and loaded from a shared object keeps working: the loader registers the +// same value again. +func MustRegister(p PluginMeta) { + if p == nil { + panic("plugin: MustRegister called with a nil plugin") + } + name := p.Name() + if name == "" { + panic("plugin: MustRegister called with an empty plugin name") + } + if err := RegisterPlugin(name, p); err != nil { + panic(err) + } +} + func GetPlugin(name string) (PluginMeta, errors.Error) { if plugins == nil { return nil, errors.Default.New("RegisterPlugin have never been called.") diff --git a/backend/core/runner/loader.go b/backend/core/runner/loader.go index f05a78632..f3541e870 100644 --- a/backend/core/runner/loader.go +++ b/backend/core/runner/loader.go @@ -20,6 +20,7 @@ package runner import ( "fmt" "io/fs" + "os" "path/filepath" goplugin "plugin" "strings" @@ -50,6 +51,15 @@ func LoadPlugins(basicRes context.BasicRes) errors.Error { func LoadGoPlugins(basicRes context.BasicRes) errors.Error { pluginsDir := basicRes.GetConfig("PLUGIN_DIR") + if pluginsDir == "" { + return nil + } + // A single-binary deployment has no plugin directory at all: the plugins are + // already registered by the generated plugins/all package. + if _, statErr := os.Stat(pluginsDir); os.IsNotExist(statErr) { + basicRes.GetLogger().Info("plugin directory %s does not exist, skipping shared-object plugins", pluginsDir) + return nil + } var wg sync.WaitGroup walkErr := filepath.WalkDir(pluginsDir, func(path string, d fs.DirEntry, err error) error { if err != nil { diff --git a/backend/plugins/all/all.go b/backend/plugins/all/all.go new file mode 100644 index 000000000..7f37819cd --- /dev/null +++ b/backend/plugins/all/all.go @@ -0,0 +1,113 @@ +//go:build !devlake_external_plugins + +// Code generated by tools/pluginsgen. DO NOT EDIT. + +// Package all links the Go plugins into the server binary. +// +// Importing this package registers every plugin listed below, so the plugin +// set becomes a build-time decision. Build with the devlake_external_plugins +// tag to leave it empty and load plugins from PLUGIN_DIR as shared objects +// instead. +package all + +import ( + "github.com/apache/devlake/core/plugin" + + ae "github.com/apache/devlake/plugins/ae/impl" + argocd "github.com/apache/devlake/plugins/argocd/impl" + asana "github.com/apache/devlake/plugins/asana/impl" + azuredevops_go "github.com/apache/devlake/plugins/azuredevops_go/impl" + bamboo "github.com/apache/devlake/plugins/bamboo/impl" + bitbucket "github.com/apache/devlake/plugins/bitbucket/impl" + bitbucket_server "github.com/apache/devlake/plugins/bitbucket_server/impl" + circleci "github.com/apache/devlake/plugins/circleci/impl" + claude_code "github.com/apache/devlake/plugins/claude_code/impl" + clickup "github.com/apache/devlake/plugins/clickup/impl" + cursor "github.com/apache/devlake/plugins/cursor/impl" + customize "github.com/apache/devlake/plugins/customize/impl" + dora "github.com/apache/devlake/plugins/dora/impl" + feishu "github.com/apache/devlake/plugins/feishu/impl" + gh_copilot "github.com/apache/devlake/plugins/gh-copilot/impl" + gitee "github.com/apache/devlake/plugins/gitee/impl" + gitextractor "github.com/apache/devlake/plugins/gitextractor/impl" + github "github.com/apache/devlake/plugins/github/impl" + github_graphql "github.com/apache/devlake/plugins/github_graphql/impl" + gitlab "github.com/apache/devlake/plugins/gitlab/impl" + grafana_irm "github.com/apache/devlake/plugins/grafana_irm/impl" + icla "github.com/apache/devlake/plugins/icla/impl" + incidentio "github.com/apache/devlake/plugins/incidentio/impl" + issue_trace "github.com/apache/devlake/plugins/issue_trace/impl" + jenkins "github.com/apache/devlake/plugins/jenkins/impl" + jira "github.com/apache/devlake/plugins/jira/impl" + kiro "github.com/apache/devlake/plugins/kiro/impl" + linear "github.com/apache/devlake/plugins/linear/impl" + linker "github.com/apache/devlake/plugins/linker/impl" + monorepo "github.com/apache/devlake/plugins/monorepo/impl" + opsgenie "github.com/apache/devlake/plugins/opsgenie/impl" + org "github.com/apache/devlake/plugins/org/impl" + pagerduty "github.com/apache/devlake/plugins/pagerduty/impl" + refdiff "github.com/apache/devlake/plugins/refdiff/impl" + rootly "github.com/apache/devlake/plugins/rootly/impl" + slack "github.com/apache/devlake/plugins/slack/impl" + sonarqube "github.com/apache/devlake/plugins/sonarqube/impl" + starrocks "github.com/apache/devlake/plugins/starrocks/impl" + taiga "github.com/apache/devlake/plugins/taiga/impl" + tapd "github.com/apache/devlake/plugins/tapd/impl" + teambition "github.com/apache/devlake/plugins/teambition/impl" + tempo "github.com/apache/devlake/plugins/tempo/impl" + testmo "github.com/apache/devlake/plugins/testmo/impl" + trello "github.com/apache/devlake/plugins/trello/impl" + webhook "github.com/apache/devlake/plugins/webhook/impl" + youtrack "github.com/apache/devlake/plugins/youtrack/impl" + zentao "github.com/apache/devlake/plugins/zentao/impl" +) + +func init() { + plugin.MustRegister(ae.AE{}) + plugin.MustRegister(argocd.ArgoCD{}) + plugin.MustRegister(asana.Asana{}) + plugin.MustRegister(azuredevops_go.Azuredevops{}) + plugin.MustRegister(bamboo.Bamboo{}) + plugin.MustRegister(bitbucket.Bitbucket{}) + plugin.MustRegister(bitbucket_server.BitbucketServer{}) + plugin.MustRegister(circleci.Circleci{}) + plugin.MustRegister(claude_code.ClaudeCode{}) + plugin.MustRegister(clickup.ClickUp{}) + plugin.MustRegister(cursor.Cursor{}) + plugin.MustRegister(customize.Customize{}) + plugin.MustRegister(dora.Dora{}) + plugin.MustRegister(feishu.Feishu{}) + plugin.MustRegister(gh_copilot.GhCopilot{}) + plugin.MustRegister(gitee.Gitee{}) + plugin.MustRegister(gitextractor.GitExtractor{}) + plugin.MustRegister(github.Github{}) + plugin.MustRegister(github_graphql.GithubGraphql{}) + plugin.MustRegister(gitlab.Gitlab{}) + plugin.MustRegister(grafana_irm.GrafanaIrm{}) + plugin.MustRegister(icla.Icla{}) + plugin.MustRegister(incidentio.Incidentio{}) + plugin.MustRegister(issue_trace.IssueTrace{}) + plugin.MustRegister(jenkins.Jenkins{}) + plugin.MustRegister(jira.Jira{}) + plugin.MustRegister(kiro.Kiro{}) + plugin.MustRegister(linear.Linear{}) + plugin.MustRegister(linker.Linker{}) + plugin.MustRegister(monorepo.Monorepo{}) + plugin.MustRegister(opsgenie.Opsgenie{}) + plugin.MustRegister(org.Org{}) + plugin.MustRegister(pagerduty.PagerDuty{}) + plugin.MustRegister(refdiff.RefDiff{}) + plugin.MustRegister(rootly.Rootly{}) + plugin.MustRegister(slack.Slack{}) + plugin.MustRegister(sonarqube.Sonarqube{}) + plugin.MustRegister(starrocks.StarRocks{}) + plugin.MustRegister(taiga.Taiga{}) + plugin.MustRegister(tapd.Tapd{}) + plugin.MustRegister(teambition.Teambition{}) + plugin.MustRegister(tempo.Tempo{}) + plugin.MustRegister(testmo.Testmo{}) + plugin.MustRegister(trello.Trello{}) + plugin.MustRegister(webhook.Webhook{}) + plugin.MustRegister(youtrack.Youtrack{}) + plugin.MustRegister(zentao.Zentao{}) +} diff --git a/backend/server/main.go b/backend/plugins/all/external.go similarity index 64% copy from backend/server/main.go copy to backend/plugins/all/external.go index 7a52c6597..414c8118c 100644 --- a/backend/server/main.go +++ b/backend/plugins/all/external.go @@ -15,20 +15,11 @@ See the License for the specific language governing permissions and limitations under the License. */ -package main +//go:build devlake_external_plugins -import ( - "github.com/apache/devlake/core/config" - "github.com/apache/devlake/core/plugin" - _ "github.com/apache/devlake/core/version" - "github.com/apache/devlake/server/api" -) - -func main() { - v := config.GetConfig() - encryptionSecret := v.GetString(plugin.EncodeKeyEnvStr) - if encryptionSecret == "" { - panic("ENCRYPTION_SECRET must be set in environment variable or .env file") - } - api.CreateAndRunApiServer() -} +// Package all is empty when plugins are loaded from shared objects at runtime. +// +// Build with -tags devlake_external_plugins (and leave PLUGIN_DIR pointing at +// the output of scripts/build-plugins.sh) to ship a plugin-free core binary and +// load plugins as .so files instead. +package all diff --git a/backend/server/main.go b/backend/server/main.go index 7a52c6597..5003dc383 100644 --- a/backend/server/main.go +++ b/backend/server/main.go @@ -21,6 +21,10 @@ import ( "github.com/apache/devlake/core/config" "github.com/apache/devlake/core/plugin" _ "github.com/apache/devlake/core/version" + // Linking the generated registry compiles every plugin into this binary and + // registers it at startup. Build with -tags devlake_external_plugins to + // leave the registry empty and load plugins from PLUGIN_DIR instead. + _ "github.com/apache/devlake/plugins/all" "github.com/apache/devlake/server/api" ) diff --git a/backend/tools/pluginsgen/main.go b/backend/tools/pluginsgen/main.go new file mode 100644 index 000000000..c19346c34 --- /dev/null +++ b/backend/tools/pluginsgen/main.go @@ -0,0 +1,260 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Command pluginsgen generates plugins/all, the package that links the Go +// plugins into the server binary. +// +// It scans plugins/*/impl for the type implementing plugin.PluginMeta and emits +// one import plus one plugin.MustRegister call per plugin. Keeping the registry +// generated means adding a plugin needs no edit to a shared list, and CI can +// assert that the checked-in file matches the tree. +package main + +import ( + "bytes" + "flag" + "fmt" + "go/ast" + "go/format" + "go/parser" + "go/token" + "os" + "path/filepath" + "sort" + "strings" +) + +const modulePath = "github.com/apache/devlake" + +// pluginMetaMethods are the methods declared by plugin.PluginMeta. Requiring +// all of them keeps the scan from picking an unrelated Name method declared on +// some other type in the same package. +var pluginMetaMethods = []string{"Name", "Description", "RootPkgPath"} + +type plugin struct { + name string // directory name, e.g. gh-copilot + alias string // import alias, e.g. gh_copilot + typ string // type implementing plugin.PluginMeta +} + +func main() { + root := flag.String("root", ".", "backend module root") + out := flag.String("o", "plugins/all/all.go", "output file, relative to root") + subset := flag.String("plugins", os.Getenv("DEVLAKE_PLUGINS"), "comma-separated plugin names to include (default: $DEVLAKE_PLUGINS, otherwise all)") + flag.Parse() + + if err := run(*root, *out, *subset); err != nil { + fmt.Fprintf(os.Stderr, "pluginsgen: %v\n", err) + os.Exit(1) + } +} + +func run(root, out, subset string) error { + plugins, err := scan(root, subset) + if err != nil { + return err + } + + src, err := render(plugins) + if err != nil { + return err + } + + target := out + if !filepath.IsAbs(target) { + target = filepath.Join(root, out) + } + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + return os.WriteFile(target, src, 0o644) +} + +// scan returns the plugins to link, sorted by name. +func scan(root, subset string) ([]plugin, error) { + dirs, err := filepath.Glob(filepath.Join(root, "plugins", "*", "impl")) + if err != nil { + return nil, err + } + if len(dirs) == 0 { + return nil, fmt.Errorf("no plugins found under %s/plugins/*/impl", root) + } + + selected, explicit := parseSubset(subset) + + var plugins []plugin + for _, dir := range dirs { + name := filepath.Base(filepath.Dir(dir)) + if !selected(name) { + continue + } + typ, err := pluginMetaType(dir) + if err != nil { + return nil, fmt.Errorf("%s: %w", name, err) + } + plugins = append(plugins, plugin{ + name: name, + alias: strings.ReplaceAll(name, "-", "_"), + typ: typ, + }) + } + + // An explicit selection may legitimately match nothing ("none" builds a + // core with no compiled-in plugins); an unfiltered scan must not. + if len(plugins) == 0 && !explicit { + return nil, fmt.Errorf("no plugin found under %s/plugins/*/impl", root) + } + sort.Slice(plugins, func(i, j int) bool { return plugins[i].name < plugins[j].name }) + return plugins, nil +} + +// parseSubset builds the selection predicate from DEVLAKE_PLUGINS. An empty +// value selects every plugin; "none" selects none, mirroring build-plugins.sh. +// The second result reports whether a selection was requested at all. +func parseSubset(subset string) (func(string) bool, bool) { + subset = strings.TrimSpace(subset) + switch subset { + case "": + return func(string) bool { return true }, false + case "none": + return func(string) bool { return false }, true + } + + wanted := map[string]struct{}{} + for _, name := range strings.Split(subset, ",") { + if name = strings.TrimSpace(name); name != "" { + wanted[name] = struct{}{} + } + } + return func(name string) bool { + _, ok := wanted[name] + return ok + }, true +} + +// pluginMetaType finds the type in dir whose method set covers +// plugin.PluginMeta. +func pluginMetaType(dir string) (string, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return "", err + } + + fset := token.NewFileSet() + methods := map[string]map[string]struct{}{} + for _, entry := range entries { + name := entry.Name() + if entry.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { + continue + } + file, err := parser.ParseFile(fset, filepath.Join(dir, name), nil, parser.SkipObjectResolution) + if err != nil { + return "", err + } + for _, decl := range file.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Recv == nil || len(fn.Recv.List) != 1 { + continue + } + receiver := receiverName(fn.Recv.List[0].Type) + if receiver == "" { + continue + } + if methods[receiver] == nil { + methods[receiver] = map[string]struct{}{} + } + methods[receiver][fn.Name.Name] = struct{}{} + } + } + + var found []string + for receiver, declared := range methods { + implements := true + for _, method := range pluginMetaMethods { + if _, ok := declared[method]; !ok { + implements = false + break + } + } + if implements { + found = append(found, receiver) + } + } + + switch len(found) { + case 1: + return found[0], nil + case 0: + return "", fmt.Errorf("no type implements plugin.PluginMeta") + default: + sort.Strings(found) + return "", fmt.Errorf("several types implement plugin.PluginMeta: %s", strings.Join(found, ", ")) + } +} + +func receiverName(expr ast.Expr) string { + switch t := expr.(type) { + case *ast.Ident: + return t.Name + case *ast.StarExpr: + return receiverName(t.X) + case *ast.IndexExpr: // generic receiver, e.g. Foo[T] + return receiverName(t.X) + case *ast.IndexListExpr: + return receiverName(t.X) + } + return "" +} + +func render(plugins []plugin) ([]byte, error) { + var buf bytes.Buffer + buf.WriteString("//go:build !devlake_external_plugins\n\n") + buf.WriteString("// Code generated by tools/pluginsgen. DO NOT EDIT.\n\n") + buf.WriteString("// Package all links the Go plugins into the server binary.\n") + buf.WriteString("//\n") + buf.WriteString("// Importing this package registers every plugin listed below, so the plugin\n") + buf.WriteString("// set becomes a build-time decision. Build with the devlake_external_plugins\n") + buf.WriteString("// tag to leave it empty and load plugins from PLUGIN_DIR as shared objects\n") + buf.WriteString("// instead.\n") + buf.WriteString("package all\n\n") + + if len(plugins) == 0 { + src, err := format.Source(buf.Bytes()) + if err != nil { + return nil, fmt.Errorf("formatting generated source: %w", err) + } + return src, nil + } + + buf.WriteString("import (\n") + buf.WriteString("\t\"github.com/apache/devlake/core/plugin\"\n\n") + for _, p := range plugins { + fmt.Fprintf(&buf, "\t%s \"%s/plugins/%s/impl\"\n", p.alias, modulePath, p.name) + } + buf.WriteString(")\n\n") + buf.WriteString("func init() {\n") + for _, p := range plugins { + fmt.Fprintf(&buf, "\tplugin.MustRegister(%s.%s{})\n", p.alias, p.typ) + } + buf.WriteString("}\n") + + src, err := format.Source(buf.Bytes()) + if err != nil { + return nil, fmt.Errorf("formatting generated source: %w", err) + } + return src, nil +}
