This is an automated email from the ASF dual-hosted git repository.
Startrekzky pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new c057341d1 feat(push-api): implement push API with authentication and
validation logic (#8879)
c057341d1 is described below
commit c057341d13cad9ac308581a1c9f49cfcfd49a846
Author: Klesh Wong <[email protected]>
AuthorDate: Sun May 17 17:15:16 2026 +0800
feat(push-api): implement push API with authentication and validation logic
(#8879)
Co-authored-by: Klesh Wong <[email protected]>
---
backend/server/api/api.go | 6 +-
backend/server/api/middlewares.go | 51 ++++++++++++++-
backend/server/api/middlewares_test.go | 74 ++++++++++++++++++++++
backend/server/api/push/README.md | 8 ++-
backend/server/services/pushapi.go | 21 +-----
backend/server/services/pushapiaccess/access.go | 54 ++++++++++++++++
.../server/services/pushapiaccess/access_test.go | 70 ++++++++++++++++++++
env.example | 4 +-
8 files changed, 264 insertions(+), 24 deletions(-)
diff --git a/backend/server/api/api.go b/backend/server/api/api.go
index 8834c4ecf..32ca49bd8 100644
--- a/backend/server/api/api.go
+++ b/backend/server/api/api.go
@@ -106,9 +106,11 @@ func CreateApiServer() *gin.Engine {
router.GET("/version", version.Get)
// Auth chain order matters: REST API key first (its own short-circuit),
- // then OIDC session, then oauth2-proxy header (only sets USER if not
yet
- // set), then the terminal 401 gate, finally CSRF on unsafe methods.
+ // then the push API key gate, then OIDC session, then oauth2-proxy
header
+ // (only sets USER if not yet set), then the terminal 401 gate, finally
+ // CSRF on unsafe methods.
router.Use(RestAuthentication(router, basicRes))
+ router.Use(RequirePushAuthentication(basicRes))
router.Use(auth.OIDCAuthentication())
router.Use(OAuth2ProxyAuthentication(basicRes))
router.Use(auth.RequireAuth())
diff --git a/backend/server/api/middlewares.go
b/backend/server/api/middlewares.go
index 86c8c6a6f..b6e517847 100644
--- a/backend/server/api/middlewares.go
+++ b/backend/server/api/middlewares.go
@@ -20,12 +20,13 @@ package api
import (
"encoding/base64"
"fmt"
- "github.com/apache/incubator-devlake/core/log"
"net/http"
"regexp"
"strings"
"time"
+ "github.com/apache/incubator-devlake/core/log"
+
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
@@ -113,7 +114,6 @@ func RestAuthentication(router *gin.Engine, basicRes
context.BasicRes) gin.Handl
path := c.Request.URL.Path
// Only open api needs to check api key
if !strings.HasPrefix(path, "/rest") {
- logger.Debug("path %s will continue", path)
c.Next()
return
}
@@ -131,6 +131,53 @@ func RestAuthentication(router *gin.Engine, basicRes
context.BasicRes) gin.Handl
}
}
+func RequirePushAuthentication(basicRes context.BasicRes) gin.HandlerFunc {
+ logger := basicRes.GetLogger()
+ return func(c *gin.Context) {
+ path := c.Request.URL.Path
+ if !strings.HasPrefix(path, "/push/") {
+ c.Next()
+ return
+ }
+
+ authHeader := c.GetHeader("Authorization")
+ if authHeader == "" {
+ c.Abort()
+ c.JSON(http.StatusUnauthorized, &apiBody{
+ Success: false,
+ Message: "token is missing",
+ })
+ return
+ }
+ apiKeyStr := strings.TrimPrefix(authHeader, "Bearer ")
+ if apiKeyStr == authHeader || apiKeyStr == "" {
+ c.Abort()
+ c.JSON(http.StatusUnauthorized, &apiBody{
+ Success: false,
+ Message: "token is not present or malformed",
+ })
+ return
+ }
+
+ db := basicRes.GetDal()
+ if db == nil {
+ logger.Error(nil, "db is not initialised")
+ c.Abort()
+ c.JSON(http.StatusInternalServerError, &apiBody{
+ Success: false,
+ Message: "database is not initialised",
+ })
+ return
+ }
+
+ apiKeyHelper := apikeyhelper.NewApiKeyHelper(basicRes, logger)
+ if !CheckAuthorizationHeader(c, logger, db, apiKeyHelper,
authHeader, path) {
+ return
+ }
+ c.Next()
+ }
+}
+
func CheckAuthorizationHeader(c *gin.Context, logger log.Logger, db dal.Dal,
apiKeyHelper *apikeyhelper.ApiKeyHelper, authHeader, path string) bool {
if authHeader == "" {
c.Abort()
diff --git a/backend/server/api/middlewares_test.go
b/backend/server/api/middlewares_test.go
new file mode 100644
index 000000000..bb6d1f268
--- /dev/null
+++ b/backend/server/api/middlewares_test.go
@@ -0,0 +1,74 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+
+ corectx "github.com/apache/incubator-devlake/core/context"
+ contextimpl "github.com/apache/incubator-devlake/impls/context"
+ "github.com/apache/incubator-devlake/impls/logruslog"
+ "github.com/gin-gonic/gin"
+ "github.com/spf13/viper"
+)
+
+func newPushTestBasicRes() corectx.BasicRes {
+ cfg := viper.New()
+ cfg.Set("ENCRYPTION_SECRET", strings.Repeat("a", 32))
+ return contextimpl.NewDefaultBasicRes(cfg, logruslog.Global, nil)
+}
+
+func TestRequirePushAuthenticationRejectsMissingToken(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ router := gin.New()
+ router.Use(RequirePushAuthentication(newPushTestBasicRes()))
+ router.POST("/push/:tableName", func(c *gin.Context) {
+ c.Status(http.StatusOK)
+ })
+
+ req := httptest.NewRequest(http.MethodPost, "/push/commits",
strings.NewReader(`[{}]`))
+ req.Header.Set("Content-Type", "application/json")
+ resp := httptest.NewRecorder()
+ router.ServeHTTP(resp, req)
+
+ if resp.Code != http.StatusUnauthorized {
+ t.Fatalf("status = %d, want %d", resp.Code,
http.StatusUnauthorized)
+ }
+}
+
+func TestRequirePushAuthenticationRejectsMalformedToken(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ router := gin.New()
+ router.Use(RequirePushAuthentication(newPushTestBasicRes()))
+ router.POST("/push/:tableName", func(c *gin.Context) {
+ c.Status(http.StatusOK)
+ })
+
+ req := httptest.NewRequest(http.MethodPost, "/push/commits",
strings.NewReader(`[{}]`))
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", "Basic dGVzdDp0ZXN0")
+ resp := httptest.NewRecorder()
+ router.ServeHTTP(resp, req)
+
+ if resp.Code != http.StatusUnauthorized {
+ t.Fatalf("status = %d, want %d", resp.Code,
http.StatusUnauthorized)
+ }
+}
diff --git a/backend/server/api/push/README.md
b/backend/server/api/push/README.md
index 999f087b4..f9aa3cb8e 100644
--- a/backend/server/api/push/README.md
+++ b/backend/server/api/push/README.md
@@ -21,6 +21,10 @@ limitations under the License.
This is a generic API service that gives our users the ability to inject data
directly to their own database using a
simple, all-purpose endpoint.
+The push API is disabled by default. To enable it safely, you must:
+- configure `PUSH_API_ALLOWED_TABLES` with the specific non-internal tables
that may be written
+- send a Bearer API key whose `allowedPath` matches the `/push/...` endpoint
you are calling
+
## The Endpoint
POST to ```localhost:8080/push/:tableName```
@@ -28,6 +32,9 @@ POST to ```localhost:8080/push/:tableName```
Where "tableName" is the name of the table you wish to insert into
For example, "commits" would be ```/push/commits```
+Internal `_devlake_*` tables are never writable through this endpoint, even if
+they are listed in `PUSH_API_ALLOWED_TABLES`.
+
## The JSON body
Include a JSON body that consists of an array of objects you wish to insert.
@@ -45,4 +52,3 @@ Please Note: You must know the schema you are inserting into
(column names, type
```
-
diff --git a/backend/server/services/pushapi.go
b/backend/server/services/pushapi.go
index d99d4e906..7882b0aa1 100644
--- a/backend/server/services/pushapi.go
+++ b/backend/server/services/pushapi.go
@@ -18,30 +18,15 @@ limitations under the License.
package services
import (
- "regexp"
- "strings"
-
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/server/services/pushapiaccess"
)
// InsertRow FIXME ...
func InsertRow(table string, rows []map[string]interface{}) (int64,
errors.Error) {
- if !regexp.MustCompile(`^[a-zA-Z0-9_]+$`).MatchString(table) {
- return 0, errors.BadInput.New("table name invalid")
- }
-
- if allowedTables := cfg.GetString("PUSH_API_ALLOWED_TABLES");
allowedTables != "" {
- allow := false
- for _, t := range strings.Split(allowedTables, ",") {
- if strings.TrimSpace(t) == table {
- allow = true
- break
- }
- }
- if !allow {
- return 0, errors.Forbidden.New("table name is not in
the allowed list")
- }
+ if err := pushapiaccess.ValidateTable(table,
cfg.GetString("PUSH_API_ALLOWED_TABLES")); err != nil {
+ return 0, err
}
err := db.Create(rows, dal.From(table))
diff --git a/backend/server/services/pushapiaccess/access.go
b/backend/server/services/pushapiaccess/access.go
new file mode 100644
index 000000000..40f213be1
--- /dev/null
+++ b/backend/server/services/pushapiaccess/access.go
@@ -0,0 +1,54 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package pushapiaccess
+
+import (
+ "regexp"
+ "strings"
+
+ "github.com/apache/incubator-devlake/core/errors"
+)
+
+var tableNameRegex = regexp.MustCompile(`^[a-zA-Z0-9_]+$`)
+
+const internalTablePrefix = "_devlake_"
+
+func ValidateTable(table string, allowedTables string) errors.Error {
+ if !tableNameRegex.MatchString(table) {
+ return errors.BadInput.New("table name invalid")
+ }
+ if strings.HasPrefix(table, internalTablePrefix) {
+ return errors.Forbidden.New("writing internal tables via push
API is forbidden")
+ }
+
+ allowlist := map[string]struct{}{}
+ for _, t := range strings.Split(allowedTables, ",") {
+ name := strings.TrimSpace(t)
+ if name == "" || strings.HasPrefix(name, internalTablePrefix) {
+ continue
+ }
+ allowlist[name] = struct{}{}
+ }
+ if len(allowlist) == 0 {
+ return errors.Forbidden.New("push API is disabled unless
PUSH_API_ALLOWED_TABLES is configured")
+ }
+ if _, ok := allowlist[table]; !ok {
+ return errors.Forbidden.New("table name is not in the allowed
list")
+ }
+ return nil
+}
diff --git a/backend/server/services/pushapiaccess/access_test.go
b/backend/server/services/pushapiaccess/access_test.go
new file mode 100644
index 000000000..969a54365
--- /dev/null
+++ b/backend/server/services/pushapiaccess/access_test.go
@@ -0,0 +1,70 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package pushapiaccess
+
+import "testing"
+
+func TestValidateTable(t *testing.T) {
+ cases := []struct {
+ name string
+ table string
+ allowedTables string
+ wantErr bool
+ }{
+ {
+ name: "allows configured application table",
+ table: "commits",
+ allowedTables: "commits, issues",
+ },
+ {
+ name: "rejects invalid table name",
+ table: "commits;drop",
+ allowedTables: "commits",
+ wantErr: true,
+ },
+ {
+ name: "default denies when allowlist unset",
+ table: "commits",
+ wantErr: true,
+ },
+ {
+ name: "rejects internal tables even when
allowlisted",
+ table: "_devlake_pipelines",
+ allowedTables: "_devlake_pipelines,commits",
+ wantErr: true,
+ },
+ {
+ name: "rejects tables missing from allowlist",
+ table: "pull_requests",
+ allowedTables: "commits,issues",
+ wantErr: true,
+ },
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ err := ValidateTable(tc.table, tc.allowedTables)
+ if tc.wantErr && err == nil {
+ t.Fatal("expected an error but got nil")
+ }
+ if !tc.wantErr && err != nil {
+ t.Fatalf("expected no error, got %v", err)
+ }
+ })
+ }
+}
diff --git a/env.example b/env.example
index 64fa6b39f..460d01b24 100755
--- a/env.example
+++ b/env.example
@@ -34,7 +34,9 @@ SKIP_SUBTASK_PROGRESS=false
PORT=8080
MODE=release
-# PUSH_API_ALLOWED_TABLES=table1,table2
+# Push API is disabled by default. To enable it, list only application tables
+# here. Internal _devlake_* tables are always forbidden.
+PUSH_API_ALLOWED_TABLES=
NOTIFICATION_ENDPOINT=
NOTIFICATION_SECRET=
