This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 420e494b4 feat(docker): add Dockerfile.local for local development and
fix migrations (#8866)
420e494b4 is described below
commit 420e494b4ede9a16c4bfb59de099c797bcd7f5f1
Author: Fábio Luciano <[email protected]>
AuthorDate: Tue May 5 06:04:26 2026 -0300
feat(docker): add Dockerfile.local for local development and fix migrations
(#8866)
* fix(migrations): add new fields and improve migration scripts for various
plugins
* feat(docker): add Dockerfile.local for local development and update
docker-compose configuration
* chore: add license to dockerfile
---------
Co-authored-by: Fábio Luciano <[email protected]>
---
backend/Dockerfile | 4 +-
backend/Dockerfile.local | 92 ++++++++++++++++++++++
.../20240821_add_index_to_tool_github_jobs.go | 5 ++
.../20250623_add_display_name_fields.go | 41 +++++++---
.../20251123_add_scope_config_id_to_s3_slice.go | 22 +++---
.../20230316_modify_character_set.go | 34 +++-----
.../20240325_modify_commit_character_type.go | 18 ++---
.../20240508_modify_commit_character_type.go | 16 +---
.../20250629_add_scope_config_id_to_projects.go | 12 ++-
docker-compose-dev.yml | 7 +-
10 files changed, 175 insertions(+), 76 deletions(-)
diff --git a/backend/Dockerfile b/backend/Dockerfile
index 60a31b3eb..4d8fc489e 100644
--- a/backend/Dockerfile
+++ b/backend/Dockerfile
@@ -63,8 +63,8 @@ COPY --from=debian-arm64 /lib/aarch64-linux-gnu
/rootfs-arm64/lib/aarch64-linux-
RUN for arch in aarch64 x86_64 ; do \
mkdir -p /tmp/build/${arch} && cd /tmp/build/${arch} && \
- wget
https://github.com/libgit2/libgit2/archive/refs/tags/v1.3.2.tar.gz -O - | tar
-xz && \
- cd libgit2-1.3.2 && \
+ wget
https://github.com/libgit2/libgit2/archive/refs/tags/v1.3.0.tar.gz -O - | tar
-xz && \
+ cd libgit2-1.3.0 && \
mkdir build && cd build && \
if [ "$arch" = "aarch64" ] ; then \
cmake .. -DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc \
diff --git a/backend/Dockerfile.local b/backend/Dockerfile.local
new file mode 100644
index 000000000..d4dbb0c07
--- /dev/null
+++ b/backend/Dockerfile.local
@@ -0,0 +1,92 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#Apache DevLake is an effort undergoing incubation at The Apache Software
+#Foundation (ASF), sponsored by the Apache Incubator PMC.
+#
+#Incubation is required of all newly accepted projects until a further review
+#indicates that the infrastructure, communications, and decision making process
+#have stabilized in a manner consistent with other successful ASF projects.
+#
+#While incubation status is not necessarily a reflection of the completeness
or stability of the code,
+#it does indicate that the project has yet to be fully endorsed by the ASF.
+
+FROM golang:1.20.5-bookworm as builder
+
+ARG GOPROXY=
+ARG HTTP_PROXY=
+ARG HTTPS_PROXY=
+
+RUN apt-get update && apt-get install -y \
+ gcc \
+ binutils \
+ libfindbin-libs-perl \
+ cmake \
+ libssh2-1-dev \
+ libssl-dev \
+ zlib1g-dev
+
+# Install libgit2 for native platform only
+RUN mkdir -p /tmp/build && cd /tmp/build && \
+ wget https://github.com/libgit2/libgit2/archive/refs/tags/v1.3.0.tar.gz -O
- | tar -xz && \
+ cd libgit2-1.3.0 && \
+ mkdir build && cd build && \
+ cmake .. -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=/usr/local && \
+ make -j$(nproc) install && \
+ ldconfig
+
+RUN go install github.com/vektra/mockery/[email protected]
+RUN go install github.com/swaggo/swag/cmd/[email protected]
+
+WORKDIR /app
+COPY . /app
+ENV GOBIN=/app/bin
+
+ARG TAG=
+ARG SHA=
+
+# Generate mocks and swagger
+RUN make mock
+RUN make swag
+
+# Build plugins
+RUN make build-plugin
+
+# Build server
+RUN VERSION=${TAG}@${SHA} make build-server
+
+FROM debian:bookworm-slim
+
+RUN apt-get update && apt-get install -y \
+ libssh2-1 \
+ libssl3 \
+ ca-certificates \
+ && rm -rf /var/lib/apt/lists/*
+
+# Copy libgit2
+COPY --from=builder /usr/local/lib/libgit2.so* /usr/local/lib/
+RUN ldconfig
+
+WORKDIR /app
+
+COPY --from=builder /app/bin ./bin
+
+# Create empty python plugins dir to avoid startup error
+RUN mkdir -p python/plugins
+
+ENV PORT=8080
+EXPOSE 8080
+
+CMD ["./bin/lake"]
diff --git
a/backend/plugins/github/models/migrationscripts/20240821_add_index_to_tool_github_jobs.go
b/backend/plugins/github/models/migrationscripts/20240821_add_index_to_tool_github_jobs.go
index 4933e557f..43ea38034 100644
---
a/backend/plugins/github/models/migrationscripts/20240821_add_index_to_tool_github_jobs.go
+++
b/backend/plugins/github/models/migrationscripts/20240821_add_index_to_tool_github_jobs.go
@@ -43,6 +43,11 @@ func (script *addIndexToGithubJobs) Up(basicRes
context.BasicRes) errors.Error {
if err := db.Exec(sql); err != nil {
return err
}
+ } else if u.Scheme == "postgres" || u.Scheme == "postgresql" {
+ sql := "CREATE INDEX IF NOT EXISTS idx_repo_id_connection_id ON
_tool_github_jobs (repo_id, connection_id)"
+ if err := db.Exec(sql); err != nil {
+ return err
+ }
}
return nil
}
diff --git
a/backend/plugins/q_dev/models/migrationscripts/20250623_add_display_name_fields.go
b/backend/plugins/q_dev/models/migrationscripts/20250623_add_display_name_fields.go
index e165cfd83..f0c772faf 100644
---
a/backend/plugins/q_dev/models/migrationscripts/20250623_add_display_name_fields.go
+++
b/backend/plugins/q_dev/models/migrationscripts/20250623_add_display_name_fields.go
@@ -21,29 +21,44 @@ import (
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
)
var _ plugin.MigrationScript = (*addDisplayNameFields)(nil)
type addDisplayNameFields struct{}
-func (*addDisplayNameFields) Up(basicRes context.BasicRes) errors.Error {
- db := basicRes.GetDal()
+type QDevConnection20250623 struct {
+ IdentityStoreId string `gorm:"type:VARCHAR(255)"`
+ IdentityStoreRegion string `gorm:"type:VARCHAR(255)"`
+}
+
+func (QDevConnection20250623) TableName() string {
+ return "_tool_q_dev_connections"
+}
+
+type QDevUserData20250623 struct {
+ DisplayName string `gorm:"type:VARCHAR(255)"`
+}
- // Add Identity Center fields to connections table
- // Ignore error if column already exists (MySQL error 1060)
- _ = db.Exec("ALTER TABLE _tool_q_dev_connections ADD COLUMN
identity_store_id VARCHAR(255)")
- _ = db.Exec("ALTER TABLE _tool_q_dev_connections ADD COLUMN
identity_store_region VARCHAR(255)")
+func (QDevUserData20250623) TableName() string {
+ return "_tool_q_dev_user_data"
+}
- // Add display_name column to user_data table
- // Ignore error if column already exists (MySQL error 1060)
- _ = db.Exec("ALTER TABLE _tool_q_dev_user_data ADD COLUMN display_name
VARCHAR(255)")
+type QDevUserMetrics20250623 struct {
+ DisplayName string `gorm:"type:VARCHAR(255)"`
+}
- // Add display_name column to user_metrics table
- // Ignore error if column already exists (MySQL error 1060)
- _ = db.Exec("ALTER TABLE _tool_q_dev_user_metrics ADD COLUMN
display_name VARCHAR(255)")
+func (QDevUserMetrics20250623) TableName() string {
+ return "_tool_q_dev_user_metrics"
+}
- return nil
+func (*addDisplayNameFields) Up(basicRes context.BasicRes) errors.Error {
+ return migrationhelper.AutoMigrateTables(basicRes,
+ &QDevConnection20250623{},
+ &QDevUserData20250623{},
+ &QDevUserMetrics20250623{},
+ )
}
func (*addDisplayNameFields) Version() uint64 {
diff --git
a/backend/plugins/q_dev/models/migrationscripts/20251123_add_scope_config_id_to_s3_slice.go
b/backend/plugins/q_dev/models/migrationscripts/20251123_add_scope_config_id_to_s3_slice.go
index e072e4f59..fdeb10583 100644
---
a/backend/plugins/q_dev/models/migrationscripts/20251123_add_scope_config_id_to_s3_slice.go
+++
b/backend/plugins/q_dev/models/migrationscripts/20251123_add_scope_config_id_to_s3_slice.go
@@ -20,23 +20,21 @@ package migrationscripts
import (
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
)
type addScopeConfigIdToS3Slice struct{}
+type QDevS3Slice20251123 struct {
+ ScopeConfigId uint64 `gorm:"type:BIGINT DEFAULT 0"`
+}
+
+func (QDevS3Slice20251123) TableName() string {
+ return "_tool_q_dev_s3_slices"
+}
+
func (*addScopeConfigIdToS3Slice) Up(basicRes context.BasicRes) errors.Error {
- db := basicRes.GetDal()
-
- // Add scope_config_id column to _tool_q_dev_s3_slices table
- err := db.Exec(`
- ALTER TABLE _tool_q_dev_s3_slices
- ADD COLUMN scope_config_id BIGINT UNSIGNED DEFAULT 0
- `)
- if err != nil {
- return errors.Convert(err)
- }
-
- return nil
+ return migrationhelper.AutoMigrateTables(basicRes,
&QDevS3Slice20251123{})
}
func (*addScopeConfigIdToS3Slice) Version() uint64 {
diff --git
a/backend/plugins/sonarqube/models/migrationscripts/20230316_modify_character_set.go
b/backend/plugins/sonarqube/models/migrationscripts/20230316_modify_character_set.go
index fa648e838..b1f62cd68 100644
---
a/backend/plugins/sonarqube/models/migrationscripts/20230316_modify_character_set.go
+++
b/backend/plugins/sonarqube/models/migrationscripts/20230316_modify_character_set.go
@@ -18,8 +18,6 @@ limitations under the License.
package migrationscripts
import (
- "net/url"
-
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
)
@@ -27,52 +25,46 @@ import (
type modifyCharacterSet struct{}
func (*modifyCharacterSet) Up(basicRes context.BasicRes) errors.Error {
- dbUrl := basicRes.GetConfig("DB_URL")
- if dbUrl == "" {
- return errors.BadInput.New("DB_URL is required")
- }
- u, err1 := url.Parse(dbUrl)
- if err1 != nil {
- return errors.Convert(err1)
- }
- if u.Scheme == "mysql" {
- err := basicRes.GetDal().Exec(`ALTER TABLE
_tool_sonarqube_projects CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`)
+ db := basicRes.GetDal()
+ // Character set conversion is MySQL-specific; PostgreSQL uses UTF-8 by
default
+ if db.Dialect() == "mysql" {
+ err := db.Exec(`ALTER TABLE _tool_sonarqube_projects CONVERT TO
CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`)
if err != nil {
return err
}
- err = basicRes.GetDal().Exec(`ALTER TABLE
_tool_sonarqube_issues CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`)
+ err = db.Exec(`ALTER TABLE _tool_sonarqube_issues CONVERT TO
CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`)
if err != nil {
return err
}
- err = basicRes.GetDal().Exec(`ALTER TABLE
_tool_sonarqube_issue_code_blocks CONVERT TO CHARACTER SET utf8mb4 COLLATE
utf8mb4_bin;`)
+ err = db.Exec(`ALTER TABLE _tool_sonarqube_issue_code_blocks
CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`)
if err != nil {
return err
}
- err = basicRes.GetDal().Exec(`ALTER TABLE
_tool_sonarqube_hotspots CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`)
+ err = db.Exec(`ALTER TABLE _tool_sonarqube_hotspots CONVERT TO
CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`)
if err != nil {
return err
}
- err = basicRes.GetDal().Exec(`ALTER TABLE
_tool_sonarqube_file_metrics CONVERT TO CHARACTER SET utf8mb4 COLLATE
utf8mb4_bin;`)
+ err = db.Exec(`ALTER TABLE _tool_sonarqube_file_metrics CONVERT
TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`)
if err != nil {
return err
}
- err = basicRes.GetDal().Exec(`ALTER TABLE
_tool_sonarqube_accounts CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`)
+ err = db.Exec(`ALTER TABLE _tool_sonarqube_accounts CONVERT TO
CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`)
if err != nil {
return err
}
- err = basicRes.GetDal().Exec(`ALTER TABLE cq_projects CONVERT
TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`)
+ err = db.Exec(`ALTER TABLE cq_projects CONVERT TO CHARACTER SET
utf8mb4 COLLATE utf8mb4_bin;`)
if err != nil {
return err
}
- err = basicRes.GetDal().Exec(`ALTER TABLE cq_issues CONVERT TO
CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`)
+ err = db.Exec(`ALTER TABLE cq_issues CONVERT TO CHARACTER SET
utf8mb4 COLLATE utf8mb4_bin;`)
if err != nil {
return err
}
- err = basicRes.GetDal().Exec(`ALTER TABLE cq_issue_code_blocks
CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`)
+ err = db.Exec(`ALTER TABLE cq_issue_code_blocks CONVERT TO
CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`)
if err != nil {
return err
}
- err = basicRes.GetDal().Exec(`ALTER TABLE cq_file_metrics
CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`)
+ err = db.Exec(`ALTER TABLE cq_file_metrics CONVERT TO CHARACTER
SET utf8mb4 COLLATE utf8mb4_bin;`)
if err != nil {
return err
}
diff --git
a/backend/plugins/sonarqube/models/migrationscripts/20240325_modify_commit_character_type.go
b/backend/plugins/sonarqube/models/migrationscripts/20240325_modify_commit_character_type.go
index 56f058980..1e61f138f 100644
---
a/backend/plugins/sonarqube/models/migrationscripts/20240325_modify_commit_character_type.go
+++
b/backend/plugins/sonarqube/models/migrationscripts/20240325_modify_commit_character_type.go
@@ -18,8 +18,6 @@ limitations under the License.
package migrationscripts
import (
- "net/url"
-
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
)
@@ -27,20 +25,14 @@ import (
type modifyCommitCharacterType struct{}
func (*modifyCommitCharacterType) Up(basicRes context.BasicRes) errors.Error {
- dbUrl := basicRes.GetConfig("DB_URL")
- if dbUrl == "" {
- return errors.BadInput.New("DB_URL is required")
- }
- u, err1 := url.Parse(dbUrl)
- if err1 != nil {
- return errors.Convert(err1)
- }
- if u.Scheme == "mysql" {
- err := basicRes.GetDal().Exec(`ALTER TABLE commits MODIFY
COLUMN message LONGTEXT CHARACTER SET binary;`)
+ db := basicRes.GetDal()
+ // Binary column types are MySQL-specific; PostgreSQL uses bytea
natively
+ if db.Dialect() == "mysql" {
+ err := db.Exec(`ALTER TABLE commits MODIFY COLUMN message
LONGTEXT CHARACTER SET binary;`)
if err != nil {
return err
}
- err = basicRes.GetDal().Exec(`ALTER TABLE commit_files MODIFY
COLUMN file_path VARBINARY(255);`)
+ err = db.Exec(`ALTER TABLE commit_files MODIFY COLUMN file_path
VARBINARY(255);`)
if err != nil {
return err
}
diff --git
a/backend/plugins/sonarqube/models/migrationscripts/20240508_modify_commit_character_type.go
b/backend/plugins/sonarqube/models/migrationscripts/20240508_modify_commit_character_type.go
index 5ba5d668d..1a130170b 100644
---
a/backend/plugins/sonarqube/models/migrationscripts/20240508_modify_commit_character_type.go
+++
b/backend/plugins/sonarqube/models/migrationscripts/20240508_modify_commit_character_type.go
@@ -18,8 +18,6 @@ limitations under the License.
package migrationscripts
import (
- "net/url"
-
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
)
@@ -27,16 +25,10 @@ import (
type modifyCommitCharacterType0508 struct{}
func (*modifyCommitCharacterType0508) Up(basicRes context.BasicRes)
errors.Error {
- dbUrl := basicRes.GetConfig("DB_URL")
- if dbUrl == "" {
- return errors.BadInput.New("DB_URL is required")
- }
- u, err1 := url.Parse(dbUrl)
- if err1 != nil {
- return errors.Convert(err1)
- }
- if u.Scheme == "mysql" {
- err := basicRes.GetDal().Exec(`ALTER TABLE commit_files MODIFY
COLUMN file_path VARBINARY(1024);`)
+ db := basicRes.GetDal()
+ // VARBINARY is MySQL-specific; PostgreSQL uses bytea natively
+ if db.Dialect() == "mysql" {
+ err := db.Exec(`ALTER TABLE commit_files MODIFY COLUMN
file_path VARBINARY(1024);`)
if err != nil {
return err
}
diff --git
a/backend/plugins/testmo/models/migrationscripts/20250629_add_scope_config_id_to_projects.go
b/backend/plugins/testmo/models/migrationscripts/20250629_add_scope_config_id_to_projects.go
index e8970412a..b13d1233c 100644
---
a/backend/plugins/testmo/models/migrationscripts/20250629_add_scope_config_id_to_projects.go
+++
b/backend/plugins/testmo/models/migrationscripts/20250629_add_scope_config_id_to_projects.go
@@ -21,13 +21,21 @@ import (
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
)
type addScopeConfigIdToProjects struct{}
+type TestmoProject20250629 struct {
+ ScopeConfigId uint64 `gorm:"type:BIGINT NOT NULL DEFAULT 0"`
+}
+
+func (TestmoProject20250629) TableName() string {
+ return "_tool_testmo_projects"
+}
+
func (*addScopeConfigIdToProjects) Up(basicRes context.BasicRes) errors.Error {
- db := basicRes.GetDal()
- return db.Exec("ALTER TABLE `_tool_testmo_projects` ADD COLUMN
`scope_config_id` bigint unsigned NOT NULL DEFAULT 0")
+ return migrationhelper.AutoMigrateTables(basicRes,
&TestmoProject20250629{})
}
func (*addScopeConfigIdToProjects) Version() uint64 {
diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml
index b56720db2..b575a6029 100644
--- a/docker-compose-dev.yml
+++ b/docker-compose-dev.yml
@@ -79,9 +79,10 @@ services:
- mysql
devlake:
- image: devlake.docker.scarf.sh/apache/devlake:latest
+ image: devlake-local:latest
build:
context: backend
+ dockerfile: Dockerfile.local
args:
HTTPS_PROXY: "${HTTPS_PROXY}"
GOPROXY: "${GOPROXY}"
@@ -93,6 +94,10 @@ services:
env_file:
- ./.env
environment:
+ DB_URL: postgres://merico:merico@postgres:5432/lake?sslmode=disable
+ E2E_DB_URL:
postgres://merico:merico@postgres:5432/lake_test?sslmode=disable
+ REMOTE_PLUGIN_DIR: ""
+ FORCE_MIGRATION: "true"
LOGGING_DIR: /app/logs
TZ: UTC
# LOGOUT_URI:
https://xxx.amazoncognito.com/logout?client_id=yyy&logout_uri=http%3A%2F%2Flocalhost%3A4180%2Foauth2%2Fsign_out
