lshmouse opened a new issue, #2989: URL: https://github.com/apache/incubator-devlake/issues/2989
### Search before asking - [X] I had searched in the [issues](https://github.com/apache/incubator-devlake/issues?q=is%3Aissue) and found no similar issues. ### What happened The devlake container crashed for the fatal error: concurrent map read and map write when scheduling a blueprine manully ``` time="2022-09-05 09:47:58" level=info msg=" [pipeline service] run pipeline, 17" time="2022-09-05 09:47:59" level=info msg=" [task service] run task in background %!(EXTRA uint64=25)" time="2022-09-05 09:47:59" level=info msg=" [task service] run task in background %!(EXTRA uint64=26)" fatal error: concurrent map read and map write goroutine 627 [running]: runtime.throw({0x1787478, 0xc0003da240}) /usr/local/go/src/runtime/panic.go:1198 +0x71 fp=0xc0006086e8 sp=0xc0006086b8 pc=0x8c0291 runtime.mapaccess1_fast64(0x9890a6, 0x16125c0, 0x19) /usr/local/go/src/runtime/map_fast64.go:21 +0x172 fp=0xc000608708 sp=0xc0006086e8 pc=0x89da92 github.com/apache/incubator-devlake/services.updateTaskProgress(0xce6006, 0xc00043bc00) /app/services/task.go:277 +0x49 fp=0xc0006087c0 sp=0xc000608708 pc=0x1443029 github.com/apache/incubator-devlake/services.runTaskStandalone·dwrap·8() /app/services/task.go:263 +0x2a fp=0xc0006087e0 sp=0xc0006087c0 pc=0x1442f6a runtime.goexit() /usr/local/go/src/runtime/asm_amd64.s:1581 +0x1 fp=0xc0006087e8 sp=0xc0006087e0 pc=0x8f0e01 created by github.com/apache/incubator-devlake/services.runTaskStandalone /app/services/task.go:263 +0x112 goroutine 1 [IO wait]: internal/poll.runtime_pollWait(0x7f4e02ca8f70, 0x72) /usr/local/go/src/runtime/netpoll.go:234 +0x89 internal/poll.(*pollDesc).wait(0xc00043ad80, 0xc000057800, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Accept(0xc00043ad80) /usr/local/go/src/internal/poll/fd_unix.go:402 +0x22c net.(*netFD).accept(0xc00043ad80) /usr/local/go/src/net/fd_unix.go:173 +0x35 net.(*TCPListener).accept(0xc00000ff38) /usr/local/go/src/net/tcpsock_posix.go:140 +0x28 net.(*TCPListener).Accept(0xc00000ff38) /usr/local/go/src/net/tcpsock.go:262 +0x3d net/http.(*Server).Serve(0xc0006300e0, {0x1927400, 0xc00000ff38}) /usr/local/go/src/net/http/server.go:3002 +0x394 net/http.(*Server).ListenAndServe(0xc0006300e0) /usr/local/go/src/net/http/server.go:2931 +0x7d net/http.ListenAndServe(...) /usr/local/go/src/net/http/server.go:3185 github.com/gin-gonic/gin.(*Engine).Run(0xc0000c9040, {0xc0005a5e98, 0x1, 0x1}) /go/pkg/mod/github.com/gin-gonic/[email protected]/gin.go:356 +0x1d8 github.com/apache/incubator-devlake/api.CreateApiService() /app/api/api.go:97 +0x4a5 main.main() /app/main.go:39 +0x7e goroutine 18 [semacquire, 5 minutes]: sync.runtime_Semacquire(0xc351a0) /usr/local/go/src/runtime/sema.go:56 +0x25 sync.(*WaitGroup).Wait(0xc00012cfa0) /usr/local/go/src/sync/waitgroup.go:130 +0x71 github.com/spf13/viper.(*Viper).WatchConfig.func1() /go/pkg/mod/github.com/spf13/[email protected]/viper.go:410 +0x306 created by github.com/spf13/viper.(*Viper).WatchConfig /go/pkg/mod/github.com/spf13/[email protected]/viper.go:349 +0x92 goroutine 19 [syscall, 5 minutes]: syscall.Syscall6(0xe8, 0x7, 0xc00028fbec, 0x7, 0xffffffffffffffff, 0x0, 0x0) /usr/local/go/src/syscall/asm_linux_amd64.s:43 +0x5 golang.org/x/sys/unix.EpollWait(0x0, {0xc00028fbec, 0x0, 0x0}, 0x0) /go/pkg/mod/golang.org/x/[email protected]/unix/zsyscall_linux_amd64.go:56 +0x58 github.com/fsnotify/fsnotify.(*fdPoller).wait(0xc000121300) /go/pkg/mod/github.com/fsnotify/[email protected]/inotify_poller.go:87 +0x7d github.com/fsnotify/fsnotify.(*Watcher).readEvents(0xc00012cfa0) /go/pkg/mod/github.com/fsnotify/[email protected]/inotify.go:193 +0x2b0 created by github.com/fsnotify/fsnotify.NewWatcher /go/pkg/mod/github.com/fsnotify/[email protected]/inotify.go:60 +0x1c7 goroutine 20 [select, 5 minutes]: github.com/spf13/viper.(*Viper).WatchConfig.func1.1() /go/pkg/mod/github.com/spf13/[email protected]/viper.go:371 +0x10b created by github.com/spf13/viper.(*Viper).WatchConfig.func1 /go/pkg/mod/github.com/spf13/[email protected]/viper.go:369 +0x2d6 goroutine 24 [chan receive]: github.com/panjf2000/ants/v2.(*Pool).purgePeriodically(0xc000314230) /go/pkg/mod/github.com/panjf2000/ants/[email protected]/pool.go:69 +0x8b created by github.com/panjf2000/ants/v2.NewPool /go/pkg/mod/github.com/panjf2000/ants/[email protected]/pool.go:137 +0x31b goroutine 7 [select, 5 minutes]: database/sql.(*DB).connectionOpener(0xc000518340, {0x1931cb8, 0xc0000a7200}) /usr/local/go/src/database/sql/sql.go:1196 +0x93 created by database/sql.OpenDB /usr/local/go/src/database/sql/sql.go:794 +0x188 goroutine 55 [select, 5 minutes]: github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1() /go/pkg/mod/github.com/go-sql-driver/[email protected]/connection.go:614 +0xb0 created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher /go/pkg/mod/github.com/go-sql-driver/[email protected]/connection.go:611 +0x105 goroutine 56 [select, 5 minutes]: database/sql.(*DB).connectionCleaner(0xc000518340, 0x0) /usr/local/go/src/database/sql/sql.go:1068 +0xbd created by database/sql.(*DB).startCleanerLocked /usr/local/go/src/database/sql/sql.go:1055 +0x105 goroutine 216 [select, 4 minutes]: github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1() /go/pkg/mod/github.com/go-sql-driver/[email protected]/connection.go:614 +0xb0 created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher /go/pkg/mod/github.com/go-sql-driver/[email protected]/connection.go:611 +0x105 goroutine 83 [select, 5 minutes]: github.com/robfig/cron/v3.(*Cron).run(0xc000318320) /go/pkg/mod/github.com/robfig/cron/[email protected]/cron.go:258 +0xade created by github.com/robfig/cron/v3.(*Cron).Start /go/pkg/mod/github.com/robfig/cron/[email protected]/cron.go:217 +0xcf goroutine 84 [sleep]: time.Sleep(0x3b9aca00) /usr/local/go/src/runtime/time.go:193 +0x12e github.com/apache/incubator-devlake/services.RunPipelineInQueue(0x2710) /app/services/pipeline.go:219 +0x1b4 created by github.com/apache/incubator-devlake/services.pipelineServiceInit /app/services/pipeline.go:91 +0x416 goroutine 642 [chan receive]: github.com/apache/incubator-devlake/services.updateTaskProgress(0xb49da6, 0xc002821620) /app/services/task.go:282 +0x114 created by github.com/apache/incubator-devlake/services.runTaskStandalone /app/services/task.go:263 +0x112 goroutine 608 [runnable]: github.com/go-sql-driver/mysql.(*buffer).readNext(0xc0000da6c0, 0xc) /go/pkg/mod/github.com/go-sql-driver/[email protected]/buffer.go:116 +0xb5 github.com/go-sql-driver/mysql.(*mysqlConn).readPacket(0xc0000da6c0) /go/pkg/mod/github.com/go-sql-driver/[email protected]/packets.go:68 +0x10a github.com/go-sql-driver/mysql.(*mysqlStmt).readPrepareResultPacket(0xc001bd22a0) /go/pkg/mod/github.com/go-sql-driver/[email protected]/packets.go:830 +0x28 github.com/go-sql-driver/mysql.(*mysqlConn).Prepare(0xc0000da6c0, {0xc002497180, 0xc00004c030}) /go/pkg/mod/github.com/go-sql-driver/[email protected]/connection.go:182 +0x18f github.com/go-sql-driver/mysql.(*mysqlConn).PrepareContext(0xa, {0x1931cf0, 0xc00004c030}, {0xc002497180, 0x3e}) /go/pkg/mod/github.com/go-sql-driver/[email protected]/connection.go:533 +0x53 database/sql.ctxDriverPrepare({0x1931cf0, 0xc00004c030}, {0x1921700, 0xc0000da6c0}, {0xc002497180, 0x3e}) /usr/local/go/src/database/sql/ctxutil.go:15 +0x11b database/sql.(*driverConn).prepareLocked(0xc0005ba000, {0x1931cf0, 0xc00004c030}, {0x0, 0x0}, {0xc002497180, 0xc00283f7f8}) /usr/local/go/src/database/sql/sql.go:578 +0x58 database/sql.(*DB).prepareDC.func2() /usr/local/go/src/database/sql/sql.go:1572 +0x4b database/sql.withLock({0x191e678, 0xc0005ba000}, 0xc00283f8d0) /usr/local/go/src/database/sql/sql.go:3396 +0x8c database/sql.(*DB).prepareDC(0xc000518340, {0x1931cf0, 0xc00004c030}, 0xc0005ba000, 0x28, {0x0, 0x0}, {0xc002497180, 0x3e}) /usr/local/go/src/database/sql/sql.go:1571 +0x186 database/sql.(*DB).prepare(0xc000436f50, {0x1931cf0, 0xc00004c030}, {0xc002497180, 0x3e}, 0x0) /usr/local/go/src/database/sql/sql.go:1559 +0xa5 database/sql.(*DB).PrepareContext(0x15f0ae0, {0x1931cf0, 0xc00004c030}, {0xc002497180, 0x3e}) /usr/local/go/src/database/sql/sql.go:1525 +0xa5 gorm.io/gorm.(*PreparedStmtDB).prepare(0xc000424300, {0x1931cf0, 0xc00004c030}, {0x1931e78, 0xc000518340}, 0x0, {0xc002497180, 0x17591ff}) /go/pkg/mod/gorm.io/[email protected]/prepare_stmt.go:63 +0x31e gorm.io/gorm.(*PreparedStmtDB).QueryContext(0xc000424300, {0x1931cf0, 0xc00004c030}, {0xc002497180, 0x3e}, {0xc00004e980, 0x1, 0x8}) /go/pkg/mod/gorm.io/[email protected]/prepare_stmt.go:95 +0xab gorm.io/gorm/callbacks.Query(0xc00077d0e0) /go/pkg/mod/gorm.io/[email protected]/callbacks/query.go:18 +0xb2 gorm.io/gorm.(*processor).Execute(0xc00012c460, 0x1928450) /go/pkg/mod/gorm.io/[email protected]/callbacks.go:130 +0x433 gorm.io/gorm.(*DB).Find(0xc00283fd68, {0x15d6420, 0xc0001264b0}, {0xc00019c510, 0x1, 0x1}) /go/pkg/mod/gorm.io/[email protected]/finisher_api.go:166 +0x13c github.com/apache/incubator-devlake/runner.RunTask({0x1931cb8, 0xc0002d2900}, 0x8, {0x19519c8, 0xc001bd4840}, 0xc0002403c0, 0xc00019c410, 0x15a0ce0) /app/runner/run_task.go:48 +0x11e github.com/apache/incubator-devlake/services.runTaskStandalone(0x19) /app/services/task.go:264 +0x1bc github.com/apache/incubator-devlake/services.runTasksStandalone.func1() /app/services/task.go:228 +0xa7 created by github.com/apache/incubator-devlake/services.runTasksStandalone /app/services/task.go:226 +0x5b goroutine 626 [chan receive]: github.com/apache/incubator-devlake/services.runTasksStandalone({0xc0005166b0, 0x2, 0xc002700b40}) /app/services/task.go:234 +0x158 github.com/apache/incubator-devlake/runner.RunPipeline(0xc000627080, {0x19519c8, 0xc000626e20}, 0x1, 0x1, 0x17ce820) /app/runner/run_pipeline.go:82 +0x948 github.com/apache/incubator-devlake/services.runPipelineStandalone(0xc002aa9cf0) /app/services/pipeline.go:389 +0xac github.com/apache/incubator-devlake/services.runPipeline(0xc0006057b8) /app/services/pipeline.go:237 +0x33 github.com/apache/incubator-devlake/services.RunPipelineInQueue.func1() /app/services/pipeline.go:225 +0xf4 created by github.com/apache/incubator-devlake/services.RunPipelineInQueue /app/services/pipeline.go:222 +0x53 goroutine 609 [runnable]: regexp/syntax.(*parser).push(0xc001caf440, 0xc0003d8e00) /usr/local/go/src/regexp/syntax/parse.go:139 +0x289 regexp/syntax.(*parser).literal(0xc001caf440, 0x5b) /usr/local/go/src/regexp/syntax/parse.go:189 +0xff regexp/syntax.Parse({0x17763c5, 0x16}, 0xd4) /usr/local/go/src/regexp/syntax/parse.go:880 +0x84c regexp.compile({0x17763c5, 0x16}, 0xbe60, 0x0) /usr/local/go/src/regexp/regexp.go:170 +0x35 regexp.Compile(...) /usr/local/go/src/regexp/regexp.go:133 regexp.MustCompile({0x17763c5, 0x16}) /usr/local/go/src/regexp/regexp.go:309 +0x31 github.com/apache/incubator-devlake/logger.(*DefaultLogger).Nested(0xc00032eaa0, {0xc0005041f0, 0x8}) /app/logger/logger.go:82 +0x5b github.com/apache/incubator-devlake/services.runTaskStandalone(0x1a) /app/services/task.go:267 +0x18c github.com/apache/incubator-devlake/services.runTasksStandalone.func1() /app/services/task.go:228 +0xa7 created by github.com/apache/incubator-devlake/services.runTasksStandalone /app/services/task.go:226 +0x5b ``` ### What you expected to happen The devlake is stable no crashed. ### How to reproduce It happens rarely. ### Anything else _No response_ ### Version v0.12.0 ### Are you willing to submit PR? - [X] Yes I am willing to submit a PR! ### Code of Conduct - [X] I agree to follow this project's [Code of Conduct](https://www.apache.org/foundation/policies/conduct) -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
