Franciszek Stachura, Sep 12, 2026 at 23:06:
> Generates random data for development or performance testing.
>
> Signed-off-by: Franciszek Stachura <[email protected]>
> ---
> For some reason, keeping a long-running connection with large bulk
> inserts causes Postgres to use a lot of memory, until the connection is
> closed. The only solution I found is making the connections
> short-lived with SetConnMaxLifetime
> (I believe that it should be used anyway).
> I don't see any good way of manually closing connections in Bun.
>
> cmd/pw/db/generate.go | 285 ++++++++++++++++++++++++++++++++++++++++++
> cmd/pw/db/main.go | 9 +-
> 2 files changed, 290 insertions(+), 4 deletions(-)
> create mode 100644 cmd/pw/db/generate.go
>
> diff --git a/cmd/pw/db/generate.go b/cmd/pw/db/generate.go
> new file mode 100644
> index 0000000..ca1f18e
> --- /dev/null
> +++ b/cmd/pw/db/generate.go
> @@ -0,0 +1,285 @@
> +// Patchwork - automated patch tracking system
> +// Copyright (C) The Patchwork Contributors (see CONTRIBUTORS)
> +//
> +// SPDX-License-Identifier: GPL-2.0-or-later
> +
> +package db
> +
> +import (
> + "context"
> + "fmt"
> + "math/rand"
> + "time"
> +
> + "github.com/getpatchwork/patchwork/cmd/pw/pw"
> + "github.com/getpatchwork/patchwork/pkg/db"
> + "github.com/getpatchwork/patchwork/pkg/events"
> +)
> +
> +type GenerateCmd struct {
> + Name string `required:"" help:"Name of the project."`
> + Series int `required:"" help:"Number of series to generate. Each
> series will have up to 20 patches."`
> + Authors int `required:"" help:"Number of authors to generate."`
> +}
> +
> +const BATCH_SIZE = 1000
> +
> +func (c *GenerateCmd) Run(ctx context.Context) error {
> + database := pw.GetDB(ctx)
> +
> + bus := events.Start(ctx, database)
> + defer bus.Shutdown()
> + ctx = db.WithBus(ctx, bus)
> +
> + projectName := c.Name
> +
> + project := db.Project{
> + Name: projectName,
> + Linkname: projectName,
> + Listid: projectName,
> + Listemail: fmt.Sprintf("%s@%s.example.com",
> projectName, projectName),
> + WebURL: "",
> + ScmURL: "",
> + WebScmURL: "",
> + ListArchiveURL: "",
> + SubjectMatch: "",
> + CommitURLFormat: "",
> + ListArchiveURLFormat: "",
> + }
> + _, err := database.NewInsert().Model(&project).Exec(ctx)
> + if err != nil {
> + return err
> + }
> +
> + fmt.Printf("Created project %q (id=%d)\n", project.Name, project.ID)
> +
> + var states []db.State
> + err = database.NewSelect().Model(&states).Scan(ctx)
> + if err != nil {
> + return err
> + }
> +
> + var tags []db.Tag
> + err = database.NewSelect().Model(&tags).Scan(ctx)
> + if err != nil {
> + return err
> + }
> +
> + var authors []db.Person
> + for i := range c.Authors {
> + personName := fmt.Sprintf("person_%s_%d", projectName, i)
> +
> + person := db.Person{
> + Email: fmt.Sprintf("%s@%s.example.com", personName,
> projectName),
> + Name: &personName,
> + }
> + _, err := database.NewInsert().Model(&person).Exec(ctx)
> + if err != nil {
> + return err
> + }
> +
> + authors = append(authors, person)
> + }
> +
> + series_batches := c.Series / BATCH_SIZE
> + last_batch_size := c.Series % BATCH_SIZE
> + if last_batch_size != 0 {
> + series_batches += 1
> + }
> +
> + for sb := range series_batches {
> + var batch_size int
> + if sb == series_batches-1 && last_batch_size != 0 {
> + batch_size = last_batch_size
> + } else {
> + batch_size = BATCH_SIZE
> + }
> +
> + // Create covers
> +
> + var batch_covers []db.Cover
> + for b := range batch_size {
> + batch_covers = append(
> + batch_covers,
> + randCover(project,
> authors[rand.Int()%len(authors)], sb*BATCH_SIZE+b),
> + )
> + }
> +
> + _, err :=
> database.NewInsert().Model(&batch_covers).Returning("id").Exec(ctx)
> + if err != nil {
> + return err
> + }
> +
> + // Create series
> +
> + var batch_series []db.Series
> + for _, cover := range batch_covers {
> + batch_series = append(batch_series, randSeries(project,
> cover))
> + }
> + _, err =
> database.NewInsert().Model(&batch_series).Returning("id").Exec(ctx)
> + if err != nil {
> + return err
> + }
> +
> + // Create patches for each series
> +
> + var batch_patches []db.Patch
> + for s, series := range batch_series {
> + for p := range series.Total {
> + name := fmt.Sprintf("series %d patch %d",
> sb*BATCH_SIZE+s, p)
> + batch_patches = append(
> + batch_patches,
> + randPatch(project, series,
> states[rand.Int()%len(states)], name, p),
> + )
> + }
> + }
> +
> + err =
> database.NewInsert().Model(&batch_patches).Returning("id").Scan(ctx)
> + if err != nil {
> + return err
> + }
> +
> + // Create tags and checks
> +
> + var batch_tags []db.PatchTag
> + var batch_checks []db.Check
> + for _, patch := range batch_patches {
> + for t := range len(tags) {
> + if rand.Int()%(t+2) != t {
> + continue
> + }
> +
> + batch_tags = append(batch_tags, db.PatchTag{
> + PatchID: patch.ID,
> + TagID: tags[t].ID,
> + Count: rand.Int() % 10,
> + })
> + }
> +
> + for range rand.Int() % 10 {
> + batch_checks = append(batch_checks,
> randCheck(patch))
> + }
> + }
> +
> + err =
> database.NewInsert().Model(&batch_tags).Returning("").Scan(ctx)
> + if err != nil {
> + return err
> + }
> +
> + err =
> database.NewInsert().Model(&batch_checks).Returning("").Scan(ctx)
> + if err != nil {
> + return err
> + }
> +
> + fmt.Printf("\rgenerated series %d/%d",
> sb*BATCH_SIZE+batch_size, c.Series)
> + }
> + fmt.Printf("\n")
> +
> + return nil
> +}
> +
> +func randString(len uint32) string {
> + res := make([]byte, len)
> + for i := range len {
> + t := rand.Uint32() % 3
> + if t == 0 {
> + // 0-9
> + res[i] = byte(48 + (rand.Uint32() % 10))
> + } else if t == 1 {
> + // A-Z
> + res[i] = byte(65 + (rand.Uint32() % 25))
> + } else {
> + // a-z
> + res[i] = byte(97 + (rand.Uint32() % 25))
> + }
> + }
> +
> + return string(res)
> +}
cmd/pw/db/generate.go:185:3: ifElseChain: rewrite if-else to switch statement
(gocritic)
if t == 0 {
^
Perhaps you can refactor it that way:
const charset = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
func randString(length uint32) string {
res := make([]byte, length)
for i := range res {
res[i] = charset[rand.Intn(len(charset))]
}
return string(res)
}
> +
> +func randTextBlock(lines uint32, lineLen uint32) string {
> + result := make([]byte, (lineLen+1)*lines)
> + for i := range lines {
> + line := randString(lineLen)
> + copy(result[i*lineLen+1:], []byte(line))
> + result[(i+1)*lineLen] = '\n'
> + }
> + return string(result)
> +}
> +
> +func randColor() int {
> + r := (rand.Int() % 200) + 40
> + g := (rand.Int() % 200) + 40
> + b := (rand.Int() % 200) + 40
> +
> + return (r << 16) | (g << 8) | b
> +}
cmd/pw/db/generate.go:210:6: func randColor is unused (unused)
func randColor() int {
^
> +
> +func randCover(project db.Project, submitter db.Person, num int) db.Cover {
> + name := fmt.Sprintf("cover %d", num)
> + content := randTextBlock((rand.Uint32()%500)+10, 72)
> + headers := randTextBlock((rand.Uint32()%40)+10, 72)
> + msgid := fmt.Sprintf("<%s@%s.example.com>", randString(15),
> project.Name)
> +
> + return db.Cover{
> + Msgid: msgid,
> + Date: time.Now(),
> + Headers: headers,
> + SubmitterID: submitter.ID,
> + Content: &content,
> + ProjectID: project.ID,
> + Name: name,
> + }
> +}
> +
> +func randSeries(project db.Project, cover db.Cover) db.Series {
> + patches := rand.Int() % 20
> +
> + return db.Series{
> + ProjectID: &project.ID,
> + CoverLetterID: &cover.ID,
> + Name: &cover.Name,
> + Date: time.Now(),
> + SubmitterID: cover.SubmitterID,
> + Version: 0,
> + Total: patches,
> +
> + ReceivedTotal: patches,
> + ReceivedAll: true,
> + }
> +}
> +
> +func randPatch(project db.Project, series db.Series, state db.State, name
> string, num int) db.Patch {
> + msgid := fmt.Sprintf("<%s@%s.example.com>", randString(15),
> project.Name)
> + content := randTextBlock((rand.Uint32()%500)+10, 72)
> + headers := randTextBlock((rand.Uint32()%40)+10, 72)
> + diff := ""
> + hash := randString(32)
> +
> + return db.Patch{
> + Msgid: msgid,
> + Date: time.Now(),
> + Headers: headers,
> + SubmitterID: series.SubmitterID,
> + Content: &content,
> + ProjectID: project.ID,
> + Name: name,
> + Diff: &diff,
> + StateID: &state.ID,
> + Archived: (rand.Int() % 2) == 0,
> + Hash: &hash,
> + SeriesID: &series.ID,
> + Number: &num,
> + }
> +}
> +
> +func randCheck(patch db.Patch) db.Check {
> + return db.Check{
> + PatchID: patch.ID,
> + Date: time.Now(),
> + State: db.CheckState(rand.Int() % 4),
> + TargetURL: "",
> + Context: fmt.Sprintf("check_%d", rand.Int()%5),
> + Description: "",
> + }
> +}
> diff --git a/cmd/pw/db/main.go b/cmd/pw/db/main.go
> index 6c12f88..8fed09c 100644
> --- a/cmd/pw/db/main.go
> +++ b/cmd/pw/db/main.go
> @@ -7,8 +7,9 @@ package db
>
> // CLI groups the database management subcommands.
> type CLI struct {
> - Sync SyncCmd `cmd:"" help:"Create or update the database schema."`
> - Status StatusCmd `cmd:"" help:"Show database schema migrations status."`
> - Export ExportCmd `cmd:"" help:"Export database contents as SQL."`
> - Import ImportCmd `cmd:"" name:"import" help:"Import SQL data from
> stdin."`
> + Sync SyncCmd `cmd:"" help:"Create or update the database
> schema."`
> + Status StatusCmd `cmd:"" help:"Show database schema migrations
> status."`
> + Export ExportCmd `cmd:"" help:"Export database contents as SQL."`
> + Import ImportCmd `cmd:"" name:"import" help:"Import SQL data from
> stdin."`
> + Generate GenerateCmd `cmd:"" name:"generate" help:"Generate random
> data."`
> }
--
Robin
# Misuse may cause suffocation.
_______________________________________________
Patchwork mailing list
[email protected]
https://lists.ozlabs.org/listinfo/patchwork
