Alright, then. Find attached a patch against the piuparts git to add debiman-piuparts-distill. You can build it by running “go build” in the debiman-piuparts-distill subdirectory.
On Wed, May 31, 2017 at 11:47 AM, Holger Levsen <hol...@layer-acht.org> wrote: > On Wed, May 31, 2017 at 11:32:46AM +0200, Michael Stapelberg wrote: > > The fact that these manpages are included at all is what the > > slave-alternative handling got us :) > > ah! > > > > ok, I'll request this once we got closer… (see below) > > Thanks :). > > done (as you know, but for the record of this bug…) > > > The code is authenticated via HTTPS. If you don’t trust GitHub and/or the > > TLS certificate infrastructure, then we should find an alternative. > > I dont trust GitHub, or rather, it's bad enough trusting alioth, I dont > want > to also trust Github… (and TLS, no…) > > (we should probably rather trust signed tags, but anyway, as we dont have > that I would prefer to only trust one repo…) > > > > Also, I don't expect debiman-piuparts-distill to change very often, do > you? > > Agreed, most likely it will not change very often. > > ok, cool. > > > > does the code need to live in github.com/Debian/debiman at all? One > copy > > > in the piuparts repo should be enough, or? > > That’s correct; I was referring to code that is shared between different > > debiman components. Specifically, > > https://github.com/Debian/debiman/blob/54dd6050a6ce8a454c14e172a8687d > 93d0fd241b/internal/write/atomically.go > > ic > > > But, if the conclusion is that debiman-piuparts-distill should live in > the > > piuparts repo, we can duplicate that code. I don’t expect it to change > > either. > > I think I would really prefer the code to live in the piuparts repo. > > Or alternativly, life in your repo and then we deploy it on demand, but not > everyday. Having it in the piuparts repo would be easier for deployments, > but > I can see how its a bit more hassle for you to maintain the code in another > repo. But then, I know how to be very liberal to accept your code changes > to > your code in our piuparts repo :) > > Patches welcome! :-D > > > -- > cheers, > Holger > -- Best regards, Michael
From 7001c015ef65f548a0baf5f5488a05d91f2a3a64 Mon Sep 17 00:00:00 2001 From: Michael Stapelberg <stapelb...@debian.org> Date: Thu, 1 Jun 2017 21:28:15 +0200 Subject: [PATCH] add debiman-piuparts-distill see https://bugs.debian.org/863089 --- debiman-piuparts-distill/atomically.go | 75 +++++++++++++++++ debiman-piuparts-distill/piuparts.go | 145 +++++++++++++++++++++++++++++++++ 2 files changed, 220 insertions(+) create mode 100644 debiman-piuparts-distill/atomically.go create mode 100644 debiman-piuparts-distill/piuparts.go diff --git a/debiman-piuparts-distill/atomically.go b/debiman-piuparts-distill/atomically.go new file mode 100644 index 00000000..c1b2d2db --- /dev/null +++ b/debiman-piuparts-distill/atomically.go @@ -0,0 +1,75 @@ +package main + +import ( + "bufio" + "compress/gzip" + "io" + "io/ioutil" + "os" + "path/filepath" +) + +func tempDir(dest string) string { + tempdir := os.Getenv("TMPDIR") + if tempdir == "" { + // Convenient for development: decreases the chance that we + // cannot move files due to /tmp being on a different file + // system. + tempdir = filepath.Dir(dest) + } + return tempdir +} + +func writeAtomically(dest string, compress bool, write func(w io.Writer) error) (err error) { + f, err := ioutil.TempFile(tempDir(dest), "debiman-") + if err != nil { + return err + } + defer func() { + // Remove the tempfile if an error occurred + if err != nil { + os.Remove(f.Name()) + } + }() + defer f.Close() + + bufw := bufio.NewWriter(f) + + w := io.Writer(bufw) + var gzipw *gzip.Writer + if compress { + // NOTE(stapelberg): gzip’s decompression phase takes the same + // time, regardless of compression level. Hence, we invest the + // maximum CPU time once to achieve the best compression. + gzipw, err = gzip.NewWriterLevel(bufw, gzip.BestCompression) + if err != nil { + return err + } + defer gzipw.Close() + w = gzipw + } + + if err := write(w); err != nil { + return err + } + + if compress { + if err := gzipw.Close(); err != nil { + return err + } + } + + if err := bufw.Flush(); err != nil { + return err + } + + if err := f.Chmod(0644); err != nil { + return err + } + + if err := f.Close(); err != nil { + return err + } + + return os.Rename(f.Name(), dest) +} diff --git a/debiman-piuparts-distill/piuparts.go b/debiman-piuparts-distill/piuparts.go new file mode 100644 index 00000000..73f773dc --- /dev/null +++ b/debiman-piuparts-distill/piuparts.go @@ -0,0 +1,145 @@ +// debiman-piuparts-distill extracts slave alternative links from +// LOG-ALTERNATIVES lines found in piuparts logs. +// +// See https://github.com/Debian/debiman/issues/12 for more details. +package main + +import ( + "bufio" + "encoding/json" + "flag" + "io" + "log" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "sync" +) + +var ( + logsDir = flag.String("logs_dir", + "", + "Directory containing piuparts logfiles") + + output = flag.String("output", + "", + "Path to write the (gzip-compressed, json-encoded) distilled links file to") + + parallel = flag.Int("parallel", + 10, + "Number of logfiles to read in parallel") +) + +var ( + logAlternativesRe = regexp.MustCompile(`LOG-ALTERNATIVES: dpkg=([^:]+): piuparts=(?:[^:]+): (.*)`) + slaveParamsRe = regexp.MustCompile(`--slave ([^ ]+) (?:[^ ]+) ([^ ]+)`) +) + +type link struct { + Pkg string `json:"binpackage"` + From string `json:"from"` + To string `json:"to"` +} + +// process reads the piuparts logfile at path. links are extracted from each +// LOG-ALTERNATIVES line and written to the links channel. +func process(path string, links chan<- link) error { + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if !strings.HasPrefix(line, "LOG-ALTERNATIVES: ") { + continue + } + matches := logAlternativesRe.FindStringSubmatch(line) + if matches == nil { + continue + } + for _, param := range slaveParamsRe.FindAllStringSubmatch(line, -1) { + links <- link{ + Pkg: matches[1], + From: param[1], + To: param[2], + } + } + } + return scanner.Err() +} + +// byPkg is a helper type for sorting the results slice by binary package. Once +// Go 1.8 becomes available on piuparts.debian.org, we can switch to sort.Slice. +type byPkg []link + +func (p byPkg) Len() int { return len(p) } +func (p byPkg) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p byPkg) Less(i, j int) bool { return p[i].Pkg < p[j].Pkg } + +func main() { + flag.Parse() + + if *output == "" { + log.Fatal("-output must be specified") + } + + if *logsDir == "" { + log.Fatal("-logs_dir must be specified") + } + + // Spawn -parallel worker goroutines, waiting for work + work := make(chan string) + linksChan := make(chan link) + var wg sync.WaitGroup + for i := 0; i < *parallel; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for path := range work { + if err := process(path, linksChan); err != nil { + log.Printf("error processing %q: %v", path, err) + } + } + }() + } + // Collect results from all workers into linksMap + linksMap := make(map[link]bool) + go func() { + for l := range linksChan { + linksMap[l] = true + } + }() + // Walk through *logsDir, enqueue all .log files onto the work channel + if err := filepath.Walk(*logsDir, func(path string, info os.FileInfo, err error) error { + if strings.HasSuffix(path, ".log") && info.Mode().IsRegular() { + work <- path + } + return nil + }); err != nil { + log.Fatal(err) + } + // Close the channel, signaling termination to the worker goroutines + close(work) + // Wait for the worker goroutines to terminate + wg.Wait() + close(linksChan) + // Convert the unsorted linksMap into a slice for sorting + links := make([]link, 0, len(linksMap)) + for l := range linksMap { + log.Printf("l = %+v", l) + links = append(links, l) + } + // for easier debugging of the resulting file: + sort.Stable(byPkg(links)) + + if err := writeAtomically(*output, true, func(w io.Writer) error { + return json.NewEncoder(w).Encode(&links) + }); err != nil { + log.Fatal(err) + } +} -- 2.11.0
