[go-nuts] why is my program retaining memory when len([]byte)>16384 ?

[mhhcbon]

Hi,

I have this program which reads file, flate encode then flate decode the 
data.

I noticed that when i used different size for the slice of []byte to read 
data, the program will retain memory when the size is > 16384.
When its lower than this value everything is fine, but 16385 breaks.

I don t quite understand the reason of this behavior, can someone help me 
to understand what s going on there ?

package main

import (
  "fmt"
  "os"
  "bytes"
  // "encoding/base64"
  "compress/flate"
  "io"
)

func main () {
  // okSize := 16384
  // koSize := 64384
  // koSize := 16385 // yes 1 more and it breaks :s

  // change read size to 16384 and everything is ok
  fReadSize := 16385
  dReadSize := 64384 // dread seems not impacting

  src, _ := os.Open("big.file 1gb")
  b := make([]byte, fReadSize)

  buf := new(bytes.Buffer)
  encoder, _ := flate.NewWriter(buf, 9)

  buf2 := new(bytes.Buffer)
  var decoder io.ReadCloser
  for {
    fmt.Println("---")
    _, err := src.Read(b); // read from file
    if err!= nil {
      break
    }
// fmt.Println(b)
    _, err = encoder.Write(b) // write to encoder
    fmt.Println(err)
    err = encoder.Flush()
    fmt.Println(err)
    r := buf.Bytes() // read from encoder
    buf.Truncate(0)
// fmt.Println(r)
    _, err = buf2.Write(r) // write to decoder
    fmt.Println(err)
    if decoder==nil {
      decoder = flate.NewReader(buf2)
    }
    r2 := make([]byte, dReadSize*2)
    _, err = decoder.Read(r2) // read from decoder
    fmt.Println(err)
  }
}



Does this helps ?

$ uname -a
Linux pc15-home 4.6.3-300.fc24.x86_64 #1 SMP Fri Jun 24 20:52:41 UTC 2016 
x86_64 x86_64 x86_64 GNU/Linux


thanks!



PS: no i m not interested into that version

package main

import (
  "os"
  "io"
  "compress/gzip"
)

func main () {
  pr, pw := io.Pipe()
  go func () {
    decoder, _ := gzip.NewReader(pr)
    io.Copy(os.Stdout, decoder)
  }()
  archiver := gzip.NewWriter(pw)
  defer archiver.Close()
  io.Copy(archiver, os.Stdin)
}

-- 
You received this message because you are subscribed to the Google Groups 
"golang-nuts" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to golang-nuts+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.