Improve small payload performance

[kostyantyn]

Noticed that after pulling master, performance of gzip compression is now lower than native Go implementation.

package lib

import (
    "testing"
    "bytes"
    "compress/gzip"
    ogzip "github.com/klauspost/compress/gzip"
    "fmt"
)

var bidReq = []byte(`{"id":"50215d10a41d474f77591bff601f6ade","imp":[{"id":"86df3bc6-7bd4-44d9-64e2-584a69790229","native":{"request":"{\"ver\":\"1.0\",\"plcmtcnt\":1,\"assets\":[{\"id\":1,\"data\":{\"type\":12}},{\"id\":2,\"required\":1,\"title\":{\"len\":50}},{\"id\":3,\"required\":1,\"img\":{\"type\":1,\"w\":80,\"h\":80}},{\"id\":4,\"required\":1,\"img\":{\"type\":3,\"w\":1200,\"h\":627}},{\"id\":5,\"data\":{\"type\":3}},{\"id\":6,\"required\":1,\"data\":{\"type\":2,\"len\":100}}]}","ver":"1.0"},"tagid":"1","bidfloor":0.6,"bidfloorcur":"USD"}],"site":{"id":"1012864","domain":"www.abc.com","cat":["IAB3"],"mobile":1,"keywords":"apps,games,discovery,recommendation"},"device":{"dnt":1,"ua":"Mozilla/5.0 (Linux; U; Android 4.2.2; km-kh; SHV-E120S Build/JZO54K) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30","ip":"175.100.59.170","geo":{"lat":11.5625,"lon":104.916,"country":"KHM","region":"12","city":"Phnom Penh","type":2},"carrier":"Viettel (cambodia) Pte., Ltd.","language":"km","model":"android","os":"Android","osv":"4.2.2","connectiontype":2,"devicetype":1},"user":{"id":"325a32d3-1dba-5ffc-82f2-1df428520728"},"at":2,"tmax":100,"wseat":["74","17","30","142","167","177","153","7","90","140","148","164","104","71","19","187","139","63","88","160","222","205","46"],"cur":["USD"]}`)

func BenchmarkNativeGzip(b *testing.B) {
    fmt.Println("BenchmarkNativeGzip")
    for i := 0; i < b.N; i++ {
        b := bytes.NewBuffer(nil)
        w := gzip.NewWriter(b)
        w.Write(bidReq)
        w.Close()
    }
}

func BenchmarkKlauspostGzip(b *testing.B) {
    fmt.Println("BenchmarkKlauspostGzip")
    for i := 0; i < b.N; i++ {
        b := bytes.NewBuffer(nil)
        w := ogzip.NewWriter(b)
        w.Write(bidReq)
        w.Close()
    }
}

/usr/local/Cellar/go/1.7.1/libexec/bin/go test -v github.com/kostyantyn/compressiontest/lib -bench "^BenchmarkNativeGzip|BenchmarkKlauspostGzip$" -run ^$
BenchmarkNativeGzip
    3000        387628 ns/op
BenchmarkKlauspostGzip
    3000        429190 ns/op
PASS
ok      github.com/pubnative/ad_server/lib  2.556s

[klauspost]

Interesting. I will see what the cause is for that.

Usually I would avoid allocations by re-using the writer:

func BenchmarkNativeGzip(b *testing.B) {
    fmt.Println("BenchmarkNativeGzip")
    w := gzip.NewWriter(nil)
    for i := 0; i < b.N; i++ {
        b := bytes.NewBuffer(nil)
        w.Reset(b)     
        w.Write(bidReq)
        w.Close()
    }
}

In real code a sync.Pool would contain your writers.

However, I have planned to look into reducing initial allocation cost.

[klauspost]

With an implementation like this that eliminates allocations:

func BenchmarkNativeGzipReset(b *testing.B) {
    b.SetBytes(int64(len(bidReq)))
    w := gzip.NewWriter(nil)
    bb := bytes.NewBuffer(nil)
    for i := 0; i < b.N; i++ {
        bb.Reset()
        w.Reset(bb)
        w.Write(bidReq)
        w.Close()
    }
}

BenchmarkNativeGzipReset-8                 20000             70612 ns/op          18.40 MB/s
BenchmarkKlauspostGzipReset-8              20000             76403 ns/op          17.00 MB/s

Still something to investigate. Having good "small payload" performance is also important.

[kostyantyn]

Thanks for the tip of re-setting a gzip writer. It gives significant improvement.

[klauspost]

My first attempt at a more lean implementation for usage without reset in #70 shows that the "snappy" (level 1-4) benefit from a smaller interface, but the "old" algorithm doesn't improve by more than a few percent.

[klauspost]

With #107 released, this is where we got to.