Commit 72a4c72

committed

compute block length accounting for compression

Taken from PR #1698 and commit da5f789

1 parent c65a8ac commit 72a4c72Copy full SHA for 72a4c72

File tree

+11

-2

lines changed

+11

-2

lines changed

Lines changed: 0 additions & 1 deletion

Original file line number	Diff line number	Diff line change
`@@ -5,7 +5,6 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE`
`5`	`5`	`github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=`
`6`	`6`	`github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=`
`7`	`7`	`github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=`
`8`		`-github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=`
`9`	`8`	`github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=`
`10`	`9`	`github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=`
`11`	`10`	`github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=`

Lines changed: 11 additions & 1 deletion

Original file line number	Diff line number	Diff line change
`@@ -156,6 +156,16 @@ func NewTableBuilder(opts Options) *Builder {`
`156`	`156`	`return b`
`157`	`157`	`}`
`158`	`158`
	`159`	`+func maxEncodedLen(ctype options.CompressionType, sz int) int {`
	`160`	`+ switch ctype {`
	`161`	`+ case options.Snappy:`
	`162`	`+ return snappy.MaxEncodedLen(sz)`
	`163`	`+ case options.ZSTD:`
	`164`	`+ return y.ZSTDCompressBound(sz)`
	`165`	`+ }`
	`166`	`+ return sz`
	`167`	`+}`
	`168`	`+`
`159`	`169`	`func (b *Builder) handleBlock() {`
`160`	`170`	`defer b.wg.Done()`
`161`	`171`
`@@ -178,7 +188,7 @@ func (b *Builder) handleBlock() {`
`178`	`188`	`// BlockBuf should always less than or equal to allocated space. If the blockBuf is greater`
`179`	`189`	`// than allocated space that means the data from this block cannot be stored in its`
`180`	`190`	`// existing location.`
`181`		`- allocatedSpace := (item.end) + padding + 1`
	`191`	`+ allocatedSpace := maxEncodedLen(b.opts.Compression, (item.end)) + padding + 1`
`182`	`192`	`y.AssertTrue(len(blockBuf) <= allocatedSpace)`
`183`	`193`
`184`	`194`	`// blockBuf was allocated on allocator. So, we don't need to copy it over.`

Comments

(0)