Macaron 1.5 (#12596)

* update macaron to v1.5 of fork

* update macaron to v1.5 of fork

* test gzip PR

* add push method impl to context_tests

* use proper gzip commit

Co-authored-by: zeripath <art27@cantab.net>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
This commit is contained in:
techknowlogick 2020-08-27 22:47:17 -04:00 committed by GitHub
parent 211321fb93
commit c5d5d63c9c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
53 changed files with 2622 additions and 665 deletions

View file

@ -12,15 +12,18 @@ type EOption func(*encoderOptions) error
// options retains accumulated state of multiple options.
type encoderOptions struct {
concurrent int
crc bool
single *bool
pad int
blockSize int
windowSize int
level EncoderLevel
fullZero bool
noEntropy bool
concurrent int
level EncoderLevel
single *bool
pad int
blockSize int
windowSize int
crc bool
fullZero bool
noEntropy bool
allLitEntropy bool
customWindow bool
customALEntropy bool
}
func (o *encoderOptions) setDefault() {
@ -30,7 +33,7 @@ func (o *encoderOptions) setDefault() {
crc: true,
single: nil,
blockSize: 1 << 16,
windowSize: 1 << 22,
windowSize: 8 << 20,
level: SpeedDefault,
}
}
@ -39,9 +42,11 @@ func (o *encoderOptions) setDefault() {
func (o encoderOptions) encoder() encoder {
switch o.level {
case SpeedDefault:
return &doubleFastEncoder{fastEncoder: fastEncoder{maxMatchOff: int32(o.windowSize)}}
return &doubleFastEncoder{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize)}}}
case SpeedBetterCompression:
return &betterFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize)}}
case SpeedFastest:
return &fastEncoder{maxMatchOff: int32(o.windowSize)}
return &fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize)}}
}
panic("unknown compression level")
}
@ -67,7 +72,7 @@ func WithEncoderConcurrency(n int) EOption {
}
// WithWindowSize will set the maximum allowed back-reference distance.
// The value must be a power of two between WindowSizeMin and WindowSizeMax.
// The value must be a power of two between MinWindowSize and MaxWindowSize.
// A larger value will enable better compression but allocate more memory and,
// for above-default values, take considerably longer.
// The default value is determined by the compression level.
@ -83,6 +88,7 @@ func WithWindowSize(n int) EOption {
}
o.windowSize = n
o.customWindow = true
if o.blockSize > o.windowSize {
o.blockSize = o.windowSize
}
@ -130,18 +136,18 @@ const (
// This is roughly equivalent to the default Zstandard mode (level 3).
SpeedDefault
// SpeedBetterCompression will yield better compression than the default.
// Currently it is about zstd level 7-8 with ~ 2x-3x the default CPU usage.
// By using this, notice that CPU usage may go up in the future.
SpeedBetterCompression
// speedLast should be kept as the last actual compression option.
// The is not for external usage, but is used to keep track of the valid options.
speedLast
// SpeedBetterCompression will (in the future) yield better compression than the default,
// but at approximately 4x the CPU usage of the default.
// For now this is not implemented.
SpeedBetterCompression = SpeedDefault
// SpeedBestCompression will choose the best available compression option.
// For now this is not implemented.
SpeedBestCompression = SpeedDefault
SpeedBestCompression = SpeedBetterCompression
)
// EncoderLevelFromString will convert a string representation of an encoding level back
@ -163,8 +169,10 @@ func EncoderLevelFromZstd(level int) EncoderLevel {
switch {
case level < 3:
return SpeedFastest
case level >= 3:
case level >= 3 && level < 6:
return SpeedDefault
case level > 5:
return SpeedBetterCompression
}
return SpeedDefault
}
@ -176,6 +184,8 @@ func (e EncoderLevel) String() string {
return "fastest"
case SpeedDefault:
return "default"
case SpeedBetterCompression:
return "better"
default:
return "invalid"
}
@ -189,6 +199,20 @@ func WithEncoderLevel(l EncoderLevel) EOption {
return fmt.Errorf("unknown encoder level")
}
o.level = l
if !o.customWindow {
switch o.level {
case SpeedFastest:
o.windowSize = 4 << 20
case SpeedDefault:
o.windowSize = 8 << 20
case SpeedBetterCompression:
o.windowSize = 16 << 20
}
}
if !o.customALEntropy {
o.allLitEntropy = l > SpeedFastest
}
return nil
}
}
@ -203,6 +227,18 @@ func WithZeroFrames(b bool) EOption {
}
}
// WithAllLitEntropyCompression will apply entropy compression if no matches are found.
// Disabling this will skip incompressible data faster, but in cases with no matches but
// skewed character distribution compression is lost.
// Default value depends on the compression level selected.
func WithAllLitEntropyCompression(b bool) EOption {
return func(o *encoderOptions) error {
o.customALEntropy = true
o.allLitEntropy = b
return nil
}
}
// WithNoEntropyCompression will always skip entropy compression of literals.
// This can be useful if content has matches, but unlikely to benefit from entropy
// compression. Usually the slight speed improvement is not worth enabling this.