mirror of
https://github.com/lukaszraczylo/traefikoidc.git
synced 2026-10-14 04:07:46 +00:00
Add redis support for distributed caching (#83)
* Add redis support for distributed caching * Move towards the self-provided Redis connection pool and RESP protocol implementation. Official redis client library won't work with yaegi. * fixup! Move towards the self-provided Redis connection pool and RESP protocol implementation. Official redis client library won't work with yaegi. * fixup! fixup! Move towards the self-provided Redis connection pool and RESP protocol implementation. Official redis client library won't work with yaegi. * fixup! fixup! fixup! Move towards the self-provided Redis connection pool and RESP protocol implementation. Official redis client library won't work with yaegi. * fixup! fixup! fixup! fixup! Move towards the self-provided Redis connection pool and RESP protocol implementation. Official redis client library won't work with yaegi. * fixup! fixup! fixup! fixup! fixup! Move towards the self-provided Redis connection pool and RESP protocol implementation. Official redis client library won't work with yaegi. * ... and another all nighter. * fixup! ... and another all nighter. * fixup! fixup! ... and another all nighter. * fixup! fixup! fixup! ... and another all nighter. * Resolve issue #85 by adding ability to set custom claims in JWT tokens * Remove redundant validation in auth middleware ( issue #89 ) * Add ability to set cookie prefix for session cookies ( #87 ) * fixup! Add ability to set cookie prefix for session cookies ( #87 ) * Add ability to set cookie max age - issue #91 * Potential fix for code scanning alert no. 10: Size computation for allocation may overflow Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> * fixup! Merge main into 0.8.0-redis: resolve conflicts --------- Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>
Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>
parent
5fcbd54955
commit
e64fc7f730
+21
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2017 Axiom Inc. <[email protected]>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
+1
@@ -0,0 +1 @@
|
||||
This is a copy of github.com/axiomhq/hyperloglog.
|
||||
+180
@@ -0,0 +1,180 @@
|
||||
package hyperloglog
|
||||
|
||||
import "encoding/binary"
|
||||
|
||||
// Original author of this file is github.com/clarkduvall/hyperloglog
|
||||
type iterable interface {
|
||||
decode(i int, last uint32) (uint32, int)
|
||||
Len() int
|
||||
Iter() *iterator
|
||||
}
|
||||
|
||||
type iterator struct {
|
||||
i int
|
||||
last uint32
|
||||
v iterable
|
||||
}
|
||||
|
||||
func (iter *iterator) Next() uint32 {
|
||||
n, i := iter.v.decode(iter.i, iter.last)
|
||||
iter.last = n
|
||||
iter.i = i
|
||||
return n
|
||||
}
|
||||
|
||||
func (iter *iterator) Peek() uint32 {
|
||||
n, _ := iter.v.decode(iter.i, iter.last)
|
||||
return n
|
||||
}
|
||||
|
||||
func (iter iterator) HasNext() bool {
|
||||
return iter.i < iter.v.Len()
|
||||
}
|
||||
|
||||
type compressedList struct {
|
||||
count uint32
|
||||
last uint32
|
||||
b variableLengthList
|
||||
}
|
||||
|
||||
func (v *compressedList) Clone() *compressedList {
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
newV := &compressedList{
|
||||
count: v.count,
|
||||
last: v.last,
|
||||
}
|
||||
|
||||
newV.b = make(variableLengthList, len(v.b))
|
||||
copy(newV.b, v.b)
|
||||
return newV
|
||||
}
|
||||
|
||||
func (v *compressedList) MarshalBinary() (data []byte, err error) {
|
||||
// Marshal the variableLengthList
|
||||
bdata, err := v.b.MarshalBinary()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// At least 4 bytes for the two fixed sized values plus the size of bdata.
|
||||
data = make([]byte, 0, 4+4+len(bdata))
|
||||
|
||||
// Marshal the count and last values.
|
||||
data = append(data, []byte{
|
||||
// Number of items in the list.
|
||||
byte(v.count >> 24),
|
||||
byte(v.count >> 16),
|
||||
byte(v.count >> 8),
|
||||
byte(v.count),
|
||||
// The last item in the list.
|
||||
byte(v.last >> 24),
|
||||
byte(v.last >> 16),
|
||||
byte(v.last >> 8),
|
||||
byte(v.last),
|
||||
}...)
|
||||
|
||||
// Append the list
|
||||
return append(data, bdata...), nil
|
||||
}
|
||||
|
||||
func (v *compressedList) UnmarshalBinary(data []byte) error {
|
||||
if len(data) < 12 {
|
||||
return ErrorTooShort
|
||||
}
|
||||
|
||||
// Set the count.
|
||||
v.count, data = binary.BigEndian.Uint32(data[:4]), data[4:]
|
||||
|
||||
// Set the last value.
|
||||
v.last, data = binary.BigEndian.Uint32(data[:4]), data[4:]
|
||||
|
||||
// Set the list.
|
||||
sz, data := binary.BigEndian.Uint32(data[:4]), data[4:]
|
||||
v.b = make([]uint8, sz)
|
||||
if uint32(len(data)) < sz {
|
||||
return ErrorTooShort
|
||||
}
|
||||
for i := uint32(0); i < sz; i++ {
|
||||
v.b[i] = data[i]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func newCompressedList() *compressedList {
|
||||
v := &compressedList{}
|
||||
v.b = make(variableLengthList, 0)
|
||||
return v
|
||||
}
|
||||
|
||||
func (v *compressedList) Len() int {
|
||||
return len(v.b)
|
||||
}
|
||||
|
||||
func (v *compressedList) decode(i int, last uint32) (uint32, int) {
|
||||
n, i := v.b.decode(i, last)
|
||||
return n + last, i
|
||||
}
|
||||
|
||||
func (v *compressedList) Append(x uint32) {
|
||||
v.count++
|
||||
v.b = v.b.Append(x - v.last)
|
||||
v.last = x
|
||||
}
|
||||
|
||||
func (v *compressedList) Iter() *iterator {
|
||||
return &iterator{0, 0, v}
|
||||
}
|
||||
|
||||
type variableLengthList []uint8
|
||||
|
||||
func (v variableLengthList) MarshalBinary() (data []byte, err error) {
|
||||
// 4 bytes for the size of the list, and a byte for each element in the
|
||||
// list.
|
||||
data = make([]byte, 0, 4+v.Len())
|
||||
|
||||
// Length of the list. We only need 32 bits because the size of the set
|
||||
// couldn't exceed that on 32 bit architectures.
|
||||
sz := v.Len()
|
||||
data = append(data, []byte{
|
||||
byte(sz >> 24),
|
||||
byte(sz >> 16),
|
||||
byte(sz >> 8),
|
||||
byte(sz),
|
||||
}...)
|
||||
|
||||
// Marshal each element in the list.
|
||||
for i := 0; i < sz; i++ {
|
||||
data = append(data, v[i])
|
||||
}
|
||||
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func (v variableLengthList) Len() int {
|
||||
return len(v)
|
||||
}
|
||||
|
||||
func (v *variableLengthList) Iter() *iterator {
|
||||
return &iterator{0, 0, v}
|
||||
}
|
||||
|
||||
func (v variableLengthList) decode(i int, last uint32) (uint32, int) {
|
||||
var x uint32
|
||||
j := i
|
||||
for ; v[j]&0x80 != 0; j++ {
|
||||
x |= uint32(v[j]&0x7f) << (uint(j-i) * 7)
|
||||
}
|
||||
x |= uint32(v[j]) << (uint(j-i) * 7)
|
||||
return x, j + 1
|
||||
}
|
||||
|
||||
func (v variableLengthList) Append(x uint32) variableLengthList {
|
||||
for x&0xffffff80 != 0 {
|
||||
v = append(v, uint8((x&0x7f)|0x80))
|
||||
x >>= 7
|
||||
}
|
||||
return append(v, uint8(x&0x7f))
|
||||
}
|
||||
+424
@@ -0,0 +1,424 @@
|
||||
package hyperloglog
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
)
|
||||
|
||||
const (
|
||||
capacity = uint8(16)
|
||||
pp = uint8(25)
|
||||
mp = uint32(1) << pp
|
||||
version = 1
|
||||
)
|
||||
|
||||
// Sketch is a HyperLogLog data-structure for the count-distinct problem,
|
||||
// approximating the number of distinct elements in a multiset.
|
||||
type Sketch struct {
|
||||
p uint8
|
||||
b uint8
|
||||
m uint32
|
||||
alpha float64
|
||||
tmpSet set
|
||||
sparseList *compressedList
|
||||
regs *registers
|
||||
}
|
||||
|
||||
// New returns a HyperLogLog Sketch with 2^14 registers (precision 14)
|
||||
func New() *Sketch {
|
||||
return New14()
|
||||
}
|
||||
|
||||
// New14 returns a HyperLogLog Sketch with 2^14 registers (precision 14)
|
||||
func New14() *Sketch {
|
||||
sk, _ := newSketch(14, true)
|
||||
return sk
|
||||
}
|
||||
|
||||
// New16 returns a HyperLogLog Sketch with 2^16 registers (precision 16)
|
||||
func New16() *Sketch {
|
||||
sk, _ := newSketch(16, true)
|
||||
return sk
|
||||
}
|
||||
|
||||
// NewNoSparse returns a HyperLogLog Sketch with 2^14 registers (precision 14)
|
||||
// that will not use a sparse representation
|
||||
func NewNoSparse() *Sketch {
|
||||
sk, _ := newSketch(14, false)
|
||||
return sk
|
||||
}
|
||||
|
||||
// New16NoSparse returns a HyperLogLog Sketch with 2^16 registers (precision 16)
|
||||
// that will not use a sparse representation
|
||||
func New16NoSparse() *Sketch {
|
||||
sk, _ := newSketch(16, false)
|
||||
return sk
|
||||
}
|
||||
|
||||
// newSketch returns a HyperLogLog Sketch with 2^precision registers
|
||||
func newSketch(precision uint8, sparse bool) (*Sketch, error) {
|
||||
if precision < 4 || precision > 18 {
|
||||
return nil, fmt.Errorf("p has to be >= 4 and <= 18")
|
||||
}
|
||||
m := uint32(math.Pow(2, float64(precision)))
|
||||
s := &Sketch{
|
||||
m: m,
|
||||
p: precision,
|
||||
alpha: alpha(float64(m)),
|
||||
}
|
||||
if sparse {
|
||||
s.tmpSet = set{}
|
||||
s.sparseList = newCompressedList()
|
||||
} else {
|
||||
s.regs = newRegisters(m)
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (sk *Sketch) sparse() bool {
|
||||
return sk.sparseList != nil
|
||||
}
|
||||
|
||||
// Clone returns a deep copy of sk.
|
||||
func (sk *Sketch) Clone() *Sketch {
|
||||
return &Sketch{
|
||||
b: sk.b,
|
||||
p: sk.p,
|
||||
m: sk.m,
|
||||
alpha: sk.alpha,
|
||||
tmpSet: sk.tmpSet.Clone(),
|
||||
sparseList: sk.sparseList.Clone(),
|
||||
regs: sk.regs.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
// Converts to normal if the sparse list is too large.
|
||||
func (sk *Sketch) maybeToNormal() {
|
||||
if uint32(len(sk.tmpSet))*100 > sk.m {
|
||||
sk.mergeSparse()
|
||||
if uint32(sk.sparseList.Len()) > sk.m {
|
||||
sk.toNormal()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Merge takes another Sketch and combines it with Sketch h.
|
||||
// If Sketch h is using the sparse Sketch, it will be converted
|
||||
// to the normal Sketch.
|
||||
func (sk *Sketch) Merge(other *Sketch) error {
|
||||
if other == nil {
|
||||
// Nothing to do
|
||||
return nil
|
||||
}
|
||||
cpOther := other.Clone()
|
||||
|
||||
if sk.p != cpOther.p {
|
||||
return errors.New("precisions must be equal")
|
||||
}
|
||||
|
||||
if sk.sparse() && other.sparse() {
|
||||
for k := range other.tmpSet {
|
||||
sk.tmpSet.add(k)
|
||||
}
|
||||
for iter := other.sparseList.Iter(); iter.HasNext(); {
|
||||
sk.tmpSet.add(iter.Next())
|
||||
}
|
||||
sk.maybeToNormal()
|
||||
return nil
|
||||
}
|
||||
|
||||
if sk.sparse() {
|
||||
sk.toNormal()
|
||||
}
|
||||
|
||||
if cpOther.sparse() {
|
||||
for k := range cpOther.tmpSet {
|
||||
i, r := decodeHash(k, cpOther.p, pp)
|
||||
sk.insert(i, r)
|
||||
}
|
||||
|
||||
for iter := cpOther.sparseList.Iter(); iter.HasNext(); {
|
||||
i, r := decodeHash(iter.Next(), cpOther.p, pp)
|
||||
sk.insert(i, r)
|
||||
}
|
||||
} else {
|
||||
if sk.b < cpOther.b {
|
||||
sk.regs.rebase(cpOther.b - sk.b)
|
||||
sk.b = cpOther.b
|
||||
} else {
|
||||
cpOther.regs.rebase(sk.b - cpOther.b)
|
||||
cpOther.b = sk.b
|
||||
}
|
||||
|
||||
for i, v := range cpOther.regs.tailcuts {
|
||||
v1 := v.get(0)
|
||||
if v1 > sk.regs.get(uint32(i)*2) {
|
||||
sk.regs.set(uint32(i)*2, v1)
|
||||
}
|
||||
v2 := v.get(1)
|
||||
if v2 > sk.regs.get(1+uint32(i)*2) {
|
||||
sk.regs.set(1+uint32(i)*2, v2)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Convert from sparse Sketch to dense Sketch.
|
||||
func (sk *Sketch) toNormal() {
|
||||
if len(sk.tmpSet) > 0 {
|
||||
sk.mergeSparse()
|
||||
}
|
||||
|
||||
sk.regs = newRegisters(sk.m)
|
||||
for iter := sk.sparseList.Iter(); iter.HasNext(); {
|
||||
i, r := decodeHash(iter.Next(), sk.p, pp)
|
||||
sk.insert(i, r)
|
||||
}
|
||||
|
||||
sk.tmpSet = nil
|
||||
sk.sparseList = nil
|
||||
}
|
||||
|
||||
func (sk *Sketch) insert(i uint32, r uint8) bool {
|
||||
changed := false
|
||||
if r-sk.b >= capacity {
|
||||
//overflow
|
||||
db := sk.regs.min()
|
||||
if db > 0 {
|
||||
sk.b += db
|
||||
sk.regs.rebase(db)
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
if r > sk.b {
|
||||
val := r - sk.b
|
||||
if c1 := capacity - 1; c1 < val {
|
||||
val = c1
|
||||
}
|
||||
|
||||
if val > sk.regs.get(i) {
|
||||
sk.regs.set(i, val)
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
return changed
|
||||
}
|
||||
|
||||
// Insert adds element e to sketch
|
||||
func (sk *Sketch) Insert(e []byte) bool {
|
||||
x := hash(e)
|
||||
return sk.InsertHash(x)
|
||||
}
|
||||
|
||||
// InsertHash adds hash x to sketch
|
||||
func (sk *Sketch) InsertHash(x uint64) bool {
|
||||
if sk.sparse() {
|
||||
changed := sk.tmpSet.add(encodeHash(x, sk.p, pp))
|
||||
if !changed {
|
||||
return false
|
||||
}
|
||||
if uint32(len(sk.tmpSet))*100 > sk.m/2 {
|
||||
sk.mergeSparse()
|
||||
if uint32(sk.sparseList.Len()) > sk.m/2 {
|
||||
sk.toNormal()
|
||||
}
|
||||
}
|
||||
return true
|
||||
} else {
|
||||
i, r := getPosVal(x, sk.p)
|
||||
return sk.insert(uint32(i), r)
|
||||
}
|
||||
}
|
||||
|
||||
// Estimate returns the cardinality of the Sketch
|
||||
func (sk *Sketch) Estimate() uint64 {
|
||||
if sk.sparse() {
|
||||
sk.mergeSparse()
|
||||
return uint64(linearCount(mp, mp-sk.sparseList.count))
|
||||
}
|
||||
|
||||
sum, ez := sk.regs.sumAndZeros(sk.b)
|
||||
m := float64(sk.m)
|
||||
var est float64
|
||||
|
||||
var beta func(float64) float64
|
||||
if sk.p < 16 {
|
||||
beta = beta14
|
||||
} else {
|
||||
beta = beta16
|
||||
}
|
||||
|
||||
if sk.b == 0 {
|
||||
est = (sk.alpha * m * (m - ez) / (sum + beta(ez)))
|
||||
} else {
|
||||
est = (sk.alpha * m * m / sum)
|
||||
}
|
||||
|
||||
return uint64(est + 0.5)
|
||||
}
|
||||
|
||||
func (sk *Sketch) mergeSparse() {
|
||||
if len(sk.tmpSet) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
keys := make(uint64Slice, 0, len(sk.tmpSet))
|
||||
for k := range sk.tmpSet {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Sort(keys)
|
||||
|
||||
newList := newCompressedList()
|
||||
for iter, i := sk.sparseList.Iter(), 0; iter.HasNext() || i < len(keys); {
|
||||
if !iter.HasNext() {
|
||||
newList.Append(keys[i])
|
||||
i++
|
||||
continue
|
||||
}
|
||||
|
||||
if i >= len(keys) {
|
||||
newList.Append(iter.Next())
|
||||
continue
|
||||
}
|
||||
|
||||
x1, x2 := iter.Peek(), keys[i]
|
||||
if x1 == x2 {
|
||||
newList.Append(iter.Next())
|
||||
i++
|
||||
} else if x1 > x2 {
|
||||
newList.Append(x2)
|
||||
i++
|
||||
} else {
|
||||
newList.Append(iter.Next())
|
||||
}
|
||||
}
|
||||
|
||||
sk.sparseList = newList
|
||||
sk.tmpSet = set{}
|
||||
}
|
||||
|
||||
// MarshalBinary implements the encoding.BinaryMarshaler interface.
|
||||
func (sk *Sketch) MarshalBinary() (data []byte, err error) {
|
||||
// Marshal a version marker.
|
||||
data = append(data, version)
|
||||
// Marshal p.
|
||||
data = append(data, sk.p)
|
||||
// Marshal b
|
||||
data = append(data, sk.b)
|
||||
|
||||
if sk.sparse() {
|
||||
// It's using the sparse Sketch.
|
||||
data = append(data, byte(1))
|
||||
|
||||
// Add the tmp_set
|
||||
tsdata, err := sk.tmpSet.MarshalBinary()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
data = append(data, tsdata...)
|
||||
|
||||
// Add the sparse Sketch
|
||||
sdata, err := sk.sparseList.MarshalBinary()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return append(data, sdata...), nil
|
||||
}
|
||||
|
||||
// It's using the dense Sketch.
|
||||
data = append(data, byte(0))
|
||||
|
||||
// Add the dense sketch Sketch.
|
||||
sz := len(sk.regs.tailcuts)
|
||||
data = append(data, []byte{
|
||||
byte(sz >> 24),
|
||||
byte(sz >> 16),
|
||||
byte(sz >> 8),
|
||||
byte(sz),
|
||||
}...)
|
||||
|
||||
// Marshal each element in the list.
|
||||
for i := 0; i < len(sk.regs.tailcuts); i++ {
|
||||
data = append(data, byte(sk.regs.tailcuts[i]))
|
||||
}
|
||||
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// ErrorTooShort is an error that UnmarshalBinary try to parse too short
|
||||
// binary.
|
||||
var ErrorTooShort = errors.New("too short binary")
|
||||
|
||||
// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface.
|
||||
func (sk *Sketch) UnmarshalBinary(data []byte) error {
|
||||
if len(data) < 8 {
|
||||
return ErrorTooShort
|
||||
}
|
||||
|
||||
// Unmarshal version. We may need this in the future if we make
|
||||
// non-compatible changes.
|
||||
_ = data[0]
|
||||
|
||||
// Unmarshal p.
|
||||
p := data[1]
|
||||
|
||||
// Unmarshal b.
|
||||
sk.b = data[2]
|
||||
|
||||
// Determine if we need a sparse Sketch
|
||||
sparse := data[3] == byte(1)
|
||||
|
||||
// Make a newSketch Sketch if the precision doesn't match or if the Sketch was used
|
||||
if sk.p != p || sk.regs != nil || len(sk.tmpSet) > 0 || (sk.sparseList != nil && sk.sparseList.Len() > 0) {
|
||||
newh, err := newSketch(p, sparse)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
newh.b = sk.b
|
||||
*sk = *newh
|
||||
}
|
||||
|
||||
// h is now initialised with the correct p. We just need to fill the
|
||||
// rest of the details out.
|
||||
if sparse {
|
||||
// Using the sparse Sketch.
|
||||
|
||||
// Unmarshal the tmp_set.
|
||||
tssz := binary.BigEndian.Uint32(data[4:8])
|
||||
sk.tmpSet = make(map[uint32]struct{}, tssz)
|
||||
|
||||
// We need to unmarshal tssz values in total, and each value requires us
|
||||
// to read 4 bytes.
|
||||
tsLastByte := int((tssz * 4) + 8)
|
||||
for i := 8; i < tsLastByte; i += 4 {
|
||||
k := binary.BigEndian.Uint32(data[i : i+4])
|
||||
sk.tmpSet[k] = struct{}{}
|
||||
}
|
||||
|
||||
// Unmarshal the sparse Sketch.
|
||||
return sk.sparseList.UnmarshalBinary(data[tsLastByte:])
|
||||
}
|
||||
|
||||
// Using the dense Sketch.
|
||||
sk.sparseList = nil
|
||||
sk.tmpSet = nil
|
||||
dsz := binary.BigEndian.Uint32(data[4:8])
|
||||
sk.regs = newRegisters(dsz * 2)
|
||||
data = data[8:]
|
||||
|
||||
for i, val := range data {
|
||||
sk.regs.tailcuts[i] = reg(val)
|
||||
if uint8(sk.regs.tailcuts[i]<<4>>4) > 0 {
|
||||
sk.regs.nz--
|
||||
}
|
||||
if uint8(sk.regs.tailcuts[i]>>4) > 0 {
|
||||
sk.regs.nz--
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
package hyperloglog
|
||||
|
||||
import (
|
||||
"math"
|
||||
)
|
||||
|
||||
type reg uint8
|
||||
type tailcuts []reg
|
||||
|
||||
type registers struct {
|
||||
tailcuts
|
||||
nz uint32
|
||||
}
|
||||
|
||||
func (r *reg) set(offset, val uint8) bool {
|
||||
var isZero bool
|
||||
if offset == 0 {
|
||||
isZero = *r < 16
|
||||
tmpVal := uint8((*r) << 4 >> 4)
|
||||
*r = reg(tmpVal | (val << 4))
|
||||
} else {
|
||||
isZero = *r&0x0f == 0
|
||||
tmpVal := uint8((*r) >> 4 << 4)
|
||||
*r = reg(tmpVal | val)
|
||||
}
|
||||
return isZero
|
||||
}
|
||||
|
||||
func (r *reg) get(offset uint8) uint8 {
|
||||
if offset == 0 {
|
||||
return uint8((*r) >> 4)
|
||||
}
|
||||
return uint8((*r) << 4 >> 4)
|
||||
}
|
||||
|
||||
func newRegisters(size uint32) *registers {
|
||||
return ®isters{
|
||||
tailcuts: make(tailcuts, size/2),
|
||||
nz: size,
|
||||
}
|
||||
}
|
||||
|
||||
func (rs *registers) clone() *registers {
|
||||
if rs == nil {
|
||||
return nil
|
||||
}
|
||||
tc := make([]reg, len(rs.tailcuts))
|
||||
copy(tc, rs.tailcuts)
|
||||
return ®isters{
|
||||
tailcuts: tc,
|
||||
nz: rs.nz,
|
||||
}
|
||||
}
|
||||
|
||||
func (rs *registers) rebase(delta uint8) {
|
||||
nz := uint32(len(rs.tailcuts)) * 2
|
||||
for i := range rs.tailcuts {
|
||||
for j := uint8(0); j < 2; j++ {
|
||||
val := rs.tailcuts[i].get(j)
|
||||
if val >= delta {
|
||||
rs.tailcuts[i].set(j, val-delta)
|
||||
if val-delta > 0 {
|
||||
nz--
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
rs.nz = nz
|
||||
}
|
||||
|
||||
func (rs *registers) set(i uint32, val uint8) {
|
||||
offset, index := uint8(i)&1, i/2
|
||||
if rs.tailcuts[index].set(offset, val) {
|
||||
rs.nz--
|
||||
}
|
||||
}
|
||||
|
||||
func (rs *registers) get(i uint32) uint8 {
|
||||
offset, index := uint8(i)&1, i/2
|
||||
return rs.tailcuts[index].get(offset)
|
||||
}
|
||||
|
||||
func (rs *registers) sumAndZeros(base uint8) (res, ez float64) {
|
||||
for _, r := range rs.tailcuts {
|
||||
for j := uint8(0); j < 2; j++ {
|
||||
v := float64(base + r.get(j))
|
||||
if v == 0 {
|
||||
ez++
|
||||
}
|
||||
res += 1.0 / math.Pow(2.0, v)
|
||||
}
|
||||
}
|
||||
rs.nz = uint32(ez)
|
||||
return res, ez
|
||||
}
|
||||
|
||||
func (rs *registers) min() uint8 {
|
||||
if rs.nz > 0 {
|
||||
return 0
|
||||
}
|
||||
min := uint8(math.MaxUint8)
|
||||
for _, r := range rs.tailcuts {
|
||||
if r == 0 || min == 0 {
|
||||
return 0
|
||||
}
|
||||
if val := uint8(r << 4 >> 4); val < min {
|
||||
min = val
|
||||
}
|
||||
if val := uint8(r >> 4); val < min {
|
||||
min = val
|
||||
}
|
||||
}
|
||||
return min
|
||||
}
|
||||
+92
@@ -0,0 +1,92 @@
|
||||
package hyperloglog
|
||||
|
||||
import (
|
||||
"math/bits"
|
||||
)
|
||||
|
||||
func getIndex(k uint32, p, pp uint8) uint32 {
|
||||
if k&1 == 1 {
|
||||
return bextr32(k, 32-p, p)
|
||||
}
|
||||
return bextr32(k, pp-p+1, p)
|
||||
}
|
||||
|
||||
// Encode a hash to be used in the sparse representation.
|
||||
func encodeHash(x uint64, p, pp uint8) uint32 {
|
||||
idx := uint32(bextr(x, 64-pp, pp))
|
||||
if bextr(x, 64-pp, pp-p) == 0 {
|
||||
zeros := bits.LeadingZeros64((bextr(x, 0, 64-pp)<<pp)|(1<<pp-1)) + 1
|
||||
return idx<<7 | uint32(zeros<<1) | 1
|
||||
}
|
||||
return idx << 1
|
||||
}
|
||||
|
||||
// Decode a hash from the sparse representation.
|
||||
func decodeHash(k uint32, p, pp uint8) (uint32, uint8) {
|
||||
var r uint8
|
||||
if k&1 == 1 {
|
||||
r = uint8(bextr32(k, 1, 6)) + pp - p
|
||||
} else {
|
||||
// We can use the 64bit clz implementation and reduce the result
|
||||
// by 32 to get a clz for a 32bit word.
|
||||
r = uint8(bits.LeadingZeros64(uint64(k<<(32-pp+p-1))) - 31) // -32 + 1
|
||||
}
|
||||
return getIndex(k, p, pp), r
|
||||
}
|
||||
|
||||
type set map[uint32]struct{}
|
||||
|
||||
func (s set) add(v uint32) bool {
|
||||
_, ok := s[v]
|
||||
if ok {
|
||||
return false
|
||||
}
|
||||
s[v] = struct{}{}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s set) Clone() set {
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
newS := make(map[uint32]struct{}, len(s))
|
||||
for k, v := range s {
|
||||
newS[k] = v
|
||||
}
|
||||
return newS
|
||||
}
|
||||
|
||||
func (s set) MarshalBinary() (data []byte, err error) {
|
||||
// 4 bytes for the size of the set, and 4 bytes for each key.
|
||||
// list.
|
||||
data = make([]byte, 0, 4+(4*len(s)))
|
||||
|
||||
// Length of the set. We only need 32 bits because the size of the set
|
||||
// couldn't exceed that on 32 bit architectures.
|
||||
sl := len(s)
|
||||
data = append(data, []byte{
|
||||
byte(sl >> 24),
|
||||
byte(sl >> 16),
|
||||
byte(sl >> 8),
|
||||
byte(sl),
|
||||
}...)
|
||||
|
||||
// Marshal each element in the set.
|
||||
for k := range s {
|
||||
data = append(data, []byte{
|
||||
byte(k >> 24),
|
||||
byte(k >> 16),
|
||||
byte(k >> 8),
|
||||
byte(k),
|
||||
}...)
|
||||
}
|
||||
|
||||
return data, nil
|
||||
}
|
||||
|
||||
type uint64Slice []uint32
|
||||
|
||||
func (p uint64Slice) Len() int { return len(p) }
|
||||
func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] }
|
||||
func (p uint64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
package hyperloglog
|
||||
|
||||
import (
|
||||
"github.com/alicebob/miniredis/v2/metro"
|
||||
"math"
|
||||
"math/bits"
|
||||
)
|
||||
|
||||
var hash = hashFunc
|
||||
|
||||
func beta14(ez float64) float64 {
|
||||
zl := math.Log(ez + 1)
|
||||
return -0.370393911*ez +
|
||||
0.070471823*zl +
|
||||
0.17393686*math.Pow(zl, 2) +
|
||||
0.16339839*math.Pow(zl, 3) +
|
||||
-0.09237745*math.Pow(zl, 4) +
|
||||
0.03738027*math.Pow(zl, 5) +
|
||||
-0.005384159*math.Pow(zl, 6) +
|
||||
0.00042419*math.Pow(zl, 7)
|
||||
}
|
||||
|
||||
func beta16(ez float64) float64 {
|
||||
zl := math.Log(ez + 1)
|
||||
return -0.37331876643753059*ez +
|
||||
-1.41704077448122989*zl +
|
||||
0.40729184796612533*math.Pow(zl, 2) +
|
||||
1.56152033906584164*math.Pow(zl, 3) +
|
||||
-0.99242233534286128*math.Pow(zl, 4) +
|
||||
0.26064681399483092*math.Pow(zl, 5) +
|
||||
-0.03053811369682807*math.Pow(zl, 6) +
|
||||
0.00155770210179105*math.Pow(zl, 7)
|
||||
}
|
||||
|
||||
func alpha(m float64) float64 {
|
||||
switch m {
|
||||
case 16:
|
||||
return 0.673
|
||||
case 32:
|
||||
return 0.697
|
||||
case 64:
|
||||
return 0.709
|
||||
}
|
||||
return 0.7213 / (1 + 1.079/m)
|
||||
}
|
||||
|
||||
func getPosVal(x uint64, p uint8) (uint64, uint8) {
|
||||
i := bextr(x, 64-p, p) // {x63,...,x64-p}
|
||||
w := x<<p | 1<<(p-1) // {x63-p,...,x0}
|
||||
rho := uint8(bits.LeadingZeros64(w)) + 1
|
||||
return i, rho
|
||||
}
|
||||
|
||||
func linearCount(m uint32, v uint32) float64 {
|
||||
fm := float64(m)
|
||||
return fm * math.Log(fm/float64(v))
|
||||
}
|
||||
|
||||
func bextr(v uint64, start, length uint8) uint64 {
|
||||
return (v >> start) & ((1 << length) - 1)
|
||||
}
|
||||
|
||||
func bextr32(v uint32, start, length uint8) uint32 {
|
||||
return (v >> start) & ((1 << length) - 1)
|
||||
}
|
||||
|
||||
func hashFunc(e []byte) uint64 {
|
||||
return metro.Hash64(e, 1337)
|
||||
}
|
||||
Reference in New Issue
Block a user