Merge pull request #16 from zuoyebang/dev

v6.0.0

butils/md5hash/md5.go

@@ -1,3 +1,17 @@
+// Copyright 2019-2024 Xu Ruibo ([email protected]) and Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package md5hash
 
 import (
@@ -6,10 +20,10 @@ import (
 	"hash"
 )
 
-// Size The size of an MD5 checksum in bytes.
+// The size of an MD5 checksum in bytes.
 const Size = 16
 
-// BlockSize The blocksize of MD5 in bytes.
+// The blocksize of MD5 in bytes.
 const BlockSize = 64
 
 const (
@@ -184,10 +198,12 @@ func MD5Hash(data []byte) (h []byte, hi, lo uint64) {
 }
 
 //go:inline
-func MD5Sum(p []byte) (h []byte, hi, lo uint64) {
-	var d = digest{s: [4]uint32{init0, init1, init2, init3}}
+func MD5Sum(p []byte, h []byte) (hi, lo uint64) {
+	var d = digest{
+		s:   [4]uint32{init0, init1, init2, init3},
+		len: uint64(len(p)),
+	}
 	// Write
-	d.len += uint64(len(p))
 	if d.nx > 0 {
 		n := copy(d.x[d.nx:], p)
 		d.nx += n
@@ -251,7 +267,6 @@ func MD5Sum(p []byte) (h []byte, hi, lo uint64) {
 		panic("d.nx != 0")
 	}
 
-	h = make([]byte, Size)
 	binary.LittleEndian.PutUint32(h[0:], d.s[0])
 	binary.LittleEndian.PutUint32(h[4:], d.s[1])
 	binary.LittleEndian.PutUint32(h[8:], d.s[2])

butils/md5hash/md5_test.go

@@ -1,6 +1,16 @@
-// Copyright 2009 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
+// Copyright 2019-2024 Xu Ruibo ([email protected]) and Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
 
 package md5hash
 
@@ -228,9 +238,10 @@ func TestAllocations(t *testing.T) {
 func TestMD5Hash(t *testing.T) {
 	in := []byte("hello, world!")
 	h, hi, lo := MD5Hash(in)
-	h2, hi2, lo2 := MD5Sum(in)
+	var h2 [Size]byte
+	hi2, lo2 := MD5Sum(in, h2[:])
 
-	if !bytes.Equal(h, h2) {
+	if !bytes.Equal(h, h2[:]) {
 		t.Errorf("MD5Hash(in) = %x, want %x", h, h2)
 	}
 	if hi != hi2 {

butils/md5hash/md5block_arm.s

@@ -12,8 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// ARM version of md5block.go
-
 #include "textflag.h"
 
 // Register definitions

butils/md5hash/md5block_arm64.s

@@ -12,9 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// ARM64 version of md5block.go
-// derived from crypto/md5/md5block_amd64.s
-
 #include "textflag.h"
 
 TEXT	·block(SB),NOSPLIT,$0-32

butils/vectormap/kvholder.go

@@ -234,8 +234,5 @@ func LoadUint32(buf []byte) (dest uint32) {
 
 //go:inline
 func Cap4Size(vSize uint32) uint32 {
-	if vSize&3 != 0 {
-		return (vSize>>2 + 1) << 2
-	}
-	return vSize
+	return (vSize + 3) &^ 3
 }

butils/vectormap/lfumap.go

@@ -53,8 +53,8 @@ func newInnerLFUMap(owner *VectorMap, sz uint32) (m *LFUMap) {
 		limit:    groups * maxAvgGroupLoad,
 	}
 	memMax := owner.memCap / Byte(owner.buckets)
-	if memMax > 64*MB || memMax <= 0 {
-		memMax = 64 * MB
+	if memMax > maxShardMemSize || memMax <= 0 {
+		memMax = maxShardMemSize
 	}
 	for i := range m.ctrl {
 		m.ctrl[i] = newEmptyMetadata()
@@ -850,6 +850,8 @@ func (m *LFUMap) Delete(l uint64, key []byte) (ok bool) {
 }
 
 func (m *LFUMap) Clear() {
+	m.putLock.Lock()
+	m.rehashLock.Lock()
 	for i, c := range m.ctrl {
 		for j := range c {
 			m.ctrl[i][j] = empty
@@ -867,8 +869,27 @@ func (m *LFUMap) Clear() {
 	}
 	m.resident, m.dead = 0, 0
 
+	kvholder := newKVHolder(Byte(m.kvHolder.cap))
+	m.kvHolder.cap = 0
+	m.kvHolder.buffer.release()
+	m.kvHolder = kvholder
+	m.rehashLock.Unlock()
+	m.putLock.Unlock()
+}
+
+func (m *LFUMap) Close() {
+	m.putLock.Lock()
+	m.rehashLock.Lock()
+	m.ctrl = nil
+	m.counters = nil
+	m.groups = nil
+	m.resident, m.dead = 0, 0
 	m.kvHolder.cap = 0
-	m.kvHolder.data = nil
+	m.kvHolder.buffer.release()
+	m.kvHolder = nil
+	m.owner = nil
+	m.rehashLock.Unlock()
+	m.putLock.Unlock()
 }
 
 func (m *LFUMap) QueryCount() (count uint64) {
@@ -1000,7 +1021,7 @@ func (m *LFUMap) Eliminate() (delCount int, skipReason int) {
 	return
 }
 
-func (m *LFUMap) GCCopy() (deadCount int, gcMem int, subSince bool, skipReason int) {
+func (m *LFUMap) GCCopy() (deadCount int, gcMem int, skipReason int) {
 	if m.garbageUsage() < garbageRate {
 		skipReason = skipReason1
 		return

butils/vectormap/lrumap.go

@@ -22,14 +22,14 @@ import (
 	"time"
 	"unsafe"
 
-	"github.com/zuoyebang/bitalostored/butils/vectormap/simd"
-
 	"github.com/zuoyebang/bitalostored/butils/md5hash"
+	"github.com/zuoyebang/bitalostored/butils/vectormap/simd"
 )
 
 var UnitTime = 30 * time.Second
 
 const LRUSubDuration = 24 * time.Hour
+const LRUMaxDuration = 20 * 24 * time.Hour
 
 type LRUMap struct {
 	owner      *VectorMap
@@ -64,8 +64,8 @@ func newInnerLRUMap(owner *VectorMap, sz uint32) (m *LRUMap) {
 		limit:       groups * maxAvgGroupLoad,
 	}
 	memMax := owner.memCap / Byte(owner.buckets)
-	if memMax > 64*MB || memMax <= 0 {
-		memMax = 64 * MB
+	if !owner.skipCheck && (memMax > maxShardMemSize || memMax <= 0) {
+		memMax = maxShardMemSize
 	}
 	for i := range m.ctrl {
 		m.ctrl[i] = newEmptyMetadata()
@@ -878,6 +878,8 @@ func (m *LRUMap) Delete(l uint64, key []byte) (ok bool) {
 }
 
 func (m *LRUMap) Clear() {
+	m.putLock.Lock()
+	m.rehashLock.Lock()
 	for i, c := range m.ctrl {
 		for j := range c {
 			m.ctrl[i][j] = empty
@@ -895,8 +897,27 @@ func (m *LRUMap) Clear() {
 	}
 	m.resident, m.dead = 0, 0
 
+	kvholder := newKVHolder(Byte(m.kvHolder.cap))
+	m.kvHolder.cap = 0
+	m.kvHolder.buffer.release()
+	m.kvHolder = kvholder
+	m.rehashLock.Unlock()
+	m.putLock.Unlock()
+}
+
+func (m *LRUMap) Close() {
+	m.putLock.Lock()
+	m.rehashLock.Lock()
+	m.ctrl = nil
+	m.sinces = nil
+	m.groups = nil
+	m.resident, m.dead = 0, 0
 	m.kvHolder.cap = 0
-	m.kvHolder.data = nil
+	m.kvHolder.buffer.release()
+	m.kvHolder = nil
+	m.owner = nil
+	m.rehashLock.Unlock()
+	m.putLock.Unlock()
 }
 
 func (m *LRUMap) QueryCount() (count uint64) {
@@ -972,6 +993,9 @@ func (m *LRUMap) rehash() {
 	m.limit = n * maxAvgGroupLoad
 	m.resident, m.dead = resident, 0
 	m.rehashLock.Unlock()
+	if m.owner.logger != nil {
+		m.owner.logger.Infof("rehash done, new size: %d", n)
+	}
 }
 
 func (m *LRUMap) loadFactor() float32 {
@@ -1021,7 +1045,27 @@ func (m *LRUMap) Eliminate() (delCount int, skipReason int) {
 	return
 }
 
-func (m *LRUMap) GCCopy() (deadCount int, gcMem int, subSince bool, skipReason int) {
+func (m *LRUMap) AdaptStartTime() (subSince bool) {
+	if time.Since(m.lastSubTime) > LRUSubDuration {
+		if time.Since(m.startTime.Add(time.Duration(m.minTopSince)*UnitTime)) > LRUMaxDuration {
+			m.minTopSince = uint16(LRUSubDuration / UnitTime)
+		}
+		m.lastSubTime = time.Now()
+		var level [16]uint16
+		for i := 0; i < 16; i++ {
+			level[i] = m.minTopSince
+		}
+		ctrLen := len(m.ctrl)
+		for i := 0; i < ctrLen; i++ {
+			simd.MSubs256epu16(unsafe.Pointer(&(m.sinces[i])), unsafe.Pointer(&level), unsafe.Pointer(&(m.sinces[i])))
+		}
+		m.startTime = m.startTime.Add(time.Duration(m.minTopSince) * UnitTime)
+		subSince = true
+	}
+	return
+}
+
+func (m *LRUMap) GCCopy() (deadCount int, gcMem int, skipReason int) {
 	if m.garbageUsage() < garbageRate {
 		skipReason = skipReason1
 		return
@@ -1074,20 +1118,6 @@ func (m *LRUMap) GCCopy() (deadCount int, gcMem int, subSince bool, skipReason i
 		}
 	}
 
-	if time.Since(m.lastSubTime) > LRUSubDuration && m.minTopSince > 0 {
-		m.lastSubTime = time.Now()
-		var level [16]uint16
-		for i := 0; i < 16; i++ {
-			level[i] = m.minTopSince
-		}
-		ctrLen := len(m.ctrl)
-		for i := 0; i < ctrLen; i++ {
-			simd.MSubs256epu16(unsafe.Pointer(&(sinces[i])), unsafe.Pointer(&level), unsafe.Pointer(&(sinces[i])))
-		}
-		m.startTime = m.startTime.Add(time.Duration(m.minTopSince) * UnitTime)
-		subSince = true
-	}
-
 	m.rehashLock.Lock()
 	m.groups = groups
 	m.ctrl = ctrl