This commit is contained in:
chrislu 2023-11-26 11:47:20 -08:00
parent 666a22b980
commit 81f11883e3
7 changed files with 20 additions and 21 deletions

View File

@ -24,7 +24,7 @@ func init() {
} }
type EtcdStore struct { type EtcdStore struct {
client *clientv3.Client client *clientv3.Client
etcdKeyPrefix string etcdKeyPrefix string
} }
@ -38,9 +38,9 @@ func (store *EtcdStore) Initialize(configuration weed_util.Configuration, prefix
servers = "localhost:2379" servers = "localhost:2379"
} }
username := configuration.GetString(prefix + "username") username := configuration.GetString(prefix + "username")
password := configuration.GetString(prefix + "password") password := configuration.GetString(prefix + "password")
store.etcdKeyPrefix = configuration.GetString(prefix + "key_prefix") store.etcdKeyPrefix = configuration.GetString(prefix + "key_prefix")
timeout := configuration.GetString(prefix + "timeout") timeout := configuration.GetString(prefix + "timeout")
if timeout == "" { if timeout == "" {
@ -93,7 +93,7 @@ func (store *EtcdStore) InsertEntry(ctx context.Context, entry *filer.Entry) (er
meta = weed_util.MaybeGzipData(meta) meta = weed_util.MaybeGzipData(meta)
} }
if _, err := store.client.Put(ctx, store.etcdKeyPrefix + string(key), string(meta)); err != nil { if _, err := store.client.Put(ctx, store.etcdKeyPrefix+string(key), string(meta)); err != nil {
return fmt.Errorf("persisting %s : %v", entry.FullPath, err) return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
} }
@ -107,7 +107,7 @@ func (store *EtcdStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (er
func (store *EtcdStore) FindEntry(ctx context.Context, fullpath weed_util.FullPath) (entry *filer.Entry, err error) { func (store *EtcdStore) FindEntry(ctx context.Context, fullpath weed_util.FullPath) (entry *filer.Entry, err error) {
key := genKey(fullpath.DirAndName()) key := genKey(fullpath.DirAndName())
resp, err := store.client.Get(ctx, store.etcdKeyPrefix + string(key)) resp, err := store.client.Get(ctx, store.etcdKeyPrefix+string(key))
if err != nil { if err != nil {
return nil, fmt.Errorf("get %s : %v", fullpath, err) return nil, fmt.Errorf("get %s : %v", fullpath, err)
} }
@ -130,7 +130,7 @@ func (store *EtcdStore) FindEntry(ctx context.Context, fullpath weed_util.FullPa
func (store *EtcdStore) DeleteEntry(ctx context.Context, fullpath weed_util.FullPath) (err error) { func (store *EtcdStore) DeleteEntry(ctx context.Context, fullpath weed_util.FullPath) (err error) {
key := genKey(fullpath.DirAndName()) key := genKey(fullpath.DirAndName())
if _, err := store.client.Delete(ctx, store.etcdKeyPrefix + string(key)); err != nil { if _, err := store.client.Delete(ctx, store.etcdKeyPrefix+string(key)); err != nil {
return fmt.Errorf("delete %s : %v", fullpath, err) return fmt.Errorf("delete %s : %v", fullpath, err)
} }
@ -140,7 +140,7 @@ func (store *EtcdStore) DeleteEntry(ctx context.Context, fullpath weed_util.Full
func (store *EtcdStore) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath) (err error) { func (store *EtcdStore) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath) (err error) {
directoryPrefix := genDirectoryKeyPrefix(fullpath, "") directoryPrefix := genDirectoryKeyPrefix(fullpath, "")
if _, err := store.client.Delete(ctx, store.etcdKeyPrefix + string(directoryPrefix), clientv3.WithPrefix()); err != nil { if _, err := store.client.Delete(ctx, store.etcdKeyPrefix+string(directoryPrefix), clientv3.WithPrefix()); err != nil {
return fmt.Errorf("deleteFolderChildren %s : %v", fullpath, err) return fmt.Errorf("deleteFolderChildren %s : %v", fullpath, err)
} }
@ -158,7 +158,7 @@ func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, dirPath weed_u
lastFileStart = genDirectoryKeyPrefix(dirPath, startFileName) lastFileStart = genDirectoryKeyPrefix(dirPath, startFileName)
} }
resp, err := store.client.Get(ctx, store.etcdKeyPrefix + string(lastFileStart), resp, err := store.client.Get(ctx, store.etcdKeyPrefix+string(lastFileStart),
clientv3.WithFromKey(), clientv3.WithLimit(limit+1)) clientv3.WithFromKey(), clientv3.WithLimit(limit+1))
if err != nil { if err != nil {
return lastFileName, fmt.Errorf("list %s : %v", dirPath, err) return lastFileName, fmt.Errorf("list %s : %v", dirPath, err)

View File

@ -8,7 +8,7 @@ import (
func (store *EtcdStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) { func (store *EtcdStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) {
_, err = store.client.Put(ctx, store.etcdKeyPrefix + string(key), string(value)) _, err = store.client.Put(ctx, store.etcdKeyPrefix+string(key), string(value))
if err != nil { if err != nil {
return fmt.Errorf("kv put: %v", err) return fmt.Errorf("kv put: %v", err)
@ -19,7 +19,7 @@ func (store *EtcdStore) KvPut(ctx context.Context, key []byte, value []byte) (er
func (store *EtcdStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) { func (store *EtcdStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) {
resp, err := store.client.Get(ctx, store.etcdKeyPrefix + string(key)) resp, err := store.client.Get(ctx, store.etcdKeyPrefix+string(key))
if err != nil { if err != nil {
return nil, fmt.Errorf("kv get: %v", err) return nil, fmt.Errorf("kv get: %v", err)
@ -34,7 +34,7 @@ func (store *EtcdStore) KvGet(ctx context.Context, key []byte) (value []byte, er
func (store *EtcdStore) KvDelete(ctx context.Context, key []byte) (err error) { func (store *EtcdStore) KvDelete(ctx context.Context, key []byte) (err error) {
_, err = store.client.Delete(ctx, store.etcdKeyPrefix + string(key)) _, err = store.client.Delete(ctx, store.etcdKeyPrefix+string(key))
if err != nil { if err != nil {
return fmt.Errorf("kv delete: %v", err) return fmt.Errorf("kv delete: %v", err)

View File

@ -234,7 +234,6 @@ var (
Help: "Bucketed histogram of s3 time to first byte request processing time.", Help: "Bucketed histogram of s3 time to first byte request processing time.",
Buckets: prometheus.ExponentialBuckets(0.001, 2, 27), Buckets: prometheus.ExponentialBuckets(0.001, 2, 27),
}, []string{"type", "bucket"}) }, []string{"type", "bucket"})
) )
func init() { func init() {

View File

@ -10,8 +10,8 @@ const (
ErrorSizeMismatch = "errorSizeMismatch" ErrorSizeMismatch = "errorSizeMismatch"
ErrorCRC = "errorCRC" ErrorCRC = "errorCRC"
ErrorIndexOutOfRange = "errorIndexOutOfRange" ErrorIndexOutOfRange = "errorIndexOutOfRange"
ErrorGetNotFound = "errorGetNotFound" ErrorGetNotFound = "errorGetNotFound"
ErrorGetInternal = "errorGetInternal" ErrorGetInternal = "errorGetInternal"
// master topology // master topology
ErrorWriteToLocalDisk = "errorWriteToLocalDisk" ErrorWriteToLocalDisk = "errorWriteToLocalDisk"

View File

@ -89,4 +89,4 @@ func PrintMemUsage(totalRowCount uint64) {
} }
func bToMb(b uint64) uint64 { func bToMb(b uint64) uint64 {
return b / 1024 / 1024 return b / 1024 / 1024
} }

View File

@ -196,7 +196,7 @@ func TestCompactSection_Get(t *testing.T) {
maps = append(maps, m) maps = append(maps, m)
totalRowCount += rowCount totalRowCount += rowCount
m.Set(1574318345753513987, ToOffset(10002), 10002) m.Set(1574318345753513987, ToOffset(10002), 10002)
nv,ok := m.Get(1574318345753513987) nv, ok := m.Get(1574318345753513987)
if ok { if ok {
t.Log(uint64(nv.Key)) t.Log(uint64(nv.Key))
} }
@ -207,14 +207,14 @@ func TestCompactSection_Get(t *testing.T) {
} }
m.Set(1574318350048481283, ToOffset(10002), 10002) m.Set(1574318350048481283, ToOffset(10002), 10002)
nv2,ok1 := m.Get(1574318350048481283) nv2, ok1 := m.Get(1574318350048481283)
if ok1 { if ok1 {
t.Log(uint64(nv2.Key)) t.Log(uint64(nv2.Key))
} }
m.Delete(nv2.Key) m.Delete(nv2.Key)
nv3,has := m.Get(nv2.Key) nv3, has := m.Get(nv2.Key)
if has && nv3.Size > 0 { if has && nv3.Size > 0 {
t.Error(uint64(nv3.Size)) t.Error(uint64(nv3.Size))
} }
} }

View File

@ -1065,7 +1065,7 @@ func (f *FlagSet) ParseEnv(environ []string) error {
} }
envKey = strings.Replace(envKey, "-", "_", -1) envKey = strings.Replace(envKey, "-", "_", -1)
envKey = strings.Replace(envKey, ".", "_", -1) envKey = strings.Replace(envKey, ".", "_", -1)
value, isSet := env[envKey] value, isSet := env[envKey]
if !isSet { if !isSet {
continue continue