seaweedfs/weed/pb/filer_pb/filer_client_bfs.go

package filer_pb

import (
	"context"
	"fmt"
	"github.com/seaweedfs/seaweedfs/weed/glog"
	"io"
	"sync"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/util"
)

func TraverseBfs(filerClient FilerClient, parentPath util.FullPath, fn func(parentPath util.FullPath, entry *Entry)) (err error) {
	K := 5

	var jobQueueWg sync.WaitGroup
	queue := util.NewQueue[util.FullPath]()
	jobQueueWg.Add(1)
	queue.Enqueue(parentPath)
	terminates := make([]chan bool, K)

	for i := 0; i < K; i++ {
		terminates[i] = make(chan bool)
		go func(j int) {
			for {
				select {
				case <-terminates[j]:
					return
				default:
					t := queue.Dequeue()
					if t == "" {
						time.Sleep(329 * time.Millisecond)
						continue
					}
					dir := t
					processErr := processOneDirectory(filerClient, dir, queue, &jobQueueWg, fn)
					if processErr != nil {
						err = processErr
					}
					jobQueueWg.Done()
				}
			}
		}(i)
	}
	jobQueueWg.Wait()
	for i := 0; i < K; i++ {
		close(terminates[i])
	}
	return
}

func processOneDirectory(filerClient FilerClient, parentPath util.FullPath, queue *util.Queue[util.FullPath], jobQueueWg *sync.WaitGroup, fn func(parentPath util.FullPath, entry *Entry)) (err error) {

	return ReadDirAllEntries(filerClient, parentPath, "", func(entry *Entry, isLast bool) error {

		fn(parentPath, entry)

		if entry.IsDirectory {
			subDir := fmt.Sprintf("%s/%s", parentPath, entry.Name)
			if parentPath == "/" {
				subDir = "/" + entry.Name
			}
			jobQueueWg.Add(1)
			queue.Enqueue(util.FullPath(subDir))
		}
		return nil
	})

}

func StreamBfs(client SeaweedFilerClient, dir util.FullPath, olderThanTsNs int64, fn func(parentPath util.FullPath, entry *Entry)error) (err error) {
	glog.V(0).Infof("TraverseBfsMetadata %v if before %v", dir, time.Unix(0, olderThanTsNs))
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	stream, err := client.TraverseBfsMetadata(ctx, &TraverseBfsMetadataRequest{
		Directory: string(dir),
	})
	if err != nil {
		return fmt.Errorf("traverse bfs metadata: %v", err)
	}
	for {
		resp, err := stream.Recv()
		if err != nil {
			if err == io.EOF {
				break
			}
			return fmt.Errorf("traverse bfs metadata: %v", err)
		}
		if err := fn(util.FullPath(resp.Directory),  resp.Entry); err != nil {
			return err
		}
	}
	return nil
}