ADHOC: add read needle meta grpc (#3581)

* ADHOC: add read needle meta grpc

* add test

* nit

Co-authored-by: root <root@HQ-10MSTD3EY.roblox.local>

weed/pb/filer_pb/filer.pb.go

@@ -1,6 +1,6 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.28.0
+// 	protoc-gen-go v1.28.1
 // 	protoc        v3.21.4
 // source: filer.proto
 

weed/pb/filer_pb/filer_grpc.pb.go

@@ -1,4 +1,8 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.2.0
+// - protoc             v3.21.4
+// source: filer.proto
 
 package filer_pb
 
@@ -11,6 +15,7 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.32.0 or later.
 const _ = grpc.SupportPackageIsVersion7
 
 // SeaweedFilerClient is the client API for SeaweedFiler service.

weed/pb/iam_pb/iam.pb.go

@@ -1,6 +1,6 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.28.0
+// 	protoc-gen-go v1.28.1
 // 	protoc        v3.21.4
 // source: iam.proto
 

weed/pb/iam_pb/iam_grpc.pb.go

@@ -1,4 +1,8 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.2.0
+// - protoc             v3.21.4
+// source: iam.proto
 
 package iam_pb
 
@@ -8,6 +12,7 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.32.0 or later.
 const _ = grpc.SupportPackageIsVersion7
 
 // SeaweedIdentityAccessManagementClient is the client API for SeaweedIdentityAccessManagement service.

weed/pb/master_pb/master.pb.go

@@ -1,6 +1,6 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.28.0
+// 	protoc-gen-go v1.28.1
 // 	protoc        v3.21.4
 // source: master.proto
 

weed/pb/master_pb/master_grpc.pb.go

@@ -1,4 +1,8 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.2.0
+// - protoc             v3.21.4
+// source: master.proto
 
 package master_pb
 
@@ -11,6 +15,7 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.32.0 or later.
 const _ = grpc.SupportPackageIsVersion7
 
 // SeaweedClient is the client API for Seaweed service.

weed/pb/mount_pb/mount.pb.go

@@ -1,6 +1,6 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.28.0
+// 	protoc-gen-go v1.28.1
 // 	protoc        v3.21.4
 // source: mount.proto
 

weed/pb/mount_pb/mount_grpc.pb.go

@@ -1,4 +1,8 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.2.0
+// - protoc             v3.21.4
+// source: mount.proto
 
 package mount_pb
 
@@ -11,6 +15,7 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.32.0 or later.
 const _ = grpc.SupportPackageIsVersion7
 
 // SeaweedMountClient is the client API for SeaweedMount service.

weed/pb/mq_pb/mq.pb.go

@@ -1,6 +1,6 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.28.0
+// 	protoc-gen-go v1.28.1
 // 	protoc        v3.21.4
 // source: mq.proto
 

weed/pb/mq_pb/mq_grpc.pb.go

@@ -1,4 +1,8 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.2.0
+// - protoc             v3.21.4
+// source: mq.proto
 
 package mq_pb
 
@@ -11,6 +15,7 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.32.0 or later.
 const _ = grpc.SupportPackageIsVersion7
 
 // SeaweedMessagingClient is the client API for SeaweedMessaging service.

weed/pb/remote_pb/remote.pb.go

@@ -1,6 +1,6 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.28.0
+// 	protoc-gen-go v1.28.1
 // 	protoc        v3.21.4
 // source: remote.proto
 

weed/pb/s3_pb/s3.pb.go

@@ -1,6 +1,6 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.28.0
+// 	protoc-gen-go v1.28.1
 // 	protoc        v3.21.4
 // source: s3.proto
 

weed/pb/s3_pb/s3_grpc.pb.go

@@ -1,4 +1,8 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.2.0
+// - protoc             v3.21.4
+// source: s3.proto
 
 package s3_pb
 
@@ -11,6 +15,7 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.32.0 or later.
 const _ = grpc.SupportPackageIsVersion7
 
 // SeaweedS3Client is the client API for SeaweedS3 service.

weed/pb/volume_server.proto

@@ -56,6 +56,8 @@ service VolumeServer {
 
     rpc ReadNeedleBlob (ReadNeedleBlobRequest) returns (ReadNeedleBlobResponse) {
     }
+    rpc ReadNeedleMeta (ReadNeedleMetaRequest) returns (ReadNeedleMetaResponse) {
+    }
     rpc WriteNeedleBlob (WriteNeedleBlobRequest) returns (WriteNeedleBlobResponse) {
     }
     rpc ReadAllNeedles (ReadAllNeedlesRequest) returns (stream ReadAllNeedlesResponse) {
@@ -285,6 +287,19 @@ message ReadNeedleBlobResponse {
     bytes needle_blob = 1;
 }
 
+message ReadNeedleMetaRequest {
+    uint32 volume_id = 1;
+    uint64 needle_id = 2;
+    int64 offset = 3; // actual offset
+    int32 size = 4;
+}
+message ReadNeedleMetaResponse {
+    uint32 cookie = 1;
+    uint64 last_modified = 2;
+    uint32 crc = 3;
+    string ttl = 4;
+}
+
 message WriteNeedleBlobRequest {
     uint32 volume_id = 1;
     uint64 needle_id = 2;

weed/pb/volume_server_pb/volume_server_grpc.pb.go

@@ -1,4 +1,8 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.2.0
+// - protoc             v3.21.4
+// source: volume_server.proto
 
 package volume_server_pb
 
@@ -11,6 +15,7 @@ import (
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.32.0 or later.
 const _ = grpc.SupportPackageIsVersion7
 
 // VolumeServerClient is the client API for VolumeServer service.
@@ -39,6 +44,7 @@ type VolumeServerClient interface {
 	ReadVolumeFileStatus(ctx context.Context, in *ReadVolumeFileStatusRequest, opts ...grpc.CallOption) (*ReadVolumeFileStatusResponse, error)
 	CopyFile(ctx context.Context, in *CopyFileRequest, opts ...grpc.CallOption) (VolumeServer_CopyFileClient, error)
 	ReadNeedleBlob(ctx context.Context, in *ReadNeedleBlobRequest, opts ...grpc.CallOption) (*ReadNeedleBlobResponse, error)
+	ReadNeedleMeta(ctx context.Context, in *ReadNeedleMetaRequest, opts ...grpc.CallOption) (*ReadNeedleMetaResponse, error)
 	WriteNeedleBlob(ctx context.Context, in *WriteNeedleBlobRequest, opts ...grpc.CallOption) (*WriteNeedleBlobResponse, error)
 	ReadAllNeedles(ctx context.Context, in *ReadAllNeedlesRequest, opts ...grpc.CallOption) (VolumeServer_ReadAllNeedlesClient, error)
 	VolumeTailSender(ctx context.Context, in *VolumeTailSenderRequest, opts ...grpc.CallOption) (VolumeServer_VolumeTailSenderClient, error)
@@ -346,6 +352,15 @@ func (c *volumeServerClient) ReadNeedleBlob(ctx context.Context, in *ReadNeedleB
 	return out, nil
 }
 
+func (c *volumeServerClient) ReadNeedleMeta(ctx context.Context, in *ReadNeedleMetaRequest, opts ...grpc.CallOption) (*ReadNeedleMetaResponse, error) {
+	out := new(ReadNeedleMetaResponse)
+	err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/ReadNeedleMeta", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 func (c *volumeServerClient) WriteNeedleBlob(ctx context.Context, in *WriteNeedleBlobRequest, opts ...grpc.CallOption) (*WriteNeedleBlobResponse, error) {
 	out := new(WriteNeedleBlobResponse)
 	err := c.cc.Invoke(ctx, "/volume_server_pb.VolumeServer/WriteNeedleBlob", in, out, opts...)
@@ -699,6 +714,7 @@ type VolumeServerServer interface {
 	ReadVolumeFileStatus(context.Context, *ReadVolumeFileStatusRequest) (*ReadVolumeFileStatusResponse, error)
 	CopyFile(*CopyFileRequest, VolumeServer_CopyFileServer) error
 	ReadNeedleBlob(context.Context, *ReadNeedleBlobRequest) (*ReadNeedleBlobResponse, error)
+	ReadNeedleMeta(context.Context, *ReadNeedleMetaRequest) (*ReadNeedleMetaResponse, error)
 	WriteNeedleBlob(context.Context, *WriteNeedleBlobRequest) (*WriteNeedleBlobResponse, error)
 	ReadAllNeedles(*ReadAllNeedlesRequest, VolumeServer_ReadAllNeedlesServer) error
 	VolumeTailSender(*VolumeTailSenderRequest, VolumeServer_VolumeTailSenderServer) error
@@ -791,6 +807,9 @@ func (UnimplementedVolumeServerServer) CopyFile(*CopyFileRequest, VolumeServer_C
 func (UnimplementedVolumeServerServer) ReadNeedleBlob(context.Context, *ReadNeedleBlobRequest) (*ReadNeedleBlobResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method ReadNeedleBlob not implemented")
 }
+func (UnimplementedVolumeServerServer) ReadNeedleMeta(context.Context, *ReadNeedleMetaRequest) (*ReadNeedleMetaResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method ReadNeedleMeta not implemented")
+}
 func (UnimplementedVolumeServerServer) WriteNeedleBlob(context.Context, *WriteNeedleBlobRequest) (*WriteNeedleBlobResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method WriteNeedleBlob not implemented")
 }
@@ -1239,6 +1258,24 @@ func _VolumeServer_ReadNeedleBlob_Handler(srv interface{}, ctx context.Context,
 	return interceptor(ctx, in, info, handler)
 }
 
+func _VolumeServer_ReadNeedleMeta_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(ReadNeedleMetaRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(VolumeServerServer).ReadNeedleMeta(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/volume_server_pb.VolumeServer/ReadNeedleMeta",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(VolumeServerServer).ReadNeedleMeta(ctx, req.(*ReadNeedleMetaRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 func _VolumeServer_WriteNeedleBlob_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(WriteNeedleBlobRequest)
 	if err := dec(in); err != nil {
@@ -1706,6 +1743,10 @@ var VolumeServer_ServiceDesc = grpc.ServiceDesc{
 			MethodName: "ReadNeedleBlob",
 			Handler:    _VolumeServer_ReadNeedleBlob_Handler,
 		},
+		{
+			MethodName: "ReadNeedleMeta",
+			Handler:    _VolumeServer_ReadNeedleMeta_Handler,
+		},
 		{
 			MethodName: "WriteNeedleBlob",
 			Handler:    _VolumeServer_WriteNeedleBlob_Handler,

weed/server/volume_grpc_read_write.go

@@ -23,6 +23,35 @@ func (vs *VolumeServer) ReadNeedleBlob(ctx context.Context, req *volume_server_p
 	return resp, nil
 }
 
+func (vs *VolumeServer) ReadNeedleMeta(ctx context.Context, req *volume_server_pb.ReadNeedleMetaRequest) (resp *volume_server_pb.ReadNeedleMetaResponse, err error) {
+	resp = &volume_server_pb.ReadNeedleMetaResponse{}
+	volumeId := needle.VolumeId(req.VolumeId)
+
+	n := &needle.Needle{
+		Id:    types.NeedleId(req.NeedleId),
+		Flags: 0x08,
+	}
+	size := req.Size
+	offset := req.Offset
+
+	hasVolume := vs.store.HasVolume(volumeId)
+	if !hasVolume {
+		return nil, fmt.Errorf("not found volume id %d and read needle metadata at ec shards is not supported", req.VolumeId)
+	}
+	err = vs.store.ReadVolumeNeedleMetaAt(volumeId, n, offset, size)
+	if err != nil {
+		return nil, err
+	}
+
+	resp.Cookie = uint32(n.Cookie)
+	resp.LastModified = n.LastModified
+	resp.Crc = n.Checksum.Value()
+	if n.HasTtl() {
+		resp.Ttl = n.Ttl.String()
+	}
+	return resp, nil
+}
+
 func (vs *VolumeServer) WriteNeedleBlob(ctx context.Context, req *volume_server_pb.WriteNeedleBlobRequest) (resp *volume_server_pb.WriteNeedleBlobResponse, err error) {
 	resp = &volume_server_pb.WriteNeedleBlobResponse{}
 	v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))

weed/storage/erasure_coding/ec_volume.go

@@ -210,10 +210,14 @@ func (ev *EcVolume) LocateEcShardNeedle(needleId types.NeedleId, version needle.
 		return types.Offset{}, 0, nil, fmt.Errorf("FindNeedleFromEcx: %v", err)
 	}
 
-	shard := ev.Shards[0]
+	intervals = ev.LocateEcShardNeedleInterval(version, offset.ToActualOffset(), types.Size(needle.GetActualSize(size, version)))
+	return
+}
 
+func (ev *EcVolume) LocateEcShardNeedleInterval(version needle.Version, offset int64, size types.Size) (intervals []Interval) {
+	shard := ev.Shards[0]
 	// calculate the locations in the ec shards
-	intervals = LocateData(ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, DataShardsCount*shard.ecdFileSize, offset.ToActualOffset(), types.Size(needle.GetActualSize(size, version)))
+	intervals = LocateData(ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, DataShardsCount*shard.ecdFileSize, offset, types.Size(needle.GetActualSize(size, version)))
 
 	return
 }

weed/storage/needle/needle_read_page.go

@@ -74,7 +74,6 @@ func (n *Needle) ReadNeedleMeta(r backend.BackendStorageFile, offset int64, size
 			return ErrorSizeMismatch
 		}
 	}
-
 	n.DataSize = util.BytesToUint32(bytes[NeedleHeaderSize : NeedleHeaderSize+DataSizeSize])
 
 	startOffset := offset + NeedleHeaderSize + DataSizeSize + int64(n.DataSize)
@@ -90,7 +89,6 @@ func (n *Needle) ReadNeedleMeta(r backend.BackendStorageFile, offset int64, size
 	if err != nil {
 		return err
 	}
-
 	var index int
 	index, err = n.readNeedleDataVersion2NonData(metaSlice)
 

weed/storage/store.go

@@ -384,6 +384,14 @@ func (s *Store) ReadVolumeNeedle(i needle.VolumeId, n *needle.Needle, readOption
 	}
 	return 0, fmt.Errorf("volume %d not found", i)
 }
+
+func (s *Store) ReadVolumeNeedleMetaAt(i needle.VolumeId, n *needle.Needle, offset int64, size int32) error {
+	if v := s.findVolume(i); v != nil {
+		return v.readNeedleMetaAt(n, offset, size)
+	}
+	return fmt.Errorf("volume %d not found", i)
+}
+
 func (s *Store) ReadVolumeNeedleDataInto(i needle.VolumeId, n *needle.Needle, readOption *ReadOption, writer io.Writer, offset int64, size int64) error {
 	if v := s.findVolume(i); v != nil {
 		return v.readNeedleDataInto(n, readOption, writer, offset, size)

weed/storage/volume_read.go

@@ -80,6 +80,24 @@ func (v *Volume) readNeedle(n *needle.Needle, readOption *ReadOption, onReadSize
 	return -1, ErrorNotFound
 }
 
+// read needle at a specific offset
+func (v *Volume) readNeedleMetaAt(n *needle.Needle, offset int64, size int32) (err error) {
+	v.dataFileAccessLock.RLock()
+	defer v.dataFileAccessLock.RUnlock()
+	// read deleted meta data
+	if size < 0 {
+		size = -size
+	}
+	err = n.ReadNeedleMeta(v.DataBackend, offset, Size(size), v.Version())
+	if err == needle.ErrorSizeMismatch && OffsetSize == 4 {
+		err = n.ReadNeedleMeta(v.DataBackend, offset+int64(MaxPossibleVolumeSize), Size(size), v.Version())
+	}
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 // read fills in Needle content by looking up n.Id from NeedleMapper
 func (v *Volume) readNeedleDataInto(n *needle.Needle, readOption *ReadOption, writer io.Writer, offset int64, size int64) (err error) {
 	v.dataFileAccessLock.RLock()

weed/storage/volume_read_test.go

@@ -0,0 +1,91 @@
+package storage
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestReadNeedMetaWithWritesAndUpdates(t *testing.T) {
+	dir := t.TempDir()
+
+	v, err := NewVolume(dir, dir, "", 1, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, 0)
+	if err != nil {
+		t.Fatalf("volume creation: %v", err)
+	}
+	type WriteInfo struct {
+		offset int64
+		size   int32
+	}
+	writeInfos := make([]WriteInfo, 30)
+	mockLastUpdateTime := uint64(1000000000000)
+	// initialize 20 needles then update first 10 needles
+	for i := 1; i <= 30; i++ {
+		n := newRandomNeedle(uint64(i % 20))
+		n.Flags = 0x08
+		n.LastModified = mockLastUpdateTime
+		mockLastUpdateTime += 2000
+		offset, _, _, err := v.writeNeedle2(n, true, false)
+		if err != nil {
+			t.Fatalf("write needle %d: %v", i, err)
+		}
+		writeInfos[i-1] = WriteInfo{offset: int64(offset), size: int32(n.Size)}
+	}
+	expectedLastUpdateTime := uint64(1000000000000)
+	for i := 0; i < 30; i++ {
+		testNeedle := new(needle.Needle)
+		testNeedle.Id = types.Uint64ToNeedleId(uint64(i + 1%20))
+		testNeedle.Flags = 0x08
+		v.readNeedleMetaAt(testNeedle, writeInfos[i].offset, writeInfos[i].size)
+		actualLastModifiedTime := testNeedle.LastModified
+		assert.Equal(t, expectedLastUpdateTime, actualLastModifiedTime, "The two words should be the same.")
+		expectedLastUpdateTime += 2000
+	}
+}
+
+func TestReadNeedMetaWithDeletesThenWrites(t *testing.T) {
+	dir := t.TempDir()
+
+	v, err := NewVolume(dir, dir, "", 1, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, 0)
+	if err != nil {
+		t.Fatalf("volume creation: %v", err)
+	}
+	type WriteInfo struct {
+		offset int64
+		size   int32
+	}
+	writeInfos := make([]WriteInfo, 10)
+	mockLastUpdateTime := uint64(1000000000000)
+	for i := 1; i <= 10; i++ {
+		n := newRandomNeedle(uint64(i % 5))
+		n.Flags = 0x08
+		n.LastModified = mockLastUpdateTime
+		mockLastUpdateTime += 2000
+		offset, _, _, err := v.writeNeedle2(n, true, false)
+		if err != nil {
+			t.Fatalf("write needle %d: %v", i, err)
+		}
+		if i < 5 {
+			size, err := v.deleteNeedle2(n)
+			if err != nil {
+				t.Fatalf("delete needle %d: %v", i, err)
+			}
+			writeInfos[i-1] = WriteInfo{offset: int64(offset), size: int32(size)}
+		} else {
+			writeInfos[i-1] = WriteInfo{offset: int64(offset), size: int32(n.Size)}
+		}
+	}
+
+	expectedLastUpdateTime := uint64(1000000000000)
+	for i := 0; i < 10; i++ {
+		testNeedle := new(needle.Needle)
+		testNeedle.Id = types.Uint64ToNeedleId(uint64(i + 1%5))
+		testNeedle.Flags = 0x08
+		v.readNeedleMetaAt(testNeedle, writeInfos[i].offset, writeInfos[i].size)
+		actualLastModifiedTime := testNeedle.LastModified
+		assert.Equal(t, expectedLastUpdateTime, actualLastModifiedTime, "The two words should be the same.")
+		expectedLastUpdateTime += 2000
+	}
+}