Skip to content

Commit

Permalink
*: Clean tree service client
Browse files Browse the repository at this point in the history 
Signed-off-by: Evgenii Baidakov <[email protected]>
  • Loading branch information
@smallhive
smallhive committed Jan 21, 2025
1 parent 07d30e9 commit 3d72e17
Show file tree
Hide file tree
Showing 4 changed files with 1 addition and 299 deletions.
2 changes: 1 addition & 1 deletion api/layer/compound.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ func (n *layer) GetObjectTaggingAndLock(ctx context.Context, objVersion *ObjectV
}
}

tags, _, err = n.treeService.GetObjectTaggingAndLock(ctx, objVersion.BktInfo, nodeVersion)
tags, err = n.treeService.GetObjectTagging(ctx, objVersion.BktInfo, nodeVersion)
if err != nil {
if errorsStd.Is(err, ErrNodeNotFound) {
return nil, nil, s3errors.GetAPIError(s3errors.ErrNoSuchKey)
Expand Down
89 changes: 0 additions & 89 deletions api/layer/tree_mock.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"context"
"fmt"
"slices"
"strings"

"github.com/nspcc-dev/neofs-s3-gw/api/data"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
Expand All @@ -21,12 +20,6 @@ type TreeServiceMock struct {
parts map[string]map[int]*data.PartInfo
}

func (t *TreeServiceMock) GetObjectTaggingAndLock(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) (map[string]string, *data.LockInfo, error) {
// TODO implement object tagging
lock, err := t.GetLock(ctx, bktInfo, objVersion.ID)
return nil, lock, err
}

func (t *TreeServiceMock) GetObjectTagging(_ context.Context, bktInfo *data.BucketInfo, nodeVersion *data.NodeVersion) (map[string]string, error) {
cnrTagsMap, ok := t.tags[bktInfo.CID.EncodeToString()]
if !ok {
Expand Down Expand Up @@ -157,31 +150,6 @@ func (t *TreeServiceMock) GetLatestVersion(_ context.Context, bktInfo *data.Buck
return nil, ErrNodeNotFound
}

func (t *TreeServiceMock) GetLatestVersionsByPrefix(_ context.Context, bktInfo *data.BucketInfo, prefix string) ([]*data.NodeVersion, error) {
cnrVersionsMap, ok := t.versions[bktInfo.CID.EncodeToString()]
if !ok {
return nil, ErrNodeNotFound
}

var result []*data.NodeVersion

for key, versions := range cnrVersionsMap {
if !strings.HasPrefix(key, prefix) {
continue
}

slices.SortFunc(versions, func(a, b *data.NodeVersion) int {
return cmp.Compare(a.ID, b.ID)
})

if len(versions) != 0 {
result = append(result, versions[len(versions)-1])
}
}

return result, nil
}

func (t *TreeServiceMock) GetUnversioned(_ context.Context, bktInfo *data.BucketInfo, objectName string) (*data.NodeVersion, error) {
cnrVersionsMap, ok := t.versions[bktInfo.CID.EncodeToString()]
if !ok {
Expand Down Expand Up @@ -242,40 +210,6 @@ func (t *TreeServiceMock) AddVersion(_ context.Context, bktInfo *data.BucketInfo
return newVersion.ID, nil
}

func (t *TreeServiceMock) RemoveVersion(_ context.Context, bktInfo *data.BucketInfo, nodeID uint64) error {
cnrVersionsMap, ok := t.versions[bktInfo.CID.EncodeToString()]
if !ok {
return ErrNodeNotFound
}

for key, versions := range cnrVersionsMap {
for i, node := range versions {
if node.ID == nodeID {
cnrVersionsMap[key] = slices.Delete(versions, i, i+1)
return nil
}
}
}

return ErrNodeNotFound
}

func (t *TreeServiceMock) GetAllVersionsByPrefix(_ context.Context, bktInfo *data.BucketInfo, prefix string) ([]*data.NodeVersion, error) {
cnrVersionsMap, ok := t.versions[bktInfo.CID.EncodeToString()]
if !ok {
return nil, nil
}

var result []*data.NodeVersion
for objName, versions := range cnrVersionsMap {
if strings.HasPrefix(objName, prefix) {
result = append(result, versions...)
}
}

return result, nil
}

func (t *TreeServiceMock) CreateMultipartUpload(_ context.Context, bktInfo *data.BucketInfo, info *data.MultipartInfo) (uint64, error) {
cnrMultipartsMap, ok := t.multiparts[bktInfo.CID.EncodeToString()]
if !ok {
Expand Down Expand Up @@ -453,26 +387,3 @@ LOOP:
delete(t.parts, uploadID)
return nil
}

func (t *TreeServiceMock) PutLock(_ context.Context, bktInfo *data.BucketInfo, nodeID uint64, lock *data.LockInfo) error {
cnrLockMap, ok := t.locks[bktInfo.CID.EncodeToString()]
if !ok {
t.locks[bktInfo.CID.EncodeToString()] = map[uint64]*data.LockInfo{
nodeID: lock,
}
return nil
}

cnrLockMap[nodeID] = lock

return nil
}

func (t *TreeServiceMock) GetLock(_ context.Context, bktInfo *data.BucketInfo, nodeID uint64) (*data.LockInfo, error) {
cnrLockMap, ok := t.locks[bktInfo.CID.EncodeToString()]
if !ok {
return nil, nil
}

return cnrLockMap[nodeID], nil
}
11 changes: 0 additions & 11 deletions api/layer/tree_service.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,14 +54,8 @@ type TreeService interface {

GetVersions(ctx context.Context, bktInfo *data.BucketInfo, objectName string) ([]*data.NodeVersion, error)
GetLatestVersion(ctx context.Context, bktInfo *data.BucketInfo, objectName string) (*data.NodeVersion, error)
GetLatestVersionsByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string) ([]*data.NodeVersion, error)
GetAllVersionsByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string) ([]*data.NodeVersion, error)
GetUnversioned(ctx context.Context, bktInfo *data.BucketInfo, objectName string) (*data.NodeVersion, error)
AddVersion(ctx context.Context, bktInfo *data.BucketInfo, newVersion *data.NodeVersion) (uint64, error)
RemoveVersion(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64) error

PutLock(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64, lock *data.LockInfo) error
GetLock(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64) (*data.LockInfo, error)

CreateMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, info *data.MultipartInfo) (uint64, error)
DeleteMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64) error
Expand All @@ -79,11 +73,6 @@ type TreeService interface {
// - [ErrPartListIsEmpty] if there is no parts in the upload id.
GetPartByNumber(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64, number int) (*data.PartInfo, error)
GetPartsAfter(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64, partID int) ([]*data.PartInfo, error)

// Compound methods for optimizations

// GetObjectTaggingAndLock unifies GetObjectTagging and GetLock methods in single tree service invocation.
GetObjectTaggingAndLock(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) (map[string]string, *data.LockInfo, error)
}

var (
Expand Down
198 changes: 0 additions & 198 deletions internal/neofs/tree.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,13 +64,6 @@ const (
homoHashKV = "HomoHash"
elementsKV = "Elements"

// keys for lock.
isLockKV = "IsLock"
legalHoldOIDKV = "LegalHoldOID"
retentionOIDKV = "RetentionOID"
untilDateKV = "UntilDate"
isComplianceKV = "IsCompliance"

// keys for delete marker nodes.
isDeleteMarkerKV = "IsDeleteMarker"
ownerKV = "Owner"
Expand Down Expand Up @@ -600,10 +593,6 @@ func pathFromName(objectName string) []string {
return strings.Split(objectName, separator)
}

func (c *TreeClient) GetLatestVersionsByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string) ([]*data.NodeVersion, error) {
return c.getVersionsByPrefix(ctx, bktInfo, prefix, true)
}

func (c *TreeClient) determinePrefixNode(ctx context.Context, bktInfo *data.BucketInfo, treeID, prefix string) (uint64, string, error) {
var rootID uint64
path := strings.Split(prefix, separator)
Expand Down Expand Up @@ -722,117 +711,6 @@ func isIntermediate(node NodeResponse) bool {
return node.GetMeta()[0].GetKey() == fileNameKV
}

func (c *TreeClient) getSubTreeVersions(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64, parentFilePath string, latestOnly bool) ([]*data.NodeVersion, error) {
subTree, err := c.getSubTree(ctx, bktInfo, versionTree, nodeID, maxGetSubTreeDepth)
if err != nil {
return nil, err
}

var parentPrefix string
if parentFilePath != "" { // The root of subTree can also have a parent
parentPrefix = strings.TrimSuffix(parentFilePath, separator) + separator // To avoid 'foo//bar'
}

var emptyOID oid.ID
var filepath string
namesMap := make(map[uint64]string, len(subTree))
versions := make(map[string][]*data.NodeVersion, len(subTree))

for i, node := range subTree {
treeNode, fileName, err := parseTreeNode(node)
if err != nil {
continue
}

if i != 0 {
if filepath, err = formFilePath(node, fileName, namesMap); err != nil {
return nil, fmt.Errorf("invalid node order: %w", err)
}
} else {
filepath = parentPrefix + fileName
namesMap[treeNode.ID] = filepath
}

if treeNode.ObjID.Equals(emptyOID) { // The node can be intermediate but we still want to update namesMap
continue
}

key := formLatestNodeKey(node.GetParentId(), fileName)
versionNodes, ok := versions[key]
if !ok {
versionNodes = []*data.NodeVersion{newNodeVersionFromTreeNode(filepath, treeNode)}
} else if !latestOnly {
versionNodes = append(versionNodes, newNodeVersionFromTreeNode(filepath, treeNode))
} else if versionNodes[0].Timestamp <= treeNode.TimeStamp {
versionNodes[0] = newNodeVersionFromTreeNode(filepath, treeNode)
}

versions[key] = versionNodes
}

result := make([]*data.NodeVersion, 0, len(versions)) // consider use len(subTree)
for _, version := range versions {
if latestOnly && version[0].IsDeleteMarker() {
continue
}
result = append(result, version...)
}

return result, nil
}

func formFilePath(node *tree.GetSubTreeResponse_Body, fileName string, namesMap map[uint64]string) (string, error) {
parentPath, ok := namesMap[node.GetParentId()]
if !ok {
return "", fmt.Errorf("couldn't get parent path")
}

filepath := parentPath + separator + fileName
namesMap[node.GetNodeId()] = filepath

return filepath, nil
}

func parseTreeNode(node *tree.GetSubTreeResponse_Body) (*TreeNode, string, error) {
treeNode, err := newTreeNode(node)
if err != nil { // invalid OID attribute
return nil, "", err
}

fileName, ok := treeNode.FileName()
if !ok {
return nil, "", fmt.Errorf("doesn't contain FileName")
}

return treeNode, fileName, nil
}

func formLatestNodeKey(parentID uint64, fileName string) string {
return strconv.FormatUint(parentID, 10) + "." + fileName
}

func (c *TreeClient) GetAllVersionsByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string) ([]*data.NodeVersion, error) {
return c.getVersionsByPrefix(ctx, bktInfo, prefix, false)
}

func (c *TreeClient) getVersionsByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]*data.NodeVersion, error) {
prefixNodes, headPrefix, err := c.getSubTreeByPrefix(ctx, bktInfo, versionTree, prefix, latestOnly)
if err != nil {
return nil, err
}

var result []*data.NodeVersion
for _, node := range prefixNodes {
versions, err := c.getSubTreeVersions(ctx, bktInfo, node.GetNodeId(), headPrefix, latestOnly)
if err != nil {
return nil, err
}
result = append(result, versions...)
}

return result, nil
}

func (c *TreeClient) GetUnversioned(ctx context.Context, bktInfo *data.BucketInfo, filepath string) (*data.NodeVersion, error) {
return c.getUnversioned(ctx, bktInfo, versionTree, filepath)
}
Expand All @@ -858,10 +736,6 @@ func (c *TreeClient) AddVersion(ctx context.Context, bktInfo *data.BucketInfo, v
return c.addVersion(ctx, bktInfo, versionTree, version)
}

func (c *TreeClient) RemoveVersion(ctx context.Context, bktInfo *data.BucketInfo, id uint64) error {
return c.removeNode(ctx, bktInfo, versionTree, id)
}

func (c *TreeClient) CreateMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, info *data.MultipartInfo) (uint64, error) {
path := pathFromName(info.Key)
meta := metaFromMultipart(info, path[len(path)-1])
Expand Down Expand Up @@ -1087,78 +961,6 @@ func (c *TreeClient) DeleteMultipartUpload(ctx context.Context, bktInfo *data.Bu
return c.removeNode(ctx, bktInfo, systemTree, multipartNodeID)
}

func (c *TreeClient) PutLock(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64, lock *data.LockInfo) error {
meta := map[string]string{isLockKV: "true"}

if lock.IsLegalHoldSet() {
meta[legalHoldOIDKV] = lock.LegalHold().EncodeToString()
}
if lock.IsRetentionSet() {
meta[retentionOIDKV] = lock.Retention().EncodeToString()
meta[untilDateKV] = lock.UntilDate()
if lock.IsCompliance() {
meta[isComplianceKV] = "true"
}
}

if lock.ID() == 0 {
_, err := c.addNode(ctx, bktInfo, versionTree, nodeID, meta)
return err
}

return c.moveNode(ctx, bktInfo, versionTree, lock.ID(), nodeID, meta)
}

func (c *TreeClient) GetLock(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64) (*data.LockInfo, error) {
lockNode, err := c.getTreeNode(ctx, bktInfo, nodeID, isLockKV)
if err != nil {
return nil, err
}

return getLock(lockNode)
}

func getLock(lockNode *TreeNode) (*data.LockInfo, error) {
if lockNode == nil {
return &data.LockInfo{}, nil
}
lockInfo := data.NewLockInfo(lockNode.ID)

if legalHold, ok := lockNode.Get(legalHoldOIDKV); ok {
var legalHoldOID oid.ID
if err := legalHoldOID.DecodeString(legalHold); err != nil {
return nil, fmt.Errorf("invalid legal hold object id: %w", err)
}
lockInfo.SetLegalHold(legalHoldOID)
}

if retention, ok := lockNode.Get(retentionOIDKV); ok {
var retentionOID oid.ID
if err := retentionOID.DecodeString(retention); err != nil {
return nil, fmt.Errorf("invalid retention object id: %w", err)
}
_, isCompliance := lockNode.Get(isComplianceKV)
untilDate, _ := lockNode.Get(untilDateKV)
lockInfo.SetRetention(retentionOID, untilDate, isCompliance)
}

return lockInfo, nil
}

func (c *TreeClient) GetObjectTaggingAndLock(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) (map[string]string, *data.LockInfo, error) {
nodes, err := c.getTreeNodes(ctx, bktInfo, objVersion.ID, isTagKV, isLockKV)
if err != nil {
return nil, nil, err
}

lockInfo, err := getLock(nodes[isLockKV])
if err != nil {
return nil, nil, err
}

return getObjectTagging(nodes[isTagKV]), lockInfo, nil
}

func (c *TreeClient) Close() error {
if c.conn != nil {
return c.conn.Close()
Expand Down

0 comments on commit 3d72e17

Please sign in to comment.