Skip to content

Commit

Permalink
Move bucket parsing to goofys from goofys_fuse
Browse files Browse the repository at this point in the history
  • Loading branch information
@vitalif
vitalif committed Jun 9, 2023
1 parent 1c5f54e commit 485e04b
Show file tree
Hide file tree
Showing 5 changed files with 99 additions and 94 deletions.
11 changes: 7 additions & 4 deletions internal/cluster_fs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,11 @@ type ClusterFs struct {
stat Stat
}

func NewClusterGoofys(ctx context.Context, bucket string, flags *FlagStorage, conns *ConnPool) *Goofys {
fs := NewGoofys(ctx, bucket, flags)
func NewClusterGoofys(ctx context.Context, bucket string, flags *FlagStorage, conns *ConnPool) (*Goofys, error) {
fs, err := NewGoofys(ctx, bucket, flags)
if err != nil {
return nil, err
}

// choose node with min id as root owner
var rootOwner NodeId
Expand All @@ -62,8 +65,8 @@ func NewClusterGoofys(ctx context.Context, bucket string, flags *FlagStorage, co
fs.inodes[fuseops.RootInodeID].readyOwner = true
fs.nextHandleID = N_HANDLES * fuseops.HandleID(conns.id)
fs.nextInodeID = N_INODES * fuseops.InodeID(conns.id)
return fs

return fs, nil
}

// REQUIRED_LOCK(parent.KeepOwnerLock)
Expand Down
5 changes: 4 additions & 1 deletion internal/cluster_fs_fuse.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -931,7 +931,10 @@ func MountCluster(
rec := &Recovery{
Flags: flags,
}
goofys := NewClusterGoofys(context.Background(), bucketName, flags, conns)
goofys, err := NewClusterGoofys(context.Background(), bucketName, flags, conns)
if err != nil {
return nil, nil, err
}
fs := &ClusterFs{
Flags: flags,
Conns: conns,
Expand Down
86 changes: 78 additions & 8 deletions internal/goofys.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,12 +193,84 @@ func ParseBucketSpec(bucket string) (spec BucketSpec, err error) {
return
}

func NewGoofys(ctx context.Context, bucket string, flags *FlagStorage) *Goofys {
return newGoofys(ctx, bucket, flags, NewBackend)
func NewGoofys(ctx context.Context, bucketName string, flags *FlagStorage) (*Goofys, error) {
if flags.DebugS3 {
SetCloudLogLevel(logrus.DebugLevel)
}
if flags.Backend == nil {
if spec, err := ParseBucketSpec(bucketName); err == nil {
switch spec.Scheme {
case "adl":
auth, err := AzureAuthorizerConfig{
Log: GetLogger("adlv1"),
}.Authorizer()
if err != nil {
err = fmt.Errorf("couldn't load azure credentials: %v",
err)
return nil, err
}
flags.Backend = &ADLv1Config{
Endpoint: spec.Bucket,
Authorizer: auth,
}
// adlv1 doesn't really have bucket
// names, but we will rebuild the
// prefix
bucketName = ""
if spec.Prefix != "" {
bucketName = ":" + spec.Prefix
}
case "wasb":
config, err := AzureBlobConfig(flags.Endpoint, spec.Bucket, "blob")
if err != nil {
return nil, err
}
flags.Backend = &config
if config.Container != "" {
bucketName = config.Container
} else {
bucketName = spec.Bucket
}
if config.Prefix != "" {
spec.Prefix = config.Prefix
}
if spec.Prefix != "" {
bucketName += ":" + spec.Prefix
}
case "abfs":
config, err := AzureBlobConfig(flags.Endpoint, spec.Bucket, "dfs")
if err != nil {
return nil, err
}
flags.Backend = &config
if config.Container != "" {
bucketName = config.Container
} else {
bucketName = spec.Bucket
}
if config.Prefix != "" {
spec.Prefix = config.Prefix
}
if spec.Prefix != "" {
bucketName += ":" + spec.Prefix
}

flags.Backend = &ADLv2Config{
Endpoint: config.Endpoint,
Authorizer: &config,
}
bucketName = spec.Bucket
if spec.Prefix != "" {
bucketName += ":" + spec.Prefix
}
}
}
}
return newGoofys(ctx, bucketName, flags, NewBackend)
}

func newGoofys(ctx context.Context, bucket string, flags *FlagStorage,
newBackend func(string, *FlagStorage) (StorageBackend, error)) *Goofys {
newBackend func(string, *FlagStorage) (StorageBackend, error)) (*Goofys, error) {
// Set up the basic struct.
fs := &Goofys{
bucket: bucket,
Expand Down Expand Up @@ -232,15 +304,13 @@ func newGoofys(ctx context.Context, bucket string, flags *FlagStorage,

cloud, err := newBackend(bucket, flags)
if err != nil {
log.Errorf("Unable to setup backend: %v", err)
return nil
return nil, fmt.Errorf("Unable to setup backend: %v", err)
}

randomObjectName := prefix + (RandStringBytesMaskImprSrc(32))
err = cloud.Init(randomObjectName)
if err != nil {
log.Errorf("Unable to access '%v': %v", bucket, err)
return nil
return nil, fmt.Errorf("Unable to access '%v': %v", bucket, err)
}
cloud.MultipartExpire(&MultipartExpireInput{})

Expand Down Expand Up @@ -292,7 +362,7 @@ func newGoofys(ctx context.Context, bucket string, flags *FlagStorage,
go fs.FDCloser()
}

return fs
return fs, nil
}

// from https://stackoverflow.com/questions/22892120/how-to-generate-a-random-string-of-a-fixed-length-in-golang
Expand Down
79 changes: 4 additions & 75 deletions internal/goofys_fuse.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -974,9 +974,6 @@ func MountFuse(
bucketName string,
flags *FlagStorage) (fs *Goofys, mfs MountedFS, err error) {

if flags.DebugS3 {
SetCloudLogLevel(logrus.DebugLevel)
}
// Mount the file system.
mountCfg := &fuse.MountConfig{
FSName: bucketName,
Expand All @@ -997,79 +994,11 @@ func MountFuse(
log.Level = logrus.DebugLevel
}

if flags.Backend == nil {
if spec, err := ParseBucketSpec(bucketName); err == nil {
switch spec.Scheme {
case "adl":
auth, err := AzureAuthorizerConfig{
Log: GetLogger("adlv1"),
}.Authorizer()
if err != nil {
err = fmt.Errorf("couldn't load azure credentials: %v",
err)
return nil, nil, err
}
flags.Backend = &ADLv1Config{
Endpoint: spec.Bucket,
Authorizer: auth,
}
// adlv1 doesn't really have bucket
// names, but we will rebuild the
// prefix
bucketName = ""
if spec.Prefix != "" {
bucketName = ":" + spec.Prefix
}
case "wasb":
config, err := AzureBlobConfig(flags.Endpoint, spec.Bucket, "blob")
if err != nil {
return nil, nil, err
}
flags.Backend = &config
if config.Container != "" {
bucketName = config.Container
} else {
bucketName = spec.Bucket
}
if config.Prefix != "" {
spec.Prefix = config.Prefix
}
if spec.Prefix != "" {
bucketName += ":" + spec.Prefix
}
case "abfs":
config, err := AzureBlobConfig(flags.Endpoint, spec.Bucket, "dfs")
if err != nil {
return nil, nil, err
}
flags.Backend = &config
if config.Container != "" {
bucketName = config.Container
} else {
bucketName = spec.Bucket
}
if config.Prefix != "" {
spec.Prefix = config.Prefix
}
if spec.Prefix != "" {
bucketName += ":" + spec.Prefix
}

flags.Backend = &ADLv2Config{
Endpoint: config.Endpoint,
Authorizer: &config,
}
bucketName = spec.Bucket
if spec.Prefix != "" {
bucketName += ":" + spec.Prefix
}
}
}
}

fs = NewGoofys(ctx, bucketName, flags)
fs, err = NewGoofys(ctx, bucketName, flags)
if fs == nil {
err = fmt.Errorf("Mount: initialization failed")
if err == nil {
err = fmt.Errorf("Mount: initialization failed")
}
return
}
fsint := NewGoofysFuse(fs)
Expand Down
12 changes: 6 additions & 6 deletions internal/goofys_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -724,7 +724,7 @@ func (s *GoofysTest) SetUpTest(t *C) {
s.setupDefaultEnv(t, "")

if hasEnv("EVENTUAL_CONSISTENCY") {
fs := newGoofys(context.Background(), bucket, flags,
fs, _ := newGoofys(context.Background(), bucket, flags,
func(bucket string, flags *FlagStorage) (StorageBackend, error) {
cloud, err := NewBackend(bucket, flags)
if err != nil {
Expand All @@ -734,7 +734,7 @@ func (s *GoofysTest) SetUpTest(t *C) {
})
s.fs = NewGoofysFuse(fs)
} else {
fs := NewGoofys(context.Background(), bucket, flags)
fs, _ := NewGoofys(context.Background(), bucket, flags)
s.fs = NewGoofysFuse(fs)
}
t.Assert(s.fs, NotNil)
Expand Down Expand Up @@ -2266,19 +2266,19 @@ func (s *GoofysTest) TestPutMimeType(t *C) {
}

func (s *GoofysTest) TestBucketPrefixSlash(t *C) {
fs := NewGoofys(context.Background(), s.fs.bucket+":dir2", s.fs.flags)
fs, _ := NewGoofys(context.Background(), s.fs.bucket+":dir2", s.fs.flags)
s.fs = NewGoofysFuse(fs)
t.Assert(s.getRoot(t).dir.mountPrefix, Equals, "dir2/")

fs = NewGoofys(context.Background(), s.fs.bucket+":dir2///", s.fs.flags)
fs, _ = NewGoofys(context.Background(), s.fs.bucket+":dir2///", s.fs.flags)
s.fs = NewGoofysFuse(fs)
t.Assert(s.getRoot(t).dir.mountPrefix, Equals, "dir2/")
}

func (s *GoofysTest) TestFuseWithPrefix(t *C) {
mountPoint := s.tmp + "/mnt" + s.fs.bucket

fs := NewGoofys(context.Background(), s.fs.bucket+":testprefix", s.fs.flags)
fs, _ := NewGoofys(context.Background(), s.fs.bucket+":testprefix", s.fs.flags)
s.fs = NewGoofysFuse(fs)

s.runFuseTest(t, mountPoint, true, s.tmp+"/fuse-test.sh", mountPoint)
Expand Down Expand Up @@ -2348,7 +2348,7 @@ func (s *GoofysTest) anonymous(t *C) {
t.Skip("cloud does not support canned ACL")
}

fs := NewGoofys(context.Background(), bucket, s.fs.flags)
fs, _ := NewGoofys(context.Background(), bucket, s.fs.flags)
s.fs = NewGoofysFuse(fs)
t.Assert(s.fs, NotNil)

Expand Down

0 comments on commit 485e04b

Please sign in to comment.