common-blob-go

Go library to work with AWS(amazon web services) S3 and GCP(google cloud platform) cloud storage

Usage

Install

go get -u github.com/AccelByte/common-blob-go

Importing

commonblobgo "github.com/AccelByte/common-blob-go"

To create a new storage client, use this function:

storage, err := storage, err := NewCloudStorage(
    ctx,
    isTesting,
    bucketProvider,
    bucketName,
    awsS3Endpoint,
    awsS3Region,
    awsS3AccessKeyID,
    awsS3SecretAccessKey,
    gcpCredentialsJSON,
    gcpStorageEmulatorHost,
)

NewCloudStorage requires such parameters :

• ctx context.Context : a context that could be cancelled to force-stop the initialization

• isTesting bool : a flag to switch between external and in-docker-compose dependencies. Used from tests

• bucketProvider string : provider type. Could be aws or gcp

• bucketName string : the name of a bucket

• awsS3Endpoint string : S3 endpoint. Used only from tests(required if bucketProvider==aws and isTesting == true)

• awsS3Region string : S3 region(required if bucketProvider==aws)

• awsS3AccessKeyID string : S3 Access key(required if bucketProvider==aws)

• awsS3SecretAccessKey string : S3 secret key(required if bucketProvider==aws)

• gcpCredentialsJSON string : GCP JSON credentials(optional if bucketProvider==gcp).

  • If empty - the library will attempt to self-configure from k8s GCP API. The Client(service account) should have the role "Service Account Token Creator"

    Click to expand

    

• gcpStorageEmulatorHost string : GCP storage host. Used only from tests(required if bucketProvider==gcp and isTesting == true)

To enable cloud storage additional features:

storage, err := storage, err := NewCloudStorageWithOption(
    ctx,
    isTesting,
    bucketProvider,
    bucketName,
    opts,
)

Cloud-specific parameter such as awsRegion, gcpStorageEmulatorHost, etc. has been moved to opts CloudStorageOption.

Supported additional cloud storage feature:

• opts.AWSEnableS3Accelerate (default: false) : a boolean that indicate S3 bucket use accelerate endpoint. Not available in testing using localstack or using path-style S3 endpoint. Note: make sure to enable transfer accelerate in S3 bucket, please refer to this documentation.

Available methods :

type CloudStorage interface {
	List(ctx context.Context, prefix string) *ListIterator // iterate over all objects in the folder
	ListWithOptions(ctx context.Context, options *ListOptions) *ListIterator // iterate over objects in the folder based on ListOptions criteria 
	Get(ctx context.Context, key string) ([]byte, error) // get the object by a name
	GetReader(ctx context.Context, key string) (io.ReadCloser, error) // get reader to operate with io.ReadCloser
	Delete(ctx context.Context, key string) error // delete the object by a name
	CreateBucket(ctx context.Context, bucketPrefix string, expirationTimeDays int64) error // create a bucket. Used only from tests
	Close() // close connection
	GetSignedURL(ctx context.Context, key string, expiry time.Duration) (string, error) // create signed URL
	Write(ctx context.Context, key string, body []byte, contentType *string) error // write the object a file-name
	GetWriter(ctx context.Context, key string) (io.WriteCloser, error) // get writer to operate with io.WriteCloser
	Attributes(ctx context.Context, key string) (*Attributes, error) // get object attributes
	Exists(ctx context.Context, key string) (bool, error) // check object existence
}

Examples:

List(ctx context.Context, prefix string) *ListIterator

    list := storage.List(ctx, bucketPrefix)

    for {
        item, err := list.Next(ctx)
        if err == io.EOF {
            break // no more object
        }

        // ...

        if item.Key == fileName {
            fileFound = true
        }
    }

ListWithOptions(ctx context.Context, options *ListOptions) *ListIterator

    list := storage.CloudStorage.ListWithOptions(parentCtx, &commonblobgo.ListOptions{
        Prefix:    bucketPrefix,

        // An empty delimiter means that the bucket is treated as a single flat namespace.
        //
        // A non-empty delimiter means that any result with the delimiter in its key after Prefix is stripped will be returned with ListObject.IsDir = true,
        // ListObject.Key truncated after the delimiter. These results represent "directories"
        Delimiter: "/", 
    })

    var directories []string
    for {
        item, err := list.Next(ctx)
        if err == io.EOF {
            break // no more object
        }

        // ...

        if item.IsDir {
            directories = append(directories, item.Key)
        }
    }

Get(ctx context.Context, key string) ([]byte, error)

    storedBody, err := storage.Get(ctx, fileName)
    if err != nil { 
        return nil, err
    }   

    fmt.Println(string(storedBody))

GetReader(ctx context.Context, key string) (io.ReadCloser, error)

    reader, err := storage.GetReader(ctx, fileName)
    if err != nil { 
        return nil, err
    }
    defer reader.Close() // Important to prevent memory leaks

    storedBody, err := ioutil.ReadAll(reader)
    fmt.Println(string(storedBody))

GetRangeReader(ctx context.Context, key string, offset int64, length int64) (io.ReadCloser, error)

    reader, err := storage.GetRangeReader(ctx, fileName, offset, length)
    if err != nil { 
        return nil, err
    }
    defer reader.Close() // Important to prevent memory leaks

    storedBody, err := ioutil.ReadAll(reader)
    fmt.Println(string(storedBody))

Delete(ctx context.Context, key string) error

    err = storage.Delete(ctx, fileName)
    if err != nil { 
        return nil, err
    }   

CreateBucket(ctx context.Context, bucketPrefix string, expirationTimeDays int64) error

    err = storage.CreateBucket(ctx, bucketPrefix, 1)
    if err != nil { 
        return nil, err
    }   

Close()

    storage, err := storage, err := NewCloudStorage(
        ctx,
        isTesting,
        bucketProvider,
        bucketName,
        awsS3Endpoint,
        awsS3Region,
        awsS3AccessKeyID,
        awsS3SecretAccessKey,
        gcpCredentialsJSON,
        gcpStorageEmulatorHost,
    )

    defer storage.Close()

GetSignedURL(ctx context.Context, key string, expiry time.Duration) (string, error)

    url, err := storage.GetSignedURL(ctx, fileName, time.Hour)
    if err != nil { 
        return nil, err
    }   

    fmt.Println(url)

Write(ctx context.Context, key string, body []byte, contentType *string) error

    err := storage.Write(ctx, fileName, bodyBytes, nil)
    if err != nil { 
        return nil, err
    }   

GetWriter(ctx context.Context, key string) (io.WriteCloser, error)

	body := []byte(`{"key": "value", "key2": "value2"}`)

	writer, err := storage.GetWriter(ctx, fileName)
    if err != nil { 
        return nil, err
    }   

	_, err = writer.Write(body[:10])
    if err != nil { 
        return nil, err
    }   

	_, err = writer.Write(body[10:20])
    if err != nil { 
        return nil, err
    }   

	_, err = writer.Write(body[20:])
    if err != nil { 
        return nil, err
    }   

	err = writer.Close()
    if err != nil { 
        return nil, err
    }   

Attributes(ctx context.Context, key string) (*Attributes, error)

    attrs, err := storage.Attributes(ctx, fileName)
    if err != nil { 
        return nil, err
    }   
    
    fmt.Println(attrs.Size)

Exists(ctx context.Context, key string) (bool, error)

    isExists, err := storage.Exists(ctx, fileName)
    if err != nil { 
        return nil, err
    }   

    if isExists {
        fmt.Println("found")
    }

License

Copyright © 2020, AccelByte Inc. Released under the Apache License, Version 2.0