feat(volumesnapshotlocation): support for MinIO object store (#16)

* Updating plugin name from "mayadata.io/cstor-blockstore" to "openebs.io/cstor-blockstore"

Signed-off-by: mayank <[email protected]>

Gopkg.toml

@@ -26,7 +26,7 @@
 
 [[constraint]]
   name = "github.com/heptio/velero"
-  version = "v1.0.0-rc.1"
+  version = "v1.0.0"
 
 [[constraint]]
   name = "github.com/aws/aws-sdk-go"

README.md

@@ -22,19 +22,20 @@ Run the following command to install OpenEBS velero-plugin
 This command will add an init container to Velero deployment to install the OpenEBS velero-plugin.
 
 ## Taking backup of CStor volume data through the Velero
-To take a backup of CStor volume through Velero, configure `VolumeSnapshotLocation` with provider `cstor-blockstore`. Sample yaml file for volumesnapshotlocation can be found at `example/06-volumesnapshotlocation.yaml`.
+To take a backup of CStor volume through Velero, configure `VolumeSnapshotLocation` with provider `openebs.io/cstor-blockstore`. Sample yaml file for volumesnapshotlocation can be found at `example/06-volumesnapshotlocation.yaml`.
 
 ```
 spec:
-  provider: cstor-blockstore
+  provider: openebs.io/cstor-blockstore
   config:
     bucket: <YOUR_BUCKET>
     prefix: <PREFIX_FOR_BACKUP_NAME>
     provider: <GCP_OR_AWS>
     region: <AWS_REGION>
 ```
 
-You can configure a backup storage location(`BackupStorageLocation`) in similar way. Currently, AWS and GCP are supported.
+You can configure a backup storage location(`BackupStorageLocation`) in similar way.
+Currently supported volumesnapshotlocations for velero-plugin are AWS, GCP and MinIO.
 
 
 ## Managing Backups
@@ -79,6 +80,12 @@ defaultbackup-20190513113453   defaultbackup   Completed   0          0
 ```
 Once the restore is completed you should see the restore marked as `Completed`.
 
+*Note: After restore is completed, you need to set targetip for the volume in pool pod.*
+*Steps to update `targetip` is as follow:*
+```
+1. kubectl exec -it <POOL_POD> -c cstor-pool -n openebs -- bash
+2. zfs set io.openebs:targetip=<PVC SERVICE IP> <POOL_NAME/VOLUME_NAME>
+```
 
 ### Creating a scheduled backup (or incremental backup for CStor volume)
 OpenEBS velero-plugin provides incremental backup support for CStor persistent volumes.
@@ -120,7 +127,7 @@ velero restore create --from-backup sched-20190513104034 --restore-volumes=true
 
 ## Compatibility matrix
 
-|     Image                |    Codebase     |  Velero v0.10.0  | Velero v0.11.0 | Velero v1.0.0-rc.1 |
+|     Image                |    Codebase     |  Velero v0.10.0  | Velero v0.11.0 | Velero v1.0.0 |
 |   -------------------    |  ---------------|   ---------      |  --------------- |  -----------------  |
 | velero-plugin:0.9.0-RC2    |     v0.9.x    |         ✓        |                  |                     |
 | velero-plugin:0.9.0-RC3    |     v0.9.x    |                  |         ✓        |                     |

example/06-volumesnapshotlocation.yaml

@@ -19,7 +19,7 @@ metadata:
   name: <LOCATION_NAME>
   namespace: velero
 spec:
-  provider: cstor-blockstore
+  provider: openebs.io/cstor-blockstore
   config:
     bucket: <YOUR_BUCKET>
     prefix: <PREFIX_FOR_BACKUP_NAME >

pkg/clouduploader/conn.go

@@ -56,8 +56,14 @@ const (
 	// GCP gcp cloud provider
 	GCP = "gcp"
 
-	// DefaultAWSRegion default aws region
-	DefaultAWSRegion = "us-east-2"
+	// AWSUrl aws s3 url
+	AWSUrl = "s3Url"
+
+	// AWSForcePath aws URL base path instead of subdomains
+	AWSForcePath = "s3ForcePathStyle"
+
+	// AWSSsl if ssl needs to be enabled
+	AWSSsl = "DisableSSL"
 )
 
 // Conn defines resource used for cloud related operation
@@ -77,9 +83,6 @@ type Conn struct {
 	// bucketname is storage-bucket name
 	bucketname string
 
-	// region is cloud specific region
-	region string
-
 	// prefix is used for file name
 	prefix string
 
@@ -91,19 +94,19 @@ type Conn struct {
 }
 
 // setupBucket creates a connection to a particular cloud provider's blob storage.
-func setupBucket(ctx context.Context, provider, bucket, region string) (*blob.Bucket, error) {
+func (c *Conn) setupBucket(ctx context.Context, provider, bucket string, config map[string]string) (*blob.Bucket, error) {
 	switch provider {
 	case AWS:
-		return setupAWS(ctx, bucket, region)
+		return c.setupAWS(ctx, bucket, config)
 	case GCP:
-		return setupGCP(ctx, bucket)
+		return c.setupGCP(ctx, bucket, config)
 	default:
 		return nil, errors.New("Provider is not supported")
 	}
 }
 
 // setupGCP creates a connection to GCP's blob storage
-func setupGCP(ctx context.Context, bucket string) (*blob.Bucket, error) {
+func (c *Conn) setupGCP(ctx context.Context, bucket string, config map[string]string) (*blob.Bucket, error) {
 	/* TBD: use cred file using env variable */
 	creds, err := gcp.DefaultCredentials(ctx)
 	if err != nil {
@@ -118,27 +121,40 @@ func setupGCP(ctx context.Context, bucket string) (*blob.Bucket, error) {
 }
 
 // setupAWS creates a connection to AWS's blob storage
-func setupAWS(ctx context.Context, bucketName, region string) (*blob.Bucket, error) {
-	var awsRegion *string
+func (c *Conn) setupAWS(ctx context.Context, bucketName string, config map[string]string) (*blob.Bucket, error) {
 	var awscred string
 
-	if region == "" {
-		awsRegion = aws.String(DefaultAWSRegion)
-	} else {
-		awsRegion = aws.String(region)
+	region, err := config[REGION]
+	if !err {
+		return nil, errors.New("No region provided for AWS")
 	}
 
 	if awscred = os.Getenv(AWSCredentialsFile); len(awscred) == 0 {
 		return nil, errors.New("error fetching aws credentials")
 	}
 
 	credentials := credentials.NewSharedCredentials(awscred, DefaultProfile)
-	d := &aws.Config{
-		Region:      awsRegion,
-		Credentials: credentials,
+	awsconfig := aws.NewConfig().
+		WithRegion(region).
+		WithCredentials(credentials)
+
+	if url, ok := config[AWSUrl]; ok {
+		awsconfig = awsconfig.WithEndpoint(url)
+	}
+
+	if pathstyle, ok := config[AWSForcePath]; ok {
+		if pathstyle == "true" {
+			awsconfig = awsconfig.WithS3ForcePathStyle(true)
+		}
+	}
+
+	if disablessl, ok := config[AWSSsl]; ok {
+		if disablessl == "true" {
+			awsconfig = awsconfig.WithDisableSSL(true)
+		}
 	}
 
-	s := session.Must(session.NewSession(d))
+	s := session.Must(session.NewSession(awsconfig))
 	return s3blob.OpenBucket(ctx, s, bucketName, nil)
 }
 
@@ -162,14 +178,8 @@ func (c *Conn) Init(config map[string]string) error {
 	}
 	c.prefix = prefix
 
-	region, err := config[REGION]
-	if !err {
-		c.Log.Warnf("No region provided..")
-	}
-	c.region = region
-
 	c.ctx = context.Background()
-	b, berr := setupBucket(c.ctx, provider, bucketName, region)
+	b, berr := c.setupBucket(c.ctx, provider, bucketName, config)
 	if berr != nil {
 		return errors.Errorf("Failed to setup bucket : %s", berr.Error())
 	}

pkg/clouduploader/operation.go

@@ -25,7 +25,7 @@ const (
 // It will create a TCP server through which client can
 // connect and upload data to cloud blob storage file
 func (c *Conn) Upload(file string) bool {
-	c.Log.Infof("Uploading snapshot to  '%s' with provider{%s} to bucket{%s}:region{%s}", file, c.provider, c.bucketname, c.region)
+	c.Log.Infof("Uploading snapshot to  '%s' with provider{%s} to bucket{%s}", file, c.provider, c.bucketname)
 	c.file = file
 	s := &Server{
 		Log: c.Log,
@@ -46,7 +46,7 @@ func (c *Conn) Upload(file string) bool {
 
 // Delete will delete file from cloud blob storage
 func (c *Conn) Delete(file string) bool {
-	c.Log.Infof("Removing snapshot:'%s' from bucket{%s} provider{%s}:region{%s}", file, c.bucket, c.provider, c.region)
+	c.Log.Infof("Removing snapshot:'%s' from bucket{%s} provider{%s}", file, c.bucket, c.provider)
 
 	if c.bucket.Delete(c.ctx, file) != nil {
 		c.Log.Errorf("Failed to remove snapshot{%s} from cloud", file)
@@ -75,7 +75,7 @@ func (c *Conn) Download(file string) bool {
 
 // Write will write data to cloud blob storage file
 func (c *Conn) Write(data []byte, file string) bool {
-	c.Log.Infof("Writing to {%s} with provider{%v} to bucket{%v}:region{%v}", file, c.provider, c.bucketname, c.region)
+	c.Log.Infof("Writing to {%s} with provider{%v} to bucket{%v}", file, c.provider, c.bucketname)
 
 	w, err := c.bucket.NewWriter(c.ctx, file, nil)
 	if err != nil {
@@ -102,7 +102,7 @@ func (c *Conn) Write(data []byte, file string) bool {
 
 // Read will return content of file from cloud blob storage
 func (c *Conn) Read(file string) ([]byte, bool) {
-	c.Log.Infof("Reading from {%s} with provider{%s} to bucket{%s}:region{%s}", file, c.provider, c.bucketname, c.region)
+	c.Log.Infof("Reading from {%s} with provider{%s} to bucket{%s}", file, c.provider, c.bucketname)
 
 	data, err := c.bucket.ReadAll(c.ctx, file)
 	if err != nil {

velero-blockstore-cstor/main.go

@@ -8,7 +8,7 @@ import (
 
 func main() {
 	veleroplugin.NewServer().
-		RegisterVolumeSnapshotter("mayadata.io/cstor-blockstore", openebsSnapPlugin).
+		RegisterVolumeSnapshotter("openebs.io/cstor-blockstore", openebsSnapPlugin).
 		Serve()
 }