Skip to content

Commit afc471d

Browse files
adreed-msftmohsha-msft
adreed-msft
and
mohsha-msft
authored
Implement version/object level WORM operations (Legal hold & Immutability Policy) (#300)
* Implement Legal hold & Immutability Policy * Add immutability policy options to relevant blob functions * Fix tests, add on create tests * Fix tests after merge * Set environment variables * Only scan for immutability if the container could have it * Extend testing, close response bodies. * Increase stack history size * Larger history size * Back to 7 * Only use SRP to cleanup immutability containers * Handle containers not existing * Add list testing * Add immutability SAS and tests * Disable race testing for now * Added Generated Code | Service Version 2020-10-02 (#315) * Added Generated Code | Service Version 2020-10-02 * Minor Edit * TestListBlobsIncludeDeletedWithVersion Co-authored-by: Mohit Sharma <[email protected]>
1 parent 83a533b commit afc471d

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+1688
-1607
lines changed

‎.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -292,3 +292,5 @@ __pycache__/
292292

293293
vendor/
294294
*.env
295+
296+
*.bin

‎azblob/access_conditions.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -13,18 +13,18 @@ type ModifiedAccessConditions struct {
1313
}
1414

1515
// pointers is for internal infrastructure. It returns the fields as pointers.
16-
func (ac ModifiedAccessConditions) pointers() (ims *time.Time, ius *time.Time, ime *ETag, inme *ETag) {
16+
func (ac ModifiedAccessConditions) pointers() (ims *time.Time, ius *time.Time, ime *string, inme *string) {
1717
if !ac.IfModifiedSince.IsZero() {
1818
ims = &ac.IfModifiedSince
1919
}
2020
if !ac.IfUnmodifiedSince.IsZero() {
2121
ius = &ac.IfUnmodifiedSince
2222
}
2323
if ac.IfMatch != ETagNone {
24-
ime = &ac.IfMatch
24+
ime = (*string)(&ac.IfMatch)
2525
}
2626
if ac.IfNoneMatch != ETagNone {
27-
inme = &ac.IfNoneMatch
27+
inme = (*string)(&ac.IfNoneMatch)
2828
}
2929
return
3030
}

‎azblob/blob.json

+37-38
Original file line numberDiff line numberDiff line change
@@ -2127,7 +2127,7 @@
21272127
"type": "integer",
21282128
"description": "The number of committed blocks present in the blob. This header is returned only for append blobs."
21292129
},
2130-
"x-ms-server-encrypted": {
2130+
"x-ms-server-encrypted": {
21312131
"x-ms-client-name": "IsServerEncrypted",
21322132
"type": "boolean",
21332133
"description": "The value of this header is set to true if the blob data and application metadata are completely encrypted using the specified algorithm. Otherwise, the value is set to false (when the blob is unencrypted, or if only parts of the blob/application metadata are encrypted)."
@@ -2323,7 +2323,7 @@
23232323
"type": "integer",
23242324
"description": "The number of committed blocks present in the blob. This header is returned only for append blobs."
23252325
},
2326-
"x-ms-server-encrypted": {
2326+
"x-ms-server-encrypted": {
23272327
"x-ms-client-name": "IsServerEncrypted",
23282328
"type": "boolean",
23292329
"description": "The value of this header is set to true if the blob data and application metadata are completely encrypted using the specified algorithm. Otherwise, the value is set to false (when the blob is unencrypted, or if only parts of the blob/application metadata are encrypted)."
@@ -2581,7 +2581,7 @@
25812581
"type": "integer",
25822582
"description": "The number of committed blocks present in the blob. This header is returned only for append blobs."
25832583
},
2584-
"x-ms-server-encrypted": {
2584+
"x-ms-server-encrypted": {
25852585
"x-ms-client-name": "IsServerEncrypted",
25862586
"type": "boolean",
25872587
"description": "The value of this header is set to true if the blob data and application metadata are completely encrypted using the specified algorithm. Otherwise, the value is set to false (when the blob is unencrypted, or if only parts of the blob/application metadata are encrypted)."
@@ -2796,12 +2796,12 @@
27962796
"type": "string",
27972797
"format": "date-time-rfc1123",
27982798
"description": "UTC date/time value generated by the service that indicates the time at which the response was initiated"
2799-
},
2800-
"x-ms-request-server-encrypted": {
2801-
"x-ms-client-name": "IsServerEncrypted",
2802-
"type": "boolean",
2803-
"description": "The value of this header is set to true if the contents of the request are successfully encrypted using the specified algorithm, and false otherwise."
2804-
}
2799+
},
2800+
"x-ms-request-server-encrypted": {
2801+
"x-ms-client-name": "IsServerEncrypted",
2802+
"type": "boolean",
2803+
"description": "The value of this header is set to true if the contents of the request are successfully encrypted using the specified algorithm, and false otherwise."
2804+
}
28052805
}
28062806
},
28072807
"default": {
@@ -2930,7 +2930,7 @@
29302930
"type": "string",
29312931
"format": "date-time-rfc1123",
29322932
"description": "UTC date/time value generated by the service that indicates the time at which the response was initiated"
2933-
},
2933+
},
29342934
"x-ms-request-server-encrypted": {
29352935
"x-ms-client-name": "IsServerEncrypted",
29362936
"type": "boolean",
@@ -3067,7 +3067,7 @@
30673067
"type": "string",
30683068
"format": "date-time-rfc1123",
30693069
"description": "UTC date/time value generated by the service that indicates the time at which the response was initiated"
3070-
},
3070+
},
30713071
"x-ms-request-server-encrypted": {
30723072
"x-ms-client-name": "IsServerEncrypted",
30733073
"type": "boolean",
@@ -3351,7 +3351,7 @@
33513351
"type": "string",
33523352
"format": "date-time-rfc1123",
33533353
"description": "UTC date/time value generated by the service that indicates the time at which the response was initiated"
3354-
},
3354+
},
33553355
"x-ms-request-server-encrypted": {
33563356
"x-ms-client-name": "IsServerEncrypted",
33573357
"type": "boolean",
@@ -4602,7 +4602,7 @@
46024602
"type": "string",
46034603
"format": "date-time-rfc1123",
46044604
"description": "UTC date/time value generated by the service that indicates the time at which the response was initiated"
4605-
},
4605+
},
46064606
"x-ms-request-server-encrypted": {
46074607
"x-ms-client-name": "IsServerEncrypted",
46084608
"type": "boolean",
@@ -4665,7 +4665,7 @@
46654665
{
46664666
"$ref": "#/parameters/LeaseIdOptional"
46674667
},
4668-
{
4668+
{
46694669
"$ref": "#/parameters/SourceIfModifiedSince"
46704670
},
46714671
{
@@ -4682,7 +4682,7 @@
46824682
},
46834683
{
46844684
"$ref": "#/parameters/ClientRequestId"
4685-
}
4685+
}
46864686
],
46874687
"responses": {
46884688
"201": {
@@ -4707,7 +4707,7 @@
47074707
"type": "string",
47084708
"format": "date-time-rfc1123",
47094709
"description": "UTC date/time value generated by the service that indicates the time at which the response was initiated"
4710-
},
4710+
},
47114711
"x-ms-request-server-encrypted": {
47124712
"x-ms-client-name": "IsServerEncrypted",
47134713
"type": "boolean",
@@ -4836,7 +4836,7 @@
48364836
"type": "string",
48374837
"format": "date-time-rfc1123",
48384838
"description": "UTC date/time value generated by the service that indicates the time at which the response was initiated"
4839-
},
4839+
},
48404840
"x-ms-request-server-encrypted": {
48414841
"x-ms-client-name": "IsServerEncrypted",
48424842
"type": "boolean",
@@ -5050,7 +5050,7 @@
50505050
"type": "string",
50515051
"format": "date-time-rfc1123",
50525052
"description": "UTC date/time value generated by the service that indicates the time at which the response was initiated"
5053-
},
5053+
},
50545054
"x-ms-request-server-encrypted": {
50555055
"x-ms-client-name": "IsServerEncrypted",
50565056
"type": "boolean",
@@ -7541,7 +7541,6 @@
75417541
"x-ms-parameter-location": "method",
75427542
"description": "The container name."
75437543
},
7544-
75457544
"ContentLength": {
75467545
"name": "Content-Length",
75477546
"in": "header",
@@ -7905,15 +7904,15 @@
79057904
"type": "string",
79067905
"x-ms-parameter-location": "method",
79077906
"description": "Bytes of source data in the specified range."
7908-
},
7909-
"SourceRangeRequiredPutPageFromUrl": {
7910-
"name": "x-ms-source-range",
7911-
"x-ms-client-name": "sourceRange",
7912-
"in": "header",
7913-
"required": true,
7914-
"type": "string",
7915-
"x-ms-parameter-location": "method",
7916-
"description": "Bytes of source data in the specified range. The length of this range should match the ContentLength header and x-ms-range/Range destination range header."
7907+
},
7908+
"SourceRangeRequiredPutPageFromUrl": {
7909+
"name": "x-ms-source-range",
7910+
"x-ms-client-name": "sourceRange",
7911+
"in": "header",
7912+
"required": true,
7913+
"type": "string",
7914+
"x-ms-parameter-location": "method",
7915+
"description": "Bytes of source data in the specified range. The length of this range should match the ContentLength header and x-ms-range/Range destination range header."
79177916
},
79187917
"SourceIfMatch": {
79197918
"name": "x-ms-source-if-match",
@@ -7967,15 +7966,15 @@
79677966
},
79687967
"description": "Specify this header value to operate only on a blob if it has not been modified since the specified date/time."
79697968
},
7970-
"SourceLeaseId": {
7971-
"name": "x-ms-source-lease-id",
7972-
"x-ms-client-name": "sourceLeaseId",
7973-
"in": "header",
7974-
"required": false,
7975-
"type": "string",
7976-
"x-ms-parameter-location": "method",
7977-
"description": "A lease ID for the source path. If specified, the source path must have an active lease and the leaase ID must match."
7978-
},
7969+
"SourceLeaseId": {
7970+
"name": "x-ms-source-lease-id",
7971+
"x-ms-client-name": "sourceLeaseId",
7972+
"in": "header",
7973+
"required": false,
7974+
"type": "string",
7975+
"x-ms-parameter-location": "method",
7976+
"description": "A lease ID for the source path. If specified, the source path must have an active lease and the leaase ID must match."
7977+
},
79797978
"SourceUrl": {
79807979
"name": "x-ms-copy-source",
79817980
"x-ms-client-name": "sourceUrl",
@@ -8004,6 +8003,6 @@
80048003
"minimum": 0,
80058004
"x-ms-parameter-location": "method",
80068005
"description": "The timeout parameter is expressed in seconds. For more information, see <a href=\"https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/setting-timeouts-for-blob-service-operations\">Setting Timeouts for Blob Service Operations.</a>"
8007-
}
8006+
}
80088007
}
80098008
}

‎azblob/chunkwriting.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ import (
1818
// This allows us to provide a local implementation that fakes the server for hermetic testing.
1919
type blockWriter interface {
2020
StageBlock(context.Context, string, io.ReadSeeker, LeaseAccessConditions, []byte, ClientProvidedKeyOptions) (*BlockBlobStageBlockResponse, error)
21-
CommitBlockList(context.Context, []string, BlobHTTPHeaders, Metadata, BlobAccessConditions, AccessTierType, BlobTagsMap, ClientProvidedKeyOptions) (*BlockBlobCommitBlockListResponse, error)
21+
CommitBlockList(context.Context, []string, BlobHTTPHeaders, Metadata, BlobAccessConditions, AccessTierType, BlobTagsMap, ClientProvidedKeyOptions, ImmutabilityPolicyOptions) (*BlockBlobCommitBlockListResponse, error)
2222
}
2323

2424
// copyFromReader copies a source io.Reader to blob storage using concurrent uploads.
@@ -183,7 +183,7 @@ func (c *copier) close() error {
183183
}
184184

185185
var err error
186-
c.result, err = c.to.CommitBlockList(c.ctx, c.id.issued(), c.o.BlobHTTPHeaders, c.o.Metadata, c.o.AccessConditions, c.o.BlobAccessTier, c.o.BlobTagsMap, c.o.ClientProvidedKeyOptions)
186+
c.result, err = c.to.CommitBlockList(c.ctx, c.id.issued(), c.o.BlobHTTPHeaders, c.o.Metadata, c.o.AccessConditions, c.o.BlobAccessTier, c.o.BlobTagsMap, c.o.ClientProvidedKeyOptions, c.o.ImmutabilityPolicyOptions)
187187
return err
188188
}
189189

@@ -206,7 +206,7 @@ func newID() *id {
206206
func (id *id) next() string {
207207
defer atomic.AddUint32(&id.num, 1)
208208

209-
binary.BigEndian.PutUint32((id.u[len(guuid.UUID{}):]), atomic.LoadUint32(&id.num))
209+
binary.BigEndian.PutUint32(id.u[len(guuid.UUID{}):], atomic.LoadUint32(&id.num))
210210
str := base64.StdEncoding.EncodeToString(id.u[:])
211211
id.all = append(id.all, str)
212212

‎azblob/chunkwriting_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ func (f *fakeBlockWriter) StageBlock(ctx context.Context, blockID string, r io.R
6060
return &BlockBlobStageBlockResponse{}, nil
6161
}
6262

63-
func (f *fakeBlockWriter) CommitBlockList(ctx context.Context, blockIDs []string, headers BlobHTTPHeaders, meta Metadata, access BlobAccessConditions, tier AccessTierType, blobTagsMap BlobTagsMap, options ClientProvidedKeyOptions) (*BlockBlobCommitBlockListResponse, error) {
63+
func (f *fakeBlockWriter) CommitBlockList(ctx context.Context, blockIDs []string, headers BlobHTTPHeaders, meta Metadata, access BlobAccessConditions, tier AccessTierType, blobTagsMap BlobTagsMap, options ClientProvidedKeyOptions, immutability ImmutabilityPolicyOptions) (*BlockBlobCommitBlockListResponse, error) {
6464
dst, err := os.OpenFile(filepath.Join(f.path, finalFileName), os.O_CREATE+os.O_WRONLY, 0600)
6565
if err != nil {
6666
return nil, err

‎azblob/highlevel.go

+16-13
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,17 @@
11
package azblob
22

33
import (
4+
"bytes"
45
"context"
56
"encoding/base64"
7+
"errors"
68
"fmt"
79
"io"
810
"net/http"
9-
10-
"bytes"
1111
"os"
1212
"sync"
1313
"time"
1414

15-
"errors"
16-
1715
"github.com/Azure/azure-pipeline-go/pipeline"
1816
)
1917

@@ -65,6 +63,10 @@ type UploadToBlockBlobOptions struct {
6563
// ClientProvidedKeyOptions indicates the client provided key by name and/or by value to encrypt/decrypt data.
6664
ClientProvidedKeyOptions ClientProvidedKeyOptions
6765

66+
// ImmutabilityPolicyOptions indicates a immutability policy or legal hold to be placed upon finishing upload.
67+
// A container with object-level immutability enabled is required.
68+
ImmutabilityPolicyOptions ImmutabilityPolicyOptions
69+
6870
// Parallelism indicates the maximum number of blocks to upload in parallel (0=default)
6971
Parallelism uint16
7072
}
@@ -95,7 +97,7 @@ func uploadReaderAtToBlockBlob(ctx context.Context, reader io.ReaderAt, readerSi
9597
if o.Progress != nil {
9698
body = pipeline.NewRequestBodyProgress(body, o.Progress)
9799
}
98-
return blockBlobURL.Upload(ctx, body, o.BlobHTTPHeaders, o.Metadata, o.AccessConditions, o.BlobAccessTier, o.BlobTagsMap, o.ClientProvidedKeyOptions)
100+
return blockBlobURL.Upload(ctx, body, o.BlobHTTPHeaders, o.Metadata, o.AccessConditions, o.BlobAccessTier, o.BlobTagsMap, o.ClientProvidedKeyOptions, o.ImmutabilityPolicyOptions)
99101
}
100102

101103
var numBlocks = uint16(((readerSize - 1) / o.BlockSize) + 1)
@@ -139,7 +141,7 @@ func uploadReaderAtToBlockBlob(ctx context.Context, reader io.ReaderAt, readerSi
139141
return nil, err
140142
}
141143
// All put blocks were successful, call Put Block List to finalize the blob
142-
return blockBlobURL.CommitBlockList(ctx, blockIDList, o.BlobHTTPHeaders, o.Metadata, o.AccessConditions, o.BlobAccessTier, o.BlobTagsMap, o.ClientProvidedKeyOptions)
144+
return blockBlobURL.CommitBlockList(ctx, blockIDList, o.BlobHTTPHeaders, o.Metadata, o.AccessConditions, o.BlobAccessTier, o.BlobTagsMap, o.ClientProvidedKeyOptions, o.ImmutabilityPolicyOptions)
143145
}
144146

145147
// UploadBufferToBlockBlob uploads a buffer in blocks to a block blob.
@@ -507,13 +509,14 @@ type UploadStreamToBlockBlobOptions struct {
507509
// BufferSize sizes the buffer used to read data from source. If < 1 MiB, defaults to 1 MiB.
508510
BufferSize int
509511
// MaxBuffers defines the number of simultaneous uploads will be performed to upload the file.
510-
MaxBuffers int
511-
BlobHTTPHeaders BlobHTTPHeaders
512-
Metadata Metadata
513-
AccessConditions BlobAccessConditions
514-
BlobAccessTier AccessTierType
515-
BlobTagsMap BlobTagsMap
516-
ClientProvidedKeyOptions ClientProvidedKeyOptions
512+
MaxBuffers int
513+
BlobHTTPHeaders BlobHTTPHeaders
514+
Metadata Metadata
515+
AccessConditions BlobAccessConditions
516+
BlobAccessTier AccessTierType
517+
BlobTagsMap BlobTagsMap
518+
ClientProvidedKeyOptions ClientProvidedKeyOptions
519+
ImmutabilityPolicyOptions ImmutabilityPolicyOptions
517520
}
518521

519522
func (u *UploadStreamToBlockBlobOptions) defaults() error {

‎azblob/request_common.go

+23
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
package azblob
22

3+
import (
4+
"time"
5+
)
6+
37
// ClientProvidedKeyOptions contains headers which may be be specified from service version 2019-02-02
48
// or higher to encrypts the data on the service-side with the given key. Use of customer-provided keys
59
// must be done over HTTPS. As the encryption key itself is provided in the request, a secure connection
@@ -31,3 +35,22 @@ func NewClientProvidedKeyOptions(ek *string, eksha256 *string, es *string) (cpk
3135
cpk.EncryptionKey, cpk.EncryptionKeySha256, cpk.EncryptionAlgorithm, cpk.EncryptionScope = ek, eksha256, EncryptionAlgorithmAES256, es
3236
return cpk
3337
}
38+
39+
type ImmutabilityPolicyOptions struct {
40+
// A container with object-level immutability enabled is required for any options.
41+
// Both ImmutabilityPolicy options must be filled to set an immutability policy.
42+
ImmutabilityPolicyUntilDate *time.Time
43+
ImmutabilityPolicyMode BlobImmutabilityPolicyModeType
44+
45+
LegalHold *bool
46+
}
47+
48+
func NewImmutabilityPolicyOptions(untilDate *time.Time, policyMode BlobImmutabilityPolicyModeType, legalHold *bool) ImmutabilityPolicyOptions {
49+
opt := ImmutabilityPolicyOptions{}
50+
opt.ImmutabilityPolicyUntilDate, opt.ImmutabilityPolicyMode, opt.LegalHold = untilDate, policyMode, legalHold
51+
return opt
52+
}
53+
54+
func (pol *ImmutabilityPolicyOptions) pointers() (*time.Time, BlobImmutabilityPolicyModeType, *bool) {
55+
return pol.ImmutabilityPolicyUntilDate, pol.ImmutabilityPolicyMode, pol.LegalHold
56+
}

0 commit comments

Comments
 (0)