Skip to content

Correctly prefix storage emulator routes #3786

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Correctly prefix storage emulator routes #3786

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
78655ba
Correctly prefix storage emulator routes
jcmorrow Sep 25, 2021
d73430c
Add route to create bucket and test for the same
jcmorrow Sep 27, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
118 changes: 66 additions & 52 deletions src/emulator/storage/apis/gcloud.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,12 @@ export function createCloudEndpoints(emulator: StorageEmulator): Router {
next();
});

gcloudStorageAPI.get("/b", (req, res) => {
gcloudStorageAPI.post(["/b", "/storage/v1/b"], (req, res) => {
storageLayer.createBucket(req.params[0]);
res.json(storageLayer.getBucketMetadata(req.params[0]));
});

gcloudStorageAPI.get(["/b", "/storage/v1/b"], (req, res) => {
res.json({
kind: "storage#buckets",
items: storageLayer.listBuckets(),
Expand Down Expand Up @@ -55,22 +60,25 @@ export function createCloudEndpoints(emulator: StorageEmulator): Router {
}
);

gcloudStorageAPI.patch("/b/:bucketId/o/:objectId", (req, res) => {
const md = storageLayer.getMetadata(req.params.bucketId, req.params.objectId);
gcloudStorageAPI.patch(
["/b/:bucketId/o/:objectId", "/storage/v1/b/:bucketId/o/:objectId"],
(req, res) => {
const md = storageLayer.getMetadata(req.params.bucketId, req.params.objectId);

if (!md) {
res.sendStatus(404);
return;
}
if (!md) {
res.sendStatus(404);
return;
}

md.update(req.body);
md.update(req.body);

const outgoingMetadata = new CloudStorageObjectMetadata(md);
res.json(outgoingMetadata).status(200).send();
return;
});
const outgoingMetadata = new CloudStorageObjectMetadata(md);
res.json(outgoingMetadata).status(200).send();
return;
}
);

gcloudStorageAPI.get("/b/:bucketId/o", (req, res) => {
gcloudStorageAPI.get(["/b/:bucketId/o", "/storage/v1/b/:bucketId/o"], (req, res) => {
// TODO validate that all query params are single strings and are not repeated.
let maxRes = undefined;
if (req.query.maxResults) {
Expand All @@ -91,17 +99,20 @@ export function createCloudEndpoints(emulator: StorageEmulator): Router {
res.json(listResult);
});

gcloudStorageAPI.delete("/b/:bucketId/o/:objectId", (req, res) => {
const md = storageLayer.getMetadata(req.params.bucketId, req.params.objectId);
gcloudStorageAPI.delete(
["/b/:bucketId/o/:objectId", "/storage/v1/b/:bucketId/o/:objectId"],
(req, res) => {
const md = storageLayer.getMetadata(req.params.bucketId, req.params.objectId);

if (!md) {
res.sendStatus(404);
return;
}
if (!md) {
res.sendStatus(404);
return;
}

storageLayer.deleteFile(req.params.bucketId, req.params.objectId);
res.status(200).send();
});
storageLayer.deleteFile(req.params.bucketId, req.params.objectId);
res.status(200).send();
}
);

gcloudStorageAPI.put("/upload/storage/v1/b/:bucketId/o", async (req, res) => {
if (!req.query.upload_id) {
Expand Down Expand Up @@ -139,38 +150,41 @@ export function createCloudEndpoints(emulator: StorageEmulator): Router {
res.status(200).json(new CloudStorageObjectMetadata(finalizedUpload.file.metadata)).send();
});

gcloudStorageAPI.post("/b/:bucketId/o/:objectId/acl", (req, res) => {
// TODO(abehaskins) Link to a doc with more info
EmulatorLogger.forEmulator(Emulators.STORAGE).log(
"WARN_ONCE",
"Cloud Storage ACLs are not supported in the Storage Emulator. All related methods will succeed, but have no effect."
);
const md = storageLayer.getMetadata(req.params.bucketId, req.params.objectId);
gcloudStorageAPI.post(
["/b/:bucketId/o/:objectId/acl", "/storage/v1/b/:bucketId/o/:objectId/acl"],
(req, res) => {
// TODO(abehaskins) Link to a doc with more info
EmulatorLogger.forEmulator(Emulators.STORAGE).log(
"WARN_ONCE",
"Cloud Storage ACLs are not supported in the Storage Emulator. All related methods will succeed, but have no effect."
);
const md = storageLayer.getMetadata(req.params.bucketId, req.params.objectId);

if (!md) {
res.sendStatus(404);
return;
}
if (!md) {
res.sendStatus(404);
return;
}

// We do an empty update to step metageneration forward;
md.update({});

res
.json({
kind: "storage#objectAccessControl",
object: md.name,
id: `${req.params.bucketId}/${md.name}/${md.generation}/allUsers`,
selfLink: `http://${EmulatorRegistry.getInfo(Emulators.STORAGE)?.host}:${
EmulatorRegistry.getInfo(Emulators.STORAGE)?.port
}/storage/v1/b/${md.bucket}/o/${encodeURIComponent(md.name)}/acl/allUsers`,
bucket: md.bucket,
entity: req.body.entity,
role: req.body.role,
etag: "someEtag",
generation: md.generation.toString(),
} as CloudStorageObjectAccessControlMetadata)
.status(200);
});
// We do an empty update to step metageneration forward;
md.update({});

res
.json({
kind: "storage#objectAccessControl",
object: md.name,
id: `${req.params.bucketId}/${md.name}/${md.generation}/allUsers`,
selfLink: `http://${EmulatorRegistry.getInfo(Emulators.STORAGE)?.host}:${
EmulatorRegistry.getInfo(Emulators.STORAGE)?.port
}/storage/v1/b/${md.bucket}/o/${encodeURIComponent(md.name)}/acl/allUsers`,
bucket: md.bucket,
entity: req.body.entity,
role: req.body.role,
etag: "someEtag",
generation: md.generation.toString(),
} as CloudStorageObjectAccessControlMetadata)
.status(200);
}
);

gcloudStorageAPI.post("/upload/storage/v1/b/:bucketId/o", (req, res) => {
if (!req.query.name) {
Expand Down
212 changes: 111 additions & 101 deletions src/emulator/storage/files.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,104 @@ export type FinalizedUpload = {
file: StoredFile;
};

export class Persistence {
private _dirPath: string;
constructor(dirPath: string) {
this._dirPath = dirPath;
if (!existsSync(dirPath)) {
mkdirSync(dirPath, {
recursive: true,
});
}
}

public get dirPath(): string {
return this._dirPath;
}

appendBytes(fileName: string, bytes: Buffer, fileOffset?: number): string {
const filepath = this.getDiskPath(fileName);

const encodedSlashIndex = filepath.toLowerCase().lastIndexOf("%2f");
const dirPath =
encodedSlashIndex >= 0 ? filepath.substring(0, encodedSlashIndex) : path.dirname(filepath);

if (!existsSync(dirPath)) {
mkdirSync(dirPath, {
recursive: true,
});
}
let fd;

try {
// TODO: This is more technically correct, but corrupts multipart files
// fd = openSync(path, "w+");
// writeSync(fd, bytes, 0, bytes.byteLength, fileOffset);

fs.appendFileSync(filepath, bytes);
return filepath;
} finally {
if (fd) {
closeSync(fd);
}
}
}

readBytes(fileName: string, size: number, fileOffset?: number): Buffer {
const path = this.getDiskPath(fileName);
let fd;
try {
fd = openSync(path, "r");
const buf = Buffer.alloc(size);
const offset = fileOffset && fileOffset > 0 ? fileOffset : 0;
readSync(fd, buf, 0, size, offset);
return buf;
} finally {
if (fd) {
closeSync(fd);
}
}
}

deleteFile(fileName: string, failSilently = false): void {
try {
unlinkSync(this.getDiskPath(fileName));
} catch (err) {
if (!failSilently) {
throw err;
}
}
}

deleteAll(): Promise<void> {
return new Promise((resolve, reject) => {
rimraf(this._dirPath, (err) => {
if (err) {
reject(err);
} else {
resolve();
}
});
});
}

renameFile(oldName: string, newName: string): void {
const dirPath = this.getDiskPath(path.dirname(newName));

if (!existsSync(dirPath)) {
mkdirSync(dirPath, {
recursive: true,
});
}

renameSync(this.getDiskPath(oldName), this.getDiskPath(newName));
}

getDiskPath(fileName: string): string {
return path.join(this._dirPath, fileName);
}
}

export class StorageLayer {
private _files!: Map<string, StoredFile>;
private _uploads!: Map<string, ResumableUpload>;
Expand All @@ -128,10 +226,10 @@ export class StorageLayer {
}

public reset(): void {
this._files = new Map();
this._files = new Map<string, StoredFile>();
this._persistence = new Persistence(`${tmpdir()}/firebase/storage/blobs`);
this._uploads = new Map();
this._buckets = new Map();
this._uploads = new Map<string, ResumableUpload>();
this._buckets = new Map<string, CloudStorageBucketMetadata>();
}

createBucket(id: string): void {
Expand All @@ -148,6 +246,16 @@ export class StorageLayer {
return [...this._buckets.values()];
}

getBucketMetadata(id: string): CloudStorageBucketMetadata {
const metadata = this._buckets.get(id);
if (metadata) {
return metadata;
} else {
this.createBucket(id);
return new CloudStorageBucketMetadata(id);
}
}

public getMetadata(bucket: string, object: string): StoredFileMetadata | undefined {
const key = this.path(bucket, object);
const val = this._files.get(key);
Expand Down Expand Up @@ -580,101 +688,3 @@ export class StorageLayer {
}
}
}

export class Persistence {
private _dirPath: string;
constructor(dirPath: string) {
this._dirPath = dirPath;
if (!existsSync(dirPath)) {
mkdirSync(dirPath, {
recursive: true,
});
}
}

public get dirPath(): string {
return this._dirPath;
}

appendBytes(fileName: string, bytes: Buffer, fileOffset?: number): string {
const filepath = this.getDiskPath(fileName);

const encodedSlashIndex = filepath.toLowerCase().lastIndexOf("%2f");
const dirPath =
encodedSlashIndex >= 0 ? filepath.substring(0, encodedSlashIndex) : path.dirname(filepath);

if (!existsSync(dirPath)) {
mkdirSync(dirPath, {
recursive: true,
});
}
let fd;

try {
// TODO: This is more technically correct, but corrupts multipart files
// fd = openSync(path, "w+");
// writeSync(fd, bytes, 0, bytes.byteLength, fileOffset);

fs.appendFileSync(filepath, bytes);
return filepath;
} finally {
if (fd) {
closeSync(fd);
}
}
}

readBytes(fileName: string, size: number, fileOffset?: number): Buffer {
const path = this.getDiskPath(fileName);
let fd;
try {
fd = openSync(path, "r");
const buf = Buffer.alloc(size);
const offset = fileOffset && fileOffset > 0 ? fileOffset : 0;
readSync(fd, buf, 0, size, offset);
return buf;
} finally {
if (fd) {
closeSync(fd);
}
}
}

deleteFile(fileName: string, failSilently = false): void {
try {
unlinkSync(this.getDiskPath(fileName));
} catch (err) {
if (!failSilently) {
throw err;
}
}
}

deleteAll(): Promise<void> {
return new Promise((resolve, reject) => {
rimraf(this._dirPath, (err) => {
if (err) {
reject(err);
} else {
resolve();
}
});
});
}

renameFile(oldName: string, newName: string): void {
const dirPath = this.getDiskPath(path.dirname(newName));

if (!existsSync(dirPath)) {
mkdirSync(dirPath, {
recursive: true,
});
}

renameSync(this.getDiskPath(oldName), this.getDiskPath(newName));
}

getDiskPath(fileName: string): string {
return path.join(this._dirPath, fileName);
}
}
Loading