Skip to content

Commit 524f98d

Browse files
tohhsinpeitohhsinpei
authored
Add StorageRulesManager class for emulator (#4245)
* Add StorageRulesManager class for emulator * Expose setSourceFile; add unit test * Address PR feedback * Address PR feedback
1 parent ad03551 commit 524f98d

File tree

5 files changed

+232
-109
lines changed

5 files changed

+232
-109
lines changed

‎src/emulator/storage/index.ts

Lines changed: 12 additions & 99 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,10 @@ import { Constants } from "../constants";
44
import { EmulatorInfo, EmulatorInstance, Emulators } from "../types";
55
import { createApp } from "./server";
66
import { StorageLayer } from "./files";
7-
import * as chokidar from "chokidar";
87
import { EmulatorLogger } from "../emulatorLogger";
9-
import * as fs from "fs";
8+
import { StorageRulesManager } from "./rules/manager";
109
import { StorageRulesetInstance, StorageRulesRuntime, StorageRulesIssues } from "./rules/runtime";
11-
import { Source } from "./rules/types";
12-
import { FirebaseError } from "../../error";
10+
import { SourceFile } from "./rules/types";
1311
import express = require("express");
1412
import { getRulesValidator } from "./rules/utils";
1513
import { Persistence } from "./persistence";
@@ -19,25 +17,24 @@ export interface StorageEmulatorArgs {
1917
projectId: string;
2018
port?: number;
2119
host?: string;
22-
rules: Source | string;
20+
rules: SourceFile | string;
2321
auto_download?: boolean;
2422
}
2523

2624
export class StorageEmulator implements EmulatorInstance {
2725
private destroyServer?: () => Promise<void>;
2826
private _app?: express.Express;
29-
private _rulesWatcher?: chokidar.FSWatcher;
30-
private _rules?: StorageRulesetInstance;
31-
private _rulesetSource?: Source;
3227

3328
private _logger = EmulatorLogger.forEmulator(Emulators.STORAGE);
3429
private _rulesRuntime: StorageRulesRuntime;
30+
private _rulesManager: StorageRulesManager;
3531
private _persistence: Persistence;
3632
private _storageLayer: StorageLayer;
3733
private _uploadService: UploadService;
3834

3935
constructor(private args: StorageEmulatorArgs) {
4036
this._rulesRuntime = new StorageRulesRuntime();
37+
this._rulesManager = new StorageRulesManager(this._rulesRuntime);
4138
this._persistence = new Persistence(this.getPersistenceTmpDir());
4239
this._storageLayer = new StorageLayer(
4340
args.projectId,
@@ -56,7 +53,7 @@ export class StorageEmulator implements EmulatorInstance {
5653
}
5754

5855
get rules(): StorageRulesetInstance | undefined {
59-
return this._rules;
56+
return this._rulesManager.ruleset;
6057
}
6158

6259
get logger(): EmulatorLogger {
@@ -72,107 +69,23 @@ export class StorageEmulator implements EmulatorInstance {
7269
async start(): Promise<void> {
7370
const { host, port } = this.getInfo();
7471
await this._rulesRuntime.start(this.args.auto_download);
72+
await this._rulesManager.setSourceFile(this.args.rules);
7573
this._app = await createApp(this.args.projectId, this);
76-
77-
if (typeof this.args.rules === "string") {
78-
const rulesFile = this.args.rules;
79-
this.updateRulesSource(rulesFile);
80-
} else {
81-
this._rulesetSource = this.args.rules;
82-
}
83-
84-
if (!this._rulesetSource || this._rulesetSource.files.length === 0) {
85-
throw new FirebaseError("Can not initialize Storage emulator without a rules source / file.");
86-
} else if (this._rulesetSource.files.length > 1) {
87-
throw new FirebaseError(
88-
"Can not initialize Storage emulator with more than one rules source / file."
89-
);
90-
}
91-
92-
await this.loadRuleset();
93-
94-
const rulesPath = this._rulesetSource.files[0].name;
95-
this._rulesWatcher = chokidar.watch(rulesPath, { persistent: true, ignoreInitial: true });
96-
this._rulesWatcher.on("change", async () => {
97-
// There have been some race conditions reported (on Windows) where reading the
98-
// file too quickly after the watcher fires results in an empty file being read.
99-
// Adding a small delay prevents that at very little cost.
100-
await new Promise((res) => setTimeout(res, 5));
101-
102-
this._logger.logLabeled(
103-
"BULLET",
104-
"storage",
105-
`Change detected, updating rules for Cloud Storage...`
106-
);
107-
this.updateRulesSource(rulesPath);
108-
await this.loadRuleset();
109-
});
110-
11174
const server = this._app.listen(port, host);
11275
this.destroyServer = utils.createDestroyer(server);
11376
}
11477

115-
private updateRulesSource(rulesFile: string): void {
116-
this._rulesetSource = {
117-
files: [
118-
{
119-
name: rulesFile,
120-
content: fs.readFileSync(rulesFile).toString(),
121-
},
122-
],
123-
};
124-
}
125-
126-
public async loadRuleset(source?: Source): Promise<StorageRulesIssues> {
127-
if (source) {
128-
this._rulesetSource = source;
129-
}
130-
131-
if (!this._rulesetSource) {
132-
const msg = "Attempting to update ruleset without a source.";
133-
this._logger.log("WARN", msg);
134-
135-
const error = JSON.stringify({ error: msg });
136-
return new StorageRulesIssues([error], []);
137-
}
138-
139-
const { ruleset, issues } = await this._rulesRuntime.loadRuleset(this._rulesetSource);
140-
141-
if (!ruleset) {
142-
issues.all.forEach((issue) => {
143-
let parsedIssue;
144-
try {
145-
parsedIssue = JSON.parse(issue);
146-
} catch {
147-
// Parse manually
148-
}
149-
150-
if (parsedIssue) {
151-
this._logger.log(
152-
"WARN",
153-
`${parsedIssue.description_.replace(/\.$/, "")} in ${
154-
parsedIssue.sourcePosition_.fileName_
155-
}:${parsedIssue.sourcePosition_.line_}`
156-
);
157-
} else {
158-
this._logger.log("WARN", issue);
159-
}
160-
});
161-
162-
delete this._rules;
163-
} else {
164-
this._rules = ruleset;
165-
}
166-
167-
return issues;
168-
}
169-
17078
async connect(): Promise<void> {
17179
// No-op
17280
}
17381

82+
async setRules(rules: SourceFile): Promise<StorageRulesIssues> {
83+
return this._rulesManager.setSourceFile(rules);
84+
}
85+
17486
async stop(): Promise<void> {
17587
await this.storageLayer.deleteAll();
88+
await this._rulesManager.close();
17689
return this.destroyServer ? this.destroyServer() : Promise.resolve();
17790
}
17891

‎src/emulator/storage/rules/manager.ts

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
import * as chokidar from "chokidar";
2+
import * as fs from "fs";
3+
import { EmulatorLogger } from "../../emulatorLogger";
4+
import { Emulators } from "../../types";
5+
import { FirebaseError } from "../../../error";
6+
import { SourceFile } from "./types";
7+
import { StorageRulesIssues, StorageRulesRuntime, StorageRulesetInstance } from "./runtime";
8+
9+
/**
10+
* Loads and maintains a {@link StorageRulesetInstance} for a given source file. Listens for
11+
* changes to the file and updates the ruleset accordingly.
12+
*/
13+
export class StorageRulesManager {
14+
private _sourceFile?: SourceFile;
15+
private _ruleset?: StorageRulesetInstance;
16+
private _watcher = new chokidar.FSWatcher();
17+
private _logger = EmulatorLogger.forEmulator(Emulators.STORAGE);
18+
19+
constructor(private _runtime: StorageRulesRuntime) {}
20+
21+
get ruleset(): StorageRulesetInstance | undefined {
22+
return this._ruleset;
23+
}
24+
25+
/**
26+
* Updates the source file and, correspondingly, the file watcher and ruleset.
27+
* @throws {FirebaseError} if file path is invalid.
28+
*/
29+
public async setSourceFile(rules: SourceFile | string): Promise<StorageRulesIssues> {
30+
const prevRulesFile = this._sourceFile?.name;
31+
let rulesFile: string;
32+
if (typeof rules === "string") {
33+
this._sourceFile = { name: rules, content: readSourceFile(rules) };
34+
rulesFile = rules;
35+
} else {
36+
// Allow invalid file path here for testing
37+
this._sourceFile = rules;
38+
rulesFile = rules.name;
39+
}
40+
41+
const issues = await this.loadRuleset();
42+
this.updateWatcher(rulesFile, prevRulesFile);
43+
return issues;
44+
}
45+
46+
/**
47+
* Deletes source file, ruleset, and removes listeners from all files.
48+
*/
49+
public async close(): Promise<void> {
50+
delete this._sourceFile;
51+
delete this._ruleset;
52+
await this._watcher.close();
53+
}
54+
55+
private updateWatcher(rulesFile: string, prevRulesFile?: string): void {
56+
if (prevRulesFile) {
57+
this._watcher.unwatch(prevRulesFile);
58+
}
59+
60+
this._watcher = chokidar
61+
.watch(rulesFile, { persistent: true, ignoreInitial: true })
62+
.on("change", async () => {
63+
// There have been some race conditions reported (on Windows) where reading the
64+
// file too quickly after the watcher fires results in an empty file being read.
65+
// Adding a small delay prevents that at very little cost.
66+
await new Promise((res) => setTimeout(res, 5));
67+
68+
this._logger.logLabeled(
69+
"BULLET",
70+
"storage",
71+
"Change detected, updating rules for Cloud Storage..."
72+
);
73+
await this.loadRuleset();
74+
});
75+
}
76+
77+
private async loadRuleset(): Promise<StorageRulesIssues> {
78+
const { ruleset, issues } = await this._runtime.loadRuleset({ files: [this._sourceFile!] });
79+
80+
if (ruleset) {
81+
this._ruleset = ruleset;
82+
return issues;
83+
}
84+
85+
delete this._ruleset;
86+
issues.all.forEach((issue: string) => {
87+
try {
88+
const parsedIssue = JSON.parse(issue);
89+
this._logger.log(
90+
"WARN",
91+
`${parsedIssue.description_.replace(/\.$/, "")} in ${
92+
parsedIssue.sourcePosition_.fileName_
93+
}:${parsedIssue.sourcePosition_.line_}`
94+
);
95+
} catch {
96+
this._logger.log("WARN", issue);
97+
}
98+
});
99+
return issues;
100+
}
101+
}
102+
103+
function readSourceFile(fileName: string): string {
104+
try {
105+
return fs.readFileSync(fileName).toString();
106+
} catch (error: any) {
107+
if (error.code === "ENOENT") {
108+
throw new FirebaseError(`File not found: ${fileName}`);
109+
}
110+
throw error;
111+
}
112+
}

‎src/emulator/storage/server.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ export function createApp(
9292

9393
const name = file.name;
9494
const content = file.content;
95-
const issues = await emulator.loadRuleset({ files: [{ name, content }] });
95+
const issues = await emulator.setRules({ name, content });
9696

9797
if (issues.errors.length > 0) {
9898
res.status(400).json({

‎src/test/emulators/storage/rules/manager.spec.ts

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
import { expect } from "chai";
2+
import { tmpdir } from "os";
3+
import { v4 as uuidv4 } from "uuid";
4+
5+
import { FirebaseError } from "../../../../error";
6+
import { StorageRulesFiles, TIMEOUT_MED } from "../../fixtures";
7+
import { StorageRulesManager } from "../../../../emulator/storage/rules/manager";
8+
import { StorageRulesRuntime } from "../../../../emulator/storage/rules/runtime";
9+
import { Persistence } from "../../../../emulator/storage/persistence";
10+
import { RulesetOperationMethod } from "../../../../emulator/storage/rules/types";
11+
12+
describe("Storage Rules Manager", function () {
13+
const rulesRuntime = new StorageRulesRuntime();
14+
const rulesManager = new StorageRulesManager(rulesRuntime);
15+
16+
// eslint-disable-next-line @typescript-eslint/no-invalid-this
17+
this.timeout(TIMEOUT_MED);
18+
19+
before(async () => {
20+
await rulesRuntime.start();
21+
});
22+
23+
after(async () => {
24+
rulesRuntime.stop();
25+
await rulesManager.close();
26+
});
27+
28+
it("should load ruleset from SourceFile object", async () => {
29+
await rulesManager.setSourceFile(StorageRulesFiles.readWriteIfTrue);
30+
expect(rulesManager.ruleset).not.to.be.undefined;
31+
});
32+
33+
it("should load ruleset from file path", async () => {
34+
// Write rules to file
35+
const fileName = "storage.rules";
36+
const testDir = `${tmpdir()}/${uuidv4()}`;
37+
const persistence = new Persistence(testDir);
38+
persistence.appendBytes(fileName, Buffer.from(StorageRulesFiles.readWriteIfTrue.content));
39+
40+
await rulesManager.setSourceFile(`${testDir}/${fileName}`);
41+
42+
expect(rulesManager.ruleset).not.to.be.undefined;
43+
});
44+
45+
it("should set source file", async () => {
46+
await rulesManager.setSourceFile(StorageRulesFiles.readWriteIfTrue);
47+
const opts = { method: RulesetOperationMethod.GET, file: {}, path: "/b/bucket/o/" };
48+
expect((await rulesManager.ruleset!.verify(opts)).permitted).to.be.true;
49+
50+
const issues = await rulesManager.setSourceFile(StorageRulesFiles.readWriteIfAuth);
51+
52+
expect(issues.errors.length).to.equal(0);
53+
expect(issues.warnings.length).to.equal(0);
54+
expect((await rulesManager.ruleset!.verify(opts)).permitted).to.be.false;
55+
});
56+
57+
it("should reload ruleset on changes to source file", async () => {
58+
const opts = { method: RulesetOperationMethod.GET, file: {}, path: "/b/bucket/o/" };
59+
60+
// Write rules to file
61+
const fileName = "storage.rules";
62+
const testDir = `${tmpdir()}/${uuidv4()}`;
63+
const persistence = new Persistence(testDir);
64+
persistence.appendBytes(fileName, Buffer.from(StorageRulesFiles.readWriteIfTrue.content));
65+
66+
await rulesManager.setSourceFile(`${testDir}/${fileName}`);
67+
expect((await rulesManager.ruleset!.verify(opts)).permitted).to.be.true;
68+
69+
// Write new rules to file
70+
persistence.deleteFile(fileName);
71+
persistence.appendBytes(fileName, Buffer.from(StorageRulesFiles.readWriteIfAuth.content));
72+
73+
await rulesManager.setSourceFile(`${testDir}/${fileName}`);
74+
expect((await rulesManager.ruleset!.verify(opts)).permitted).to.be.false;
75+
});
76+
77+
it("should throw FirebaseError when attempting to set invalid source file", async () => {
78+
const invalidFileName = "foo";
79+
await expect(rulesManager.setSourceFile(invalidFileName)).to.be.rejectedWith(
80+
FirebaseError,
81+
`File not found: ${invalidFileName}`
82+
);
83+
});
84+
85+
it("should delete ruleset when storage manager is closed", async () => {
86+
await rulesManager.setSourceFile(StorageRulesFiles.readWriteIfTrue);
87+
expect(rulesManager.ruleset).not.to.be.undefined;
88+
89+
await rulesManager.close();
90+
expect(rulesManager.ruleset).to.be.undefined;
91+
});
92+
});

0 commit comments

Comments
 (0)