From d50723ec89dbf039ff921e55f5b72aa5d3672723 Mon Sep 17 00:00:00 2001 From: ifielker Date: Thu, 20 Feb 2020 22:40:09 -0500 Subject: [PATCH 1/4] Added UpdateModel, publishModel,and unpublishModel functionality plus tests --- .../machine-learning-api-client.ts | 34 +- src/machine-learning/machine-learning.ts | 66 ++- test/integration/machine-learning.spec.ts | 177 +++++++- .../machine-learning-api-client.spec.ts | 118 ++++++ .../machine-learning/machine-learning.spec.ts | 395 +++++++++++++++++- 5 files changed, 750 insertions(+), 40 deletions(-) diff --git a/src/machine-learning/machine-learning-api-client.ts b/src/machine-learning/machine-learning-api-client.ts index 295014fc01..0d928b8d95 100644 --- a/src/machine-learning/machine-learning-api-client.ts +++ b/src/machine-learning/machine-learning-api-client.ts @@ -31,6 +31,20 @@ export interface StatusErrorResponse { readonly message: string; } +/** + * A Firebase ML Model input object + */ +export class ModelOptions { + public displayName?: string; + public tags?: string[]; + + public tfliteModel?: { gcsTfliteUri: string; }; +} + +export class ModelUpdateOptions extends ModelOptions { + public state?: { published?: boolean; }; +} + export interface ModelContent { readonly displayName?: string; readonly tags?: string[]; @@ -80,7 +94,7 @@ export class MachineLearningApiClient { this.httpClient = new AuthorizedHttpClient(app); } - public createModel(model: ModelContent): Promise { + public createModel(model: ModelOptions): Promise { if (!validator.isNonNullObject(model) || !validator.isNonEmptyString(model.displayName)) { const err = new FirebaseMachineLearningError('invalid-argument', 'Invalid model content.'); @@ -97,6 +111,24 @@ export class MachineLearningApiClient { }); } + public updateModel(modelId: string, model: ModelUpdateOptions, updateMask: string[]): Promise { + if (!validator.isNonEmptyString(modelId) || + !validator.isNonNullObject(model) || + !validator.isNonEmptyArray(updateMask)) { + const err = new FirebaseMachineLearningError('invalid-argument', 'Invalid model or mask content.'); + return Promise.reject(err); + } + return this.getUrl() + .then((url) => { + const request: HttpRequestConfig = { + method: 'PATCH', + url: `${url}/models/${modelId}?updateMask=${updateMask.join()}`, + data: model, + }; + return this.sendRequest(request); + }); + } + public getModel(modelId: string): Promise { return Promise.resolve() diff --git a/src/machine-learning/machine-learning.ts b/src/machine-learning/machine-learning.ts index c9e7b8b515..7f31d77ab6 100644 --- a/src/machine-learning/machine-learning.ts +++ b/src/machine-learning/machine-learning.ts @@ -16,7 +16,8 @@ import {FirebaseApp} from '../firebase-app'; import {FirebaseServiceInterface, FirebaseServiceInternalsInterface} from '../firebase-service'; -import {MachineLearningApiClient, ModelResponse, OperationResponse, ModelContent} from './machine-learning-api-client'; +import {MachineLearningApiClient, ModelResponse, OperationResponse, + ModelOptions, ModelUpdateOptions} from './machine-learning-api-client'; import {FirebaseError} from '../utils/error'; import * as validator from '../utils/validator'; @@ -95,7 +96,7 @@ export class MachineLearning implements FirebaseServiceInterface { * @return {Promise} A Promise fulfilled with the created model. */ public createModel(model: ModelOptions): Promise { - return this.convertOptionstoContent(model, true) + return this.maybeSignUrl(model, true) .then((modelContent) => this.client.createModel(modelContent)) .then((operation) => handleOperation(operation)); } @@ -109,8 +110,13 @@ export class MachineLearning implements FirebaseServiceInterface { * @return {Promise} A Promise fulfilled with the updated model. */ public updateModel(modelId: string, model: ModelOptions): Promise { - throw new Error('NotImplemented'); - } + const updateMask = getMaskFromOptions(model); + return this.maybeSignUrl(model, true) + .then((modelContent) => this.client.updateModel(modelId, modelContent, updateMask)) + .then((operation) => { + return handleOperation(operation); + }); + } /** * Publishes a model in Firebase ML. @@ -120,7 +126,12 @@ export class MachineLearning implements FirebaseServiceInterface { * @return {Promise} A Promise fulfilled with the published model. */ public publishModel(modelId: string): Promise { - throw new Error('NotImplemented'); + const updateMask = ['state.published']; + const options: ModelUpdateOptions = {state: {published: true}}; + return this.client.updateModel(modelId, options, updateMask) + .then((operation) => { + return handleOperation(operation); + }); } /** @@ -131,7 +142,12 @@ export class MachineLearning implements FirebaseServiceInterface { * @return {Promise} A Promise fulfilled with the unpublished model. */ public unpublishModel(modelId: string): Promise { - throw new Error('NotImplemented'); + const updateMask = ['state.published']; + const options: ModelUpdateOptions = {state: {published: false}}; + return this.client.updateModel(modelId, options, updateMask) + .then((operation) => { + return handleOperation(operation); + }); } /** @@ -171,23 +187,23 @@ export class MachineLearning implements FirebaseServiceInterface { return this.client.deleteModel(modelId); } - private convertOptionstoContent(options: ModelOptions, forUpload?: boolean): Promise { - const modelContent = deepCopy(options); + private maybeSignUrl(options: ModelOptions, forUpload?: boolean): Promise { + const modelOptions = deepCopy(options); - if (forUpload && modelContent.tfliteModel?.gcsTfliteUri) { - return this.signUrl(modelContent.tfliteModel.gcsTfliteUri) + if (forUpload && modelOptions.tfliteModel?.gcsTfliteUri) { + return this.signUrl(modelOptions.tfliteModel.gcsTfliteUri) .then ((uri: string) => { - modelContent.tfliteModel!.gcsTfliteUri = uri; - return modelContent; + modelOptions.tfliteModel!.gcsTfliteUri = uri; + return modelOptions; }) .catch((err: Error) => { throw new FirebaseMachineLearningError( 'internal-error', `Error during signing upload url: ${err.message}`); - }) as Promise; + }); } - return Promise.resolve(modelContent) as Promise; + return Promise.resolve(modelOptions); } private signUrl(unsignedUrl: string): Promise { @@ -288,22 +304,26 @@ export interface TFLiteModel { } -/** - * A Firebase ML Model input object - */ -export class ModelOptions { - public displayName?: string; - public tags?: string[]; - public tfliteModel?: { gcsTfliteUri: string; }; -} +function getMaskFromOptions(options: ModelOptions): string[] { + const mask: string[] = []; + if (options.displayName) { + mask.push('display_name'); + } + if (options.tags) { + mask.push('tags'); + } + if (options.tfliteModel) { + mask.push('tflite_model.gcs_tflite_uri'); + } + return mask; +} function extractModelId(resourceName: string): string { return resourceName.split('/').pop()!; } - function handleOperation(op: OperationResponse): Model { // Backend currently does not return operations that are not done. if (op.done) { diff --git a/test/integration/machine-learning.spec.ts b/test/integration/machine-learning.spec.ts index cb56438ce7..b890c60b54 100644 --- a/test/integration/machine-learning.spec.ts +++ b/test/integration/machine-learning.spec.ts @@ -127,6 +127,177 @@ describe('admin.machineLearning', () => { }); }); + describe('updateModel()', () => { + const UPDATE_NAME: admin.machineLearning.ModelOptions = { + displayName: 'update-model-new-name', + }; + it('rejects with not-found when the Model does not exist', () => { + const nonExistingId = '00000000'; + return admin.machineLearning().updateModel(nonExistingId, UPDATE_NAME) + .should.eventually.be.rejected.and.have.property( + 'code', 'machine-learning/not-found'); + }); + + it('rejects with invalid-argument when the ModelId is invalid', () => { + return admin.machineLearning().updateModel('invalid-model-id', UPDATE_NAME) + .should.eventually.be.rejected.and.have.property( + 'code', 'machine-learning/invalid-argument'); + }); + + it ('rejects with invalid-argument when modelOptions are invalid', () => { + const modelOptions: admin.machineLearning.ModelOptions = { + displayName: 'Invalid Name#*^!', + }; + return createTemporaryModel({displayName: 'node-integration-invalid-argument'}) + .then((model) => admin.machineLearning().updateModel(model.modelId, modelOptions) + .should.eventually.be.rejected.and.have.property( + 'code', 'machine-learning/invalid-argument')); + }); + + it('updates the displayName', () => { + const DISPLAY_NAME = 'node-integration-test-update-1b'; + return createTemporaryModel({displayName: 'node-integration-test-update-1a'}) + .then((model) => { + const modelOptions: admin.machineLearning.ModelOptions = { + displayName: DISPLAY_NAME, + }; + return admin.machineLearning().updateModel(model.modelId, modelOptions) + .then((updatedModel) => { + verifyModel(updatedModel, modelOptions); + }); + }); + }); + + + it('sets tags for a model', () => { + // TODO(ifielker): Uncomment & replace when BE change lands. + // const ORIGINAL_TAGS = ['tag-node-update-1']; + const ORIGINAL_TAGS: string[] = []; + const NEW_TAGS = ['tag-node-update-2', 'tag-node-update-3']; + + return createTemporaryModel({ + displayName: 'node-integration-test-update-2', + tags: ORIGINAL_TAGS, + }).then((expectedModel) => { + const modelOptions: admin.machineLearning.ModelOptions = { + tags: NEW_TAGS, + }; + return admin.machineLearning().updateModel(expectedModel.modelId, modelOptions) + .then((actualModel) => { + expect(actualModel.tags!.length).to.equal(2); + expect(actualModel.tags).to.have.same.members(NEW_TAGS); + }); + }); + }); + + it('updates the tflite file', () => { + Promise.all([ + createTemporaryModel(), + uploadModelToGcs('model1.tflite', 'valid_model.tflite')]) + .then(([model, fileName]) => { + const modelOptions: admin.machineLearning.ModelOptions = { + tfliteModel: {gcsTfliteUri: fileName}, + }; + return admin.machineLearning().updateModel(model.modelId, modelOptions) + .then((updatedModel) => { + verifyModel(updatedModel, modelOptions); + }); + }); + }); + + it('can update more than 1 field', () => { + const DISPLAY_NAME = 'node-integration-test-update-3b'; + const TAGS = ['node-integration-tag-1', 'node-integration-tag-2']; + return createTemporaryModel({displayName: 'node-integration-test-update-3a'}) + .then((model) => { + const modelOptions: admin.machineLearning.ModelOptions = { + displayName: DISPLAY_NAME, + tags: TAGS, + }; + return admin.machineLearning().updateModel(model.modelId, modelOptions) + .then((updatedModel) => { + expect(updatedModel.displayName).to.equal(DISPLAY_NAME); + expect(updatedModel.tags).to.have.same.members(TAGS); + }); + }); + }); + }); + + describe('publishModel()', () => { + it('should reject when model does not exist', () => { + const nonExistingName = '00000000'; + return admin.machineLearning().publishModel(nonExistingName) + .should.eventually.be.rejected.and.have.property( + 'code', 'machine-learning/not-found'); + }); + + it('rejects with invalid-argument when the ModelId is invalid', () => { + return admin.machineLearning().publishModel('invalid-model-id') + .should.eventually.be.rejected.and.have.property( + 'code', 'machine-learning/invalid-argument'); + }); + + it('publishes the model successfully', () => { + const modelOptions: admin.machineLearning.ModelOptions = { + displayName: 'node-integration-test-publish-1', + tfliteModel: {gcsTfliteUri: 'this will be replaced below'}, + }; + return uploadModelToGcs('model1.tflite', 'valid_model.tflite') + .then((fileName: string) => { + modelOptions.tfliteModel!.gcsTfliteUri = fileName; + createTemporaryModel(modelOptions) + .then((createdModel) => { + expect(createdModel.validationError).to.be.empty; + expect(createdModel.published).to.be.false; + admin.machineLearning().publishModel(createdModel.modelId) + .then((publishedModel) => { + expect(publishedModel.published).to.be.true; + }); + }); + }); + }); + }); + + describe('unpublishModel()', () => { + it('should reject when model does not exist', () => { + const nonExistingName = '00000000'; + return admin.machineLearning().unpublishModel(nonExistingName) + .should.eventually.be.rejected.and.have.property( + 'code', 'machine-learning/not-found'); + }); + + it('rejects with invalid-argument when the ModelId is invalid', () => { + return admin.machineLearning().unpublishModel('invalid-model-id') + .should.eventually.be.rejected.and.have.property( + 'code', 'machine-learning/invalid-argument'); + }); + + it('unpublishes the model successfully', () => { + const modelOptions: admin.machineLearning.ModelOptions = { + displayName: 'node-integration-test-unpublish-1', + tfliteModel: {gcsTfliteUri: 'this will be replaced below'}, + }; + return uploadModelToGcs('model1.tflite', 'valid_model.tflite') + .then((fileName: string) => { + modelOptions.tfliteModel!.gcsTfliteUri = fileName; + createTemporaryModel(modelOptions) + .then((createdModel) => { + expect(createdModel.validationError).to.be.empty; + expect(createdModel.published).to.be.false; + admin.machineLearning().publishModel(createdModel.modelId) + .then((publishedModel) => { + expect(publishedModel.published).to.be.true; + admin.machineLearning().unpublishModel(publishedModel.modelId) + .then((unpublishedModel) => { + expect(unpublishedModel.published).to.be.false; + }); + }); + }); + }); + }); + }); + + describe('getModel()', () => { it('rejects with not-found when the Model does not exist', () => { const nonExistingName = '00000000'; @@ -181,7 +352,11 @@ describe('admin.machineLearning', () => { }); function verifyModel(model: admin.machineLearning.Model, expectedOptions: admin.machineLearning.ModelOptions) { - expect(model.displayName).to.equal(expectedOptions.displayName); + if (expectedOptions.displayName) { + expect(model.displayName).to.equal(expectedOptions.displayName); + } else { + expect(model.displayName).not.to.be.empty; + } expect(model.createTime).to.not.be.empty; expect(model.updateTime).to.not.be.empty; expect(model.etag).to.not.be.empty; diff --git a/test/unit/machine-learning/machine-learning-api-client.spec.ts b/test/unit/machine-learning/machine-learning-api-client.spec.ts index b4348b5b7c..dc31341db9 100644 --- a/test/unit/machine-learning/machine-learning-api-client.spec.ts +++ b/test/unit/machine-learning/machine-learning-api-client.spec.ts @@ -189,6 +189,124 @@ describe('MachineLearningApiClient', () => { }); }); + describe('updateModel', () => { + const NAME_ONLY_CONTENT: ModelContent = {displayName: 'name1'}; + const NAME_ONLY_MASK = ['display_name']; + const MODEL_RESPONSE = { + name: 'projects/test-project/models/1234567', + createTime: '2020-02-07T23:45:23.288047Z', + updateTime: '2020-02-08T23:45:23.288047Z', + etag: 'etag123', + modelHash: 'modelHash123', + displayName: 'model_1', + tags: ['tag_1', 'tag_2'], + state: {published: true}, + tfliteModel: { + gcsTfliteUri: 'gs://test-project-bucket/Firebase/ML/Models/model1.tflite', + sizeBytes: 16900988, + }, + }; + const STATUS_ERROR_RESPONSE = { + code: 3, + message: 'Invalid Argument message', + }; + const OPERATION_SUCCESS_RESPONSE = { + done: true, + response: MODEL_RESPONSE, + }; + const OPERATION_ERROR_RESPONSE = { + done: true, + error: STATUS_ERROR_RESPONSE, + }; + + const invalidContent: any[] = [null, undefined]; + invalidContent.forEach((content) => { + it(`should reject when called with: ${JSON.stringify(content)}`, () => { + return apiClient.updateModel(MODEL_ID, content, NAME_ONLY_MASK) + .should.eventually.be.rejected.and.have.property( + 'message', 'Invalid model or mask content.'); + }); + }); + + it('should reject when called with empty mask', () => { + return apiClient.updateModel(MODEL_ID, {}, []) + .should.eventually.be.rejected.and.have.property( + 'message', 'Invalid model or mask content.'); + }); + + it('should reject when project id is not available', () => { + return clientWithoutProjectId.updateModel(MODEL_ID, NAME_ONLY_CONTENT, NAME_ONLY_MASK) + .should.eventually.be.rejectedWith(noProjectId); + }); + + it('should throw when an error response is received', () => { + const stub = sinon + .stub(HttpClient.prototype, 'send') + .rejects(utils.errorFrom(ERROR_RESPONSE, 404)); + stubs.push(stub); + const expected = new FirebaseMachineLearningError('not-found', 'Requested entity not found'); + return apiClient.updateModel(MODEL_ID, NAME_ONLY_CONTENT, NAME_ONLY_MASK) + .should.eventually.be.rejected.and.deep.equal(expected); + }); + + it('should resolve with the created resource on success', () => { + const stub = sinon + .stub(HttpClient.prototype, 'send') + .resolves(utils.responseFrom(OPERATION_SUCCESS_RESPONSE)); + stubs.push(stub); + return apiClient.updateModel(MODEL_ID, NAME_ONLY_CONTENT, NAME_ONLY_MASK) + .then((resp) => { + expect(resp.done).to.be.true; + expect(resp.name).to.be.empty; + expect(resp.response).to.deep.equal(MODEL_RESPONSE); + }); + }); + + it('should resolve with error when the operation fails', () => { + const stub = sinon + .stub(HttpClient.prototype, 'send') + .resolves(utils.responseFrom(OPERATION_ERROR_RESPONSE)); + stubs.push(stub); + return apiClient.updateModel(MODEL_ID, NAME_ONLY_CONTENT, NAME_ONLY_MASK) + .then((resp) => { + expect(resp.done).to.be.true; + expect(resp.name).to.be.empty; + expect(resp.error).to.deep.equal(STATUS_ERROR_RESPONSE); + }); + }); + + it('should reject with unknown-error when error code is not present', () => { + const stub = sinon + .stub(HttpClient.prototype, 'send') + .rejects(utils.errorFrom({}, 404)); + stubs.push(stub); + const expected = new FirebaseMachineLearningError('unknown-error', 'Unknown server error: {}'); + return apiClient.updateModel(MODEL_ID, NAME_ONLY_CONTENT, NAME_ONLY_MASK) + .should.eventually.be.rejected.and.deep.equal(expected); + }); + + it('should reject with unknown-error for non-json response', () => { + const stub = sinon + .stub(HttpClient.prototype, 'send') + .rejects(utils.errorFrom('not json', 404)); + stubs.push(stub); + const expected = new FirebaseMachineLearningError( + 'unknown-error', 'Unexpected response with status: 404 and body: not json'); + return apiClient.updateModel(MODEL_ID, NAME_ONLY_CONTENT, NAME_ONLY_MASK) + .should.eventually.be.rejected.and.deep.equal(expected); + }); + + it('should reject with when failed with a FirebaseAppError', () => { + const expected = new FirebaseAppError('network-error', 'socket hang up'); + const stub = sinon + .stub(HttpClient.prototype, 'send') + .rejects(expected); + stubs.push(stub); + return apiClient.updateModel(MODEL_ID, NAME_ONLY_CONTENT, NAME_ONLY_MASK) + .should.eventually.be.rejected.and.deep.equal(expected); + }); + }); + describe('getModel', () => { const INVALID_NAMES: any[] = [null, undefined, '', 1, true, {}, []]; INVALID_NAMES.forEach((invalidName) => { diff --git a/test/unit/machine-learning/machine-learning.spec.ts b/test/unit/machine-learning/machine-learning.spec.ts index beddfd74d9..b77c0e2725 100644 --- a/test/unit/machine-learning/machine-learning.spec.ts +++ b/test/unit/machine-learning/machine-learning.spec.ts @@ -19,11 +19,11 @@ import * as _ from 'lodash'; import * as chai from 'chai'; import * as sinon from 'sinon'; -import { MachineLearning, ModelOptions } from '../../../src/machine-learning/machine-learning'; +import { MachineLearning } from '../../../src/machine-learning/machine-learning'; import { FirebaseApp } from '../../../src/firebase-app'; import * as mocks from '../../resources/mocks'; -import { MachineLearningApiClient, - StatusErrorResponse, ModelResponse } from '../../../src/machine-learning/machine-learning-api-client'; +import { MachineLearningApiClient, StatusErrorResponse, + ModelOptions, ModelResponse } from '../../../src/machine-learning/machine-learning-api-client'; import { FirebaseMachineLearningError } from '../../../src/machine-learning/machine-learning-utils'; import { deepCopy } from '../../../src/utils/deep-copy'; @@ -31,6 +31,7 @@ const expect = chai.expect; describe('MachineLearning', () => { + const MODEL_ID = '1234567'; const EXPECTED_ERROR = new FirebaseMachineLearningError('internal-error', 'message'); const MODEL_RESPONSE: { name: string; @@ -195,7 +196,7 @@ describe('MachineLearning', () => { .stub(MachineLearningApiClient.prototype, 'getModel') .rejects(EXPECTED_ERROR); stubs.push(stub); - return machineLearning.getModel('1234567') + return machineLearning.getModel(MODEL_ID) .should.eventually.be.rejected.and.deep.equal(EXPECTED_ERROR); }); @@ -204,7 +205,7 @@ describe('MachineLearning', () => { .stub(MachineLearningApiClient.prototype, 'getModel') .resolves(null); stubs.push(stub); - return machineLearning.getModel('1234567') + return machineLearning.getModel(MODEL_ID) .should.eventually.be.rejected.and.have.property( 'message', 'Invalid Model response: null'); }); @@ -216,7 +217,7 @@ describe('MachineLearning', () => { .stub(MachineLearningApiClient.prototype, 'getModel') .resolves(response); stubs.push(stub); - return machineLearning.getModel('1234567') + return machineLearning.getModel(MODEL_ID) .should.eventually.be.rejected.and.have.property( 'message', `Invalid Model response: ${JSON.stringify(response)}`); }); @@ -228,7 +229,7 @@ describe('MachineLearning', () => { .stub(MachineLearningApiClient.prototype, 'getModel') .resolves(response); stubs.push(stub); - return machineLearning.getModel('1234567') + return machineLearning.getModel(MODEL_ID) .should.eventually.be.rejected.and.have.property( 'message', `Invalid Model response: ${JSON.stringify(response)}`); }); @@ -240,7 +241,7 @@ describe('MachineLearning', () => { .stub(MachineLearningApiClient.prototype, 'getModel') .resolves(response); stubs.push(stub); - return machineLearning.getModel('1234567') + return machineLearning.getModel(MODEL_ID) .should.eventually.be.rejected.and.have.property( 'message', `Invalid Model response: ${JSON.stringify(response)}`); }); @@ -252,7 +253,7 @@ describe('MachineLearning', () => { .stub(MachineLearningApiClient.prototype, 'getModel') .resolves(response); stubs.push(stub); - return machineLearning.getModel('1234567') + return machineLearning.getModel(MODEL_ID) .should.eventually.be.rejected.and.have.property( 'message', `Invalid Model response: ${JSON.stringify(response)}`); }); @@ -264,7 +265,7 @@ describe('MachineLearning', () => { .stub(MachineLearningApiClient.prototype, 'getModel') .resolves(response); stubs.push(stub); - return machineLearning.getModel('1234567') + return machineLearning.getModel(MODEL_ID) .should.eventually.be.rejected.and.have.property( 'message', `Invalid Model response: ${JSON.stringify(response)}`); }); @@ -275,9 +276,9 @@ describe('MachineLearning', () => { .resolves(MODEL_RESPONSE); stubs.push(stub); - return machineLearning.getModel('1234567') + return machineLearning.getModel(MODEL_ID) .then((model) => { - expect(model.modelId).to.equal('1234567'); + expect(model.modelId).to.equal(MODEL_ID); expect(model.displayName).to.equal('model_1'); expect(model.tags).to.deep.equal(['tag_1', 'tag_2']); expect(model.createTime).to.equal(CREATE_TIME_UTC); @@ -301,7 +302,7 @@ describe('MachineLearning', () => { .stub(MachineLearningApiClient.prototype, 'deleteModel') .rejects(EXPECTED_ERROR); stubs.push(stub); - return machineLearning.deleteModel('1234567') + return machineLearning.deleteModel(MODEL_ID) .should.eventually.be.rejected.and.deep.equal(EXPECTED_ERROR); }); @@ -311,7 +312,7 @@ describe('MachineLearning', () => { .resolves({}); stubs.push(stub); - return machineLearning.deleteModel('1234567'); + return machineLearning.deleteModel(MODEL_ID); }); }); @@ -416,7 +417,7 @@ describe('MachineLearning', () => { return machineLearning.createModel(MODEL_OPTIONS_WITH_GCS) .then((model) => { - expect(model.modelId).to.equal('1234567'); + expect(model.modelId).to.equal(MODEL_ID); expect(model.displayName).to.equal('model_1'); expect(model.tags).to.deep.equal(['tag_1', 'tag_2']); expect(model.createTime).to.equal(CREATE_TIME_UTC); @@ -444,4 +445,368 @@ describe('MachineLearning', () => { 'message', 'Invalid Argument message'); }); }); + + describe('updateModel', () => { + const GCS_TFLITE_URI = 'gs://test-bucket/Firebase/ML/Models/model1.tflite'; + const MODEL_OPTIONS_NO_GCS: ModelOptions = { + displayName: 'display_name', + tags: ['tag1', 'tag2'], + }; + const MODEL_OPTIONS_WITH_GCS: ModelOptions = { + displayName: 'display_name_2', + tags: ['tag3', 'tag4'], + tfliteModel: { + gcsTfliteUri: GCS_TFLITE_URI, + }, + }; + + it('should propagate API errors', () => { + const stub = sinon + .stub(MachineLearningApiClient.prototype, 'updateModel') + .rejects(EXPECTED_ERROR); + stubs.push(stub); + return machineLearning.updateModel(MODEL_ID, MODEL_OPTIONS_NO_GCS) + .should.eventually.be.rejected.and.deep.equal(EXPECTED_ERROR); + }); + + it('should reject when API response is invalid', () => { + const stub = sinon + .stub(MachineLearningApiClient.prototype, 'updateModel') + .resolves(null); + stubs.push(stub); + return machineLearning.updateModel(MODEL_ID, MODEL_OPTIONS_WITH_GCS) + .should.eventually.be.rejected.and.have.property( + 'message', 'Cannot read property \'done\' of null'); + }); + + it('should reject when API response does not contain a name', () => { + const op = deepCopy(OPERATION_RESPONSE); + op.response!.name = ''; + const stub = sinon + .stub(MachineLearningApiClient.prototype, 'updateModel') + .resolves(op); + stubs.push(stub); + return machineLearning.updateModel(MODEL_ID, MODEL_OPTIONS_NO_GCS) + .should.eventually.be.rejected.and.have.property( + 'message', `Invalid Model response: ${JSON.stringify(op.response)}`); + }); + + it('should reject when API response does not contain a createTime', () => { + const op = deepCopy(OPERATION_RESPONSE); + op.response!.createTime = ''; + const stub = sinon + .stub(MachineLearningApiClient.prototype, 'updateModel') + .resolves(op); + stubs.push(stub); + return machineLearning.updateModel(MODEL_ID, MODEL_OPTIONS_NO_GCS) + .should.eventually.be.rejected.and.have.property( + 'message', `Invalid Model response: ${JSON.stringify(op.response)}`); + }); + + it('should reject when API response does not contain a updateTime', () => { + const op = deepCopy(OPERATION_RESPONSE); + op.response!.updateTime = ''; + const stub = sinon + .stub(MachineLearningApiClient.prototype, 'updateModel') + .resolves(op); + stubs.push(stub); + return machineLearning.updateModel(MODEL_ID, MODEL_OPTIONS_NO_GCS) + .should.eventually.be.rejected.and.have.property( + 'message', `Invalid Model response: ${JSON.stringify(op.response)}`); + }); + + it('should reject when API response does not contain a displayName', () => { + const op = deepCopy(OPERATION_RESPONSE); + op.response!.displayName = ''; + const stub = sinon + .stub(MachineLearningApiClient.prototype, 'updateModel') + .resolves(op); + stubs.push(stub); + return machineLearning.updateModel(MODEL_ID, MODEL_OPTIONS_NO_GCS) + .should.eventually.be.rejected.and.have.property( + 'message', `Invalid Model response: ${JSON.stringify(op.response)}`); + }); + + it('should reject when API response does not contain an etag', () => { + const op = deepCopy(OPERATION_RESPONSE); + op.response!.etag = ''; + const stub = sinon + .stub(MachineLearningApiClient.prototype, 'updateModel') + .resolves(op); + stubs.push(stub); + return machineLearning.updateModel(MODEL_ID, MODEL_OPTIONS_NO_GCS) + .should.eventually.be.rejected.and.have.property( + 'message', `Invalid Model response: ${JSON.stringify(op.response)}`); + }); + + it('should resolve with Model on success', () => { + const stub = sinon + .stub(MachineLearningApiClient.prototype, 'updateModel') + .resolves(OPERATION_RESPONSE); + stubs.push(stub); + + return machineLearning.updateModel(MODEL_ID, MODEL_OPTIONS_WITH_GCS) + .then((model) => { + expect(model.modelId).to.equal(MODEL_ID); + expect(model.displayName).to.equal('model_1'); + expect(model.tags).to.deep.equal(['tag_1', 'tag_2']); + expect(model.createTime).to.equal(CREATE_TIME_UTC); + expect(model.updateTime).to.equal(UPDATE_TIME_UTC); + expect(model.validationError).to.be.empty; + expect(model.published).to.be.true; + expect(model.etag).to.equal('etag123'); + expect(model.modelHash).to.equal('modelHash123'); + + const tflite = model.tfliteModel!; + expect(tflite.gcsTfliteUri).to.be.equal( + 'gs://test-project-bucket/Firebase/ML/Models/model1.tflite'); + expect(tflite.sizeBytes).to.be.equal(16900988); + }); + }); + + it('should resolve with Error on operation error', () => { + const stub = sinon + .stub(MachineLearningApiClient.prototype, 'updateModel') + .resolves(OPERATION_RESPONSE_ERROR); + stubs.push(stub); + + return machineLearning.updateModel(MODEL_ID, MODEL_OPTIONS_WITH_GCS) + .should.eventually.be.rejected.and.have.property( + 'message', 'Invalid Argument message'); + }); + }); + + describe('publishModel', () => { + it('should propagate API errors', () => { + const stub = sinon + .stub(MachineLearningApiClient.prototype, 'updateModel') + .rejects(EXPECTED_ERROR); + stubs.push(stub); + return machineLearning.publishModel(MODEL_ID) + .should.eventually.be.rejected.and.deep.equal(EXPECTED_ERROR); + }); + + it('should reject when API response is invalid', () => { + const stub = sinon + .stub(MachineLearningApiClient.prototype, 'updateModel') + .resolves(null); + stubs.push(stub); + return machineLearning.publishModel(MODEL_ID) + .should.eventually.be.rejected.and.have.property( + 'message', 'Cannot read property \'done\' of null'); + }); + + it('should reject when API response does not contain a name', () => { + const op = deepCopy(OPERATION_RESPONSE); + op.response!.name = ''; + const stub = sinon + .stub(MachineLearningApiClient.prototype, 'updateModel') + .resolves(op); + stubs.push(stub); + return machineLearning.publishModel(MODEL_ID) + .should.eventually.be.rejected.and.have.property( + 'message', `Invalid Model response: ${JSON.stringify(op.response)}`); + }); + + it('should reject when API response does not contain a createTime', () => { + const op = deepCopy(OPERATION_RESPONSE); + op.response!.createTime = ''; + const stub = sinon + .stub(MachineLearningApiClient.prototype, 'updateModel') + .resolves(op); + stubs.push(stub); + return machineLearning.publishModel(MODEL_ID) + .should.eventually.be.rejected.and.have.property( + 'message', `Invalid Model response: ${JSON.stringify(op.response)}`); + }); + + it('should reject when API response does not contain a updateTime', () => { + const op = deepCopy(OPERATION_RESPONSE); + op.response!.updateTime = ''; + const stub = sinon + .stub(MachineLearningApiClient.prototype, 'updateModel') + .resolves(op); + stubs.push(stub); + return machineLearning.publishModel(MODEL_ID) + .should.eventually.be.rejected.and.have.property( + 'message', `Invalid Model response: ${JSON.stringify(op.response)}`); + }); + + it('should reject when API response does not contain a displayName', () => { + const op = deepCopy(OPERATION_RESPONSE); + op.response!.displayName = ''; + const stub = sinon + .stub(MachineLearningApiClient.prototype, 'updateModel') + .resolves(op); + stubs.push(stub); + return machineLearning.publishModel(MODEL_ID) + .should.eventually.be.rejected.and.have.property( + 'message', `Invalid Model response: ${JSON.stringify(op.response)}`); + }); + + it('should reject when API response does not contain an etag', () => { + const op = deepCopy(OPERATION_RESPONSE); + op.response!.etag = ''; + const stub = sinon + .stub(MachineLearningApiClient.prototype, 'updateModel') + .resolves(op); + stubs.push(stub); + return machineLearning.publishModel(MODEL_ID) + .should.eventually.be.rejected.and.have.property( + 'message', `Invalid Model response: ${JSON.stringify(op.response)}`); + }); + + it('should resolve with Model on success', () => { + const stub = sinon + .stub(MachineLearningApiClient.prototype, 'updateModel') + .resolves(OPERATION_RESPONSE); + stubs.push(stub); + + return machineLearning.publishModel(MODEL_ID) + .then((model) => { + expect(model.modelId).to.equal(MODEL_ID); + expect(model.displayName).to.equal('model_1'); + expect(model.tags).to.deep.equal(['tag_1', 'tag_2']); + expect(model.createTime).to.equal(CREATE_TIME_UTC); + expect(model.updateTime).to.equal(UPDATE_TIME_UTC); + expect(model.validationError).to.be.empty; + expect(model.published).to.be.true; + expect(model.etag).to.equal('etag123'); + expect(model.modelHash).to.equal('modelHash123'); + + const tflite = model.tfliteModel!; + expect(tflite.gcsTfliteUri).to.be.equal( + 'gs://test-project-bucket/Firebase/ML/Models/model1.tflite'); + expect(tflite.sizeBytes).to.be.equal(16900988); + }); + }); + + it('should resolve with Error on operation error', () => { + const stub = sinon + .stub(MachineLearningApiClient.prototype, 'updateModel') + .resolves(OPERATION_RESPONSE_ERROR); + stubs.push(stub); + + return machineLearning.publishModel(MODEL_ID) + .should.eventually.be.rejected.and.have.property( + 'message', 'Invalid Argument message'); + }); + }); + + describe('unpublishModel', () => { + it('should propagate API errors', () => { + const stub = sinon + .stub(MachineLearningApiClient.prototype, 'updateModel') + .rejects(EXPECTED_ERROR); + stubs.push(stub); + return machineLearning.unpublishModel(MODEL_ID) + .should.eventually.be.rejected.and.deep.equal(EXPECTED_ERROR); + }); + + it('should reject when API response is invalid', () => { + const stub = sinon + .stub(MachineLearningApiClient.prototype, 'updateModel') + .resolves(null); + stubs.push(stub); + return machineLearning.unpublishModel(MODEL_ID) + .should.eventually.be.rejected.and.have.property( + 'message', 'Cannot read property \'done\' of null'); + }); + + it('should reject when API response does not contain a name', () => { + const op = deepCopy(OPERATION_RESPONSE); + op.response!.name = ''; + const stub = sinon + .stub(MachineLearningApiClient.prototype, 'updateModel') + .resolves(op); + stubs.push(stub); + return machineLearning.unpublishModel(MODEL_ID) + .should.eventually.be.rejected.and.have.property( + 'message', `Invalid Model response: ${JSON.stringify(op.response)}`); + }); + + it('should reject when API response does not contain a createTime', () => { + const op = deepCopy(OPERATION_RESPONSE); + op.response!.createTime = ''; + const stub = sinon + .stub(MachineLearningApiClient.prototype, 'updateModel') + .resolves(op); + stubs.push(stub); + return machineLearning.unpublishModel(MODEL_ID) + .should.eventually.be.rejected.and.have.property( + 'message', `Invalid Model response: ${JSON.stringify(op.response)}`); + }); + + it('should reject when API response does not contain a updateTime', () => { + const op = deepCopy(OPERATION_RESPONSE); + op.response!.updateTime = ''; + const stub = sinon + .stub(MachineLearningApiClient.prototype, 'updateModel') + .resolves(op); + stubs.push(stub); + return machineLearning.unpublishModel(MODEL_ID) + .should.eventually.be.rejected.and.have.property( + 'message', `Invalid Model response: ${JSON.stringify(op.response)}`); + }); + + it('should reject when API response does not contain a displayName', () => { + const op = deepCopy(OPERATION_RESPONSE); + op.response!.displayName = ''; + const stub = sinon + .stub(MachineLearningApiClient.prototype, 'updateModel') + .resolves(op); + stubs.push(stub); + return machineLearning.unpublishModel(MODEL_ID) + .should.eventually.be.rejected.and.have.property( + 'message', `Invalid Model response: ${JSON.stringify(op.response)}`); + }); + + it('should reject when API response does not contain an etag', () => { + const op = deepCopy(OPERATION_RESPONSE); + op.response!.etag = ''; + const stub = sinon + .stub(MachineLearningApiClient.prototype, 'updateModel') + .resolves(op); + stubs.push(stub); + return machineLearning.unpublishModel(MODEL_ID) + .should.eventually.be.rejected.and.have.property( + 'message', `Invalid Model response: ${JSON.stringify(op.response)}`); + }); + + it('should resolve with Model on success', () => { + const stub = sinon + .stub(MachineLearningApiClient.prototype, 'updateModel') + .resolves(OPERATION_RESPONSE); + stubs.push(stub); + + return machineLearning.unpublishModel(MODEL_ID) + .then((model) => { + expect(model.modelId).to.equal(MODEL_ID); + expect(model.displayName).to.equal('model_1'); + expect(model.tags).to.deep.equal(['tag_1', 'tag_2']); + expect(model.createTime).to.equal(CREATE_TIME_UTC); + expect(model.updateTime).to.equal(UPDATE_TIME_UTC); + expect(model.validationError).to.be.empty; + expect(model.published).to.be.true; + expect(model.etag).to.equal('etag123'); + expect(model.modelHash).to.equal('modelHash123'); + + const tflite = model.tfliteModel!; + expect(tflite.gcsTfliteUri).to.be.equal( + 'gs://test-project-bucket/Firebase/ML/Models/model1.tflite'); + expect(tflite.sizeBytes).to.be.equal(16900988); + }); + }); + + it('should resolve with Error on operation error', () => { + const stub = sinon + .stub(MachineLearningApiClient.prototype, 'updateModel') + .resolves(OPERATION_RESPONSE_ERROR); + stubs.push(stub); + + return machineLearning.unpublishModel(MODEL_ID) + .should.eventually.be.rejected.and.have.property( + 'message', 'Invalid Argument message'); + }); + }); }); From e5c2d454082426872e274921f23ec81dc64874a1 Mon Sep 17 00:00:00 2001 From: ifielker Date: Mon, 24 Feb 2020 10:50:08 -0500 Subject: [PATCH 2/4] review comments --- .../machine-learning-api-client.ts | 12 ++--- src/machine-learning/machine-learning.ts | 46 +++++++------------ test/integration/machine-learning.spec.ts | 3 +- 3 files changed, 24 insertions(+), 37 deletions(-) diff --git a/src/machine-learning/machine-learning-api-client.ts b/src/machine-learning/machine-learning-api-client.ts index 0d928b8d95..b8ad3d9a11 100644 --- a/src/machine-learning/machine-learning-api-client.ts +++ b/src/machine-learning/machine-learning-api-client.ts @@ -34,15 +34,15 @@ export interface StatusErrorResponse { /** * A Firebase ML Model input object */ -export class ModelOptions { - public displayName?: string; - public tags?: string[]; +export interface ModelOptions { + displayName?: string; + tags?: string[]; - public tfliteModel?: { gcsTfliteUri: string; }; + tfliteModel?: { gcsTfliteUri: string; }; } -export class ModelUpdateOptions extends ModelOptions { - public state?: { published?: boolean; }; +export interface ModelUpdateOptions extends ModelOptions { + state?: { published?: boolean; }; } export interface ModelContent { diff --git a/src/machine-learning/machine-learning.ts b/src/machine-learning/machine-learning.ts index 7f31d77ab6..9d37bed181 100644 --- a/src/machine-learning/machine-learning.ts +++ b/src/machine-learning/machine-learning.ts @@ -96,7 +96,7 @@ export class MachineLearning implements FirebaseServiceInterface { * @return {Promise} A Promise fulfilled with the created model. */ public createModel(model: ModelOptions): Promise { - return this.maybeSignUrl(model, true) + return this.signUrlIfPresent(model) .then((modelContent) => this.client.createModel(modelContent)) .then((operation) => handleOperation(operation)); } @@ -111,11 +111,9 @@ export class MachineLearning implements FirebaseServiceInterface { */ public updateModel(modelId: string, model: ModelOptions): Promise { const updateMask = getMaskFromOptions(model); - return this.maybeSignUrl(model, true) + return this.signUrlIfPresent(model) .then((modelContent) => this.client.updateModel(modelId, modelContent, updateMask)) - .then((operation) => { - return handleOperation(operation); - }); + .then((operation) => handleOperation(operation)); } /** @@ -126,12 +124,7 @@ export class MachineLearning implements FirebaseServiceInterface { * @return {Promise} A Promise fulfilled with the published model. */ public publishModel(modelId: string): Promise { - const updateMask = ['state.published']; - const options: ModelUpdateOptions = {state: {published: true}}; - return this.client.updateModel(modelId, options, updateMask) - .then((operation) => { - return handleOperation(operation); - }); + return this.setPublishStatus(modelId, true); } /** @@ -142,12 +135,7 @@ export class MachineLearning implements FirebaseServiceInterface { * @return {Promise} A Promise fulfilled with the unpublished model. */ public unpublishModel(modelId: string): Promise { - const updateMask = ['state.published']; - const options: ModelUpdateOptions = {state: {published: false}}; - return this.client.updateModel(modelId, options, updateMask) - .then((operation) => { - return handleOperation(operation); - }); + return this.setPublishStatus(modelId, false); } /** @@ -159,9 +147,7 @@ export class MachineLearning implements FirebaseServiceInterface { */ public getModel(modelId: string): Promise { return this.client.getModel(modelId) - .then((modelResponse) => { - return new Model(modelResponse); - }); + .then((modelResponse) => new Model(modelResponse)); } /** @@ -187,10 +173,16 @@ export class MachineLearning implements FirebaseServiceInterface { return this.client.deleteModel(modelId); } - private maybeSignUrl(options: ModelOptions, forUpload?: boolean): Promise { - const modelOptions = deepCopy(options); + private setPublishStatus(modelId: string, publish: boolean): Promise { + const updateMask = ['state.published']; + const options: ModelUpdateOptions = {state: {published: publish}}; + return this.client.updateModel(modelId, options, updateMask) + .then((operation) => handleOperation(operation)); + } - if (forUpload && modelOptions.tfliteModel?.gcsTfliteUri) { + private signUrlIfPresent(options: ModelOptions): Promise { + const modelOptions = deepCopy(options); + if (modelOptions.tfliteModel?.gcsTfliteUri) { return this.signUrl(modelOptions.tfliteModel.gcsTfliteUri) .then ((uri: string) => { modelOptions.tfliteModel!.gcsTfliteUri = uri; @@ -202,7 +194,6 @@ export class MachineLearning implements FirebaseServiceInterface { `Error during signing upload url: ${err.message}`); }); } - return Promise.resolve(modelOptions); } @@ -224,9 +215,7 @@ export class MachineLearning implements FirebaseServiceInterface { return blob.getSignedUrl({ action: 'read', expires: Date.now() + URL_VALID_DURATION, - }).then((x) => { - return x[0]; - }); + }).then((x) => x[0]); } } @@ -303,9 +292,6 @@ export interface TFLiteModel { readonly gcsTfliteUri: string; } - - - function getMaskFromOptions(options: ModelOptions): string[] { const mask: string[] = []; if (options.displayName) { diff --git a/test/integration/machine-learning.spec.ts b/test/integration/machine-learning.spec.ts index b890c60b54..a70e726893 100644 --- a/test/integration/machine-learning.spec.ts +++ b/test/integration/machine-learning.spec.ts @@ -128,9 +128,11 @@ describe('admin.machineLearning', () => { }); describe('updateModel()', () => { + const UPDATE_NAME: admin.machineLearning.ModelOptions = { displayName: 'update-model-new-name', }; + it('rejects with not-found when the Model does not exist', () => { const nonExistingId = '00000000'; return admin.machineLearning().updateModel(nonExistingId, UPDATE_NAME) @@ -168,7 +170,6 @@ describe('admin.machineLearning', () => { }); }); - it('sets tags for a model', () => { // TODO(ifielker): Uncomment & replace when BE change lands. // const ORIGINAL_TAGS = ['tag-node-update-1']; From 9404a941744be1492999637f170765fc0528443b Mon Sep 17 00:00:00 2001 From: ifielker Date: Mon, 24 Feb 2020 13:56:31 -0500 Subject: [PATCH 3/4] using utils.generateUpdateMask --- src/machine-learning/machine-learning.ts | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/src/machine-learning/machine-learning.ts b/src/machine-learning/machine-learning.ts index 9d37bed181..6cb9929427 100644 --- a/src/machine-learning/machine-learning.ts +++ b/src/machine-learning/machine-learning.ts @@ -23,6 +23,7 @@ import {FirebaseError} from '../utils/error'; import * as validator from '../utils/validator'; import {FirebaseMachineLearningError} from './machine-learning-utils'; import { deepCopy } from '../utils/deep-copy'; +import * as utils from '../utils'; /** * Internals of an ML instance. @@ -110,7 +111,7 @@ export class MachineLearning implements FirebaseServiceInterface { * @return {Promise} A Promise fulfilled with the updated model. */ public updateModel(modelId: string, model: ModelOptions): Promise { - const updateMask = getMaskFromOptions(model); + const updateMask = utils.generateUpdateMask(model); return this.signUrlIfPresent(model) .then((modelContent) => this.client.updateModel(modelId, modelContent, updateMask)) .then((operation) => handleOperation(operation)); @@ -292,20 +293,6 @@ export interface TFLiteModel { readonly gcsTfliteUri: string; } -function getMaskFromOptions(options: ModelOptions): string[] { - const mask: string[] = []; - if (options.displayName) { - mask.push('display_name'); - } - if (options.tags) { - mask.push('tags'); - } - if (options.tfliteModel) { - mask.push('tflite_model.gcs_tflite_uri'); - } - return mask; -} - function extractModelId(resourceName: string): string { return resourceName.split('/').pop()!; } From 9f690abd9b204318e0d01c365e1e783a410529e7 Mon Sep 17 00:00:00 2001 From: ifielker Date: Mon, 24 Feb 2020 21:59:05 -0500 Subject: [PATCH 4/4] more review comments --- src/machine-learning/machine-learning.ts | 2 +- .../machine-learning-api-client.spec.ts | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/machine-learning/machine-learning.ts b/src/machine-learning/machine-learning.ts index 6cb9929427..3baa9c50a1 100644 --- a/src/machine-learning/machine-learning.ts +++ b/src/machine-learning/machine-learning.ts @@ -216,7 +216,7 @@ export class MachineLearning implements FirebaseServiceInterface { return blob.getSignedUrl({ action: 'read', expires: Date.now() + URL_VALID_DURATION, - }).then((x) => x[0]); + }).then((signUrl) => signUrl[0]); } } diff --git a/test/unit/machine-learning/machine-learning-api-client.spec.ts b/test/unit/machine-learning/machine-learning-api-client.spec.ts index dc31341db9..64882b42a2 100644 --- a/test/unit/machine-learning/machine-learning-api-client.spec.ts +++ b/test/unit/machine-learning/machine-learning-api-client.spec.ts @@ -191,7 +191,7 @@ describe('MachineLearningApiClient', () => { describe('updateModel', () => { const NAME_ONLY_CONTENT: ModelContent = {displayName: 'name1'}; - const NAME_ONLY_MASK = ['display_name']; + const NAME_ONLY_MASK = ['displayName']; const MODEL_RESPONSE = { name: 'projects/test-project/models/1234567', createTime: '2020-02-07T23:45:23.288047Z', @@ -249,7 +249,7 @@ describe('MachineLearningApiClient', () => { .should.eventually.be.rejected.and.deep.equal(expected); }); - it('should resolve with the created resource on success', () => { + it('should resolve with the updated resource on success', () => { const stub = sinon .stub(HttpClient.prototype, 'send') .resolves(utils.responseFrom(OPERATION_SUCCESS_RESPONSE)); @@ -259,6 +259,12 @@ describe('MachineLearningApiClient', () => { expect(resp.done).to.be.true; expect(resp.name).to.be.empty; expect(resp.response).to.deep.equal(MODEL_RESPONSE); + expect(stub).to.have.been.calledOnce.and.calledWith({ + method: 'PATCH', + headers: EXPECTED_HEADERS, + url: `https://mlkit.googleapis.com/v1beta1/projects/test-project/models/${MODEL_ID}?updateMask=displayName`, + data: NAME_ONLY_CONTENT, + }); }); });