Skip to content

Commit 466bb29

Browse files
flovilmartflovilmart
authored
Merge pull request #2 from JeremyPlease/master
Parse hosted files migration tools
2 parents cd2b3d7 + 1b1abda commit 466bb29

File tree

8 files changed

+516
-50
lines changed

8 files changed

+516
-50
lines changed

‎.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -31,3 +31,5 @@ node_modules
3131

3232
# Optional REPL history
3333
.node_repl_history
34+
35+
.DS_Store

‎README.md

+41-10
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,51 @@
11
# parse-files-utils
2-
Utilities to list and migrate Parse files
2+
Utilities to list and migrate Parse files.
33

4-
This utility will print in the terminal all the files URL's from the parse server
4+
This utility will do the following:
55

6-
This can be really useful when you migrate your files and want to move the files from the Parse S3 host to you own.
6+
1. Get all files across all classess in a Parse database.
7+
2. Print file URLs to console OR transfer to S3, GCS, or filesystem.
8+
3. Rename files so that [Parse Server](https://github.com/ParsePlatform/parse-server) no longer detects that they are hosted by Parse.
9+
4. Update MongoDB with new file names.
710

8-
This utility won't save the files anywhere else. You can save the results to a file or pipe the results to another program:
11+
#### \*WARNING\*
12+
As soon as this script transfers files away from Parse.com hosted files (and renames them in the database)
13+
any clients that use api.parse.com will no longer be able to access the files.
14+
See the section titled "5. Files" in the [Parse Migration Guide](https://parse.com/migration)
15+
and Parse Server [issue #1582](https://github.com/ParsePlatform/parse-server/issues/1582).
916

10-
## usage
17+
## Installation
1118

19+
1. Clone the repo: `git clone [email protected]:parse-server-modules/parse-files-utils.git`
20+
2. cd into repo: `cd parse-file-utils`
21+
3. Install dependencies: `npm install`
22+
23+
## Usage
24+
25+
The quickest way to get started is to run `npm start` and follow the command prompts.
26+
27+
You can optionally specify a js/json configuration file (see [config.example.js](./config.example.js)).
1228
```
13-
$ node index.js MY_APP_ID MY_MASTER_KEY
29+
$ npm start config.js
1430
```
1531

16-
you can optionally specify a server URL
32+
### Available configuration options
1733

18-
```
19-
$ node index.js MY_APP_ID MY_MASTER_KEY MY_SERVER_URL
20-
```
34+
* `applicationId`: Parse application id.
35+
* `masterKey`: Parse master key.
36+
* `mongoURL`: MongoDB connection url.
37+
* `serverURL`: The URL for the Parse server (default: http://api.parse.com/1).
38+
* `filesToTransfer`: Which files to transfer. Accepted options: `parseOnly`, `parseServerOnly`, `all`.
39+
* `renameInDatabase` (boolean): Whether or not to rename files in MongoDB.
40+
* `filesAdapter`: A Parse Server file adapter with a function for `createFile(filename, data)`
41+
(ie. [parse-server-fs-adapter](https://github.com/parse-server-modules/parse-server-fs-adapter),
42+
[parse-server-s3-adapter](https://github.com/parse-server-modules/parse-server-s3-adapter),
43+
[parse-server-gcs-adapter](https://github.com/parse-server-modules/parse-server-gcs-adapter)).
44+
* `filesystemPath`: The path/directory to save files to when transfering to filesystem.
45+
* `aws_accessKeyId`: AWS access key id.
46+
* `aws_secretAccessKey`: AWS secret access key.
47+
* `aws_bucket`: S3 bucket name.
48+
* `gcs_projectId`: GCS project id.
49+
* `gcs_keyFilename`: GCS key filename (ie. `credentials.json`).
50+
* `gcs_bucket`: GCS bucket name.
51+
* `asyncLimit`: The number of files to process at the same time (default: 5).

‎config.example.js

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
var FileAdapter = require('parse-server-fs-adapter');
2+
var S3Adapter = require('parse-server-s3-adapter');
3+
var GCSAdapter = require('parse-server-gcs-adapter');
4+
5+
module.exports = {
6+
applicationId: "PARSE_APPLICATION_ID",
7+
masterKey: "PARSE_MASTER_KEY",
8+
mongoURL: "mongodb://<username>:<password>@mongourl.com:27017/database_name",
9+
serverURL: "https://api.customparseserver.com/parse",
10+
filesToTransfer: 'parseOnly',
11+
renameInDatabase: false,
12+
13+
// For filesystem configuration
14+
filesystemPath: './downloaded_files',
15+
16+
// For S3 configuration
17+
aws_accessKeyId: "ACCESS_KEY_ID",
18+
aws_secretAccessKey: "SECRET_ACCESS_KEY",
19+
aws_bucket: "BUCKET_NAME",
20+
21+
// For GCS configuration
22+
gcs_projectId: "GCS_PROJECT_ID",
23+
gcs_keyFilename: "credentials.json",
24+
gcs_bucket: "BUCKET_NAME",
25+
26+
// Or set filesAdapter to a Parse Server file adapter
27+
// filesAdapter: new FileAdapter({
28+
// filesSubDirectory: './downloaded_files'
29+
// }),
30+
// filesAdapter: new S3Adapter({
31+
// accessKey: 'ACCESS_KEY_ID',
32+
// secretKey: 'SECRET_ACCESS_KEY',
33+
// bucket: 'BUCKET_NAME'
34+
// }),
35+
// filesAdapter: new GCSAdapter({
36+
// projectId: "GCS_PROJECT_ID",
37+
// keyFilename: "credentials.json",
38+
// bucket: "BUCKET_NAME",
39+
// }),
40+
};

‎index.js

+13-7
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,16 @@
1-
var appID = process.argv[2];
2-
var masterKey = process.argv[3];
3-
var serverURL = process.argv[4];
1+
var path = require('path');
2+
var configFilePath = process.argv[2];
3+
var config = {};
44

5-
if (!appID || !masterKey) {
6-
process.stderr.write('An appId and a masterKey are required\n');
7-
process.exit(1);
5+
if (configFilePath) {
6+
configFilePath = path.resolve(configFilePath);
7+
8+
try {
9+
config = require(configFilePath);
10+
} catch(e) {
11+
console.log('Cannot load '+configFilePath);
12+
process.exit(1);
13+
}
814
}
915

10-
var utils = require('./lib')(appID, masterKey, serverURL);
16+
var utils = require('./lib')(config);

‎lib/index.js

+74-33
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,56 @@
11
var Parse = require('parse/node');
2+
var inquirer = require('inquirer');
3+
24
var schemas = require('./schemas');
5+
var transfer = require('./transfer');
6+
var questions = require('./questions.js');
7+
8+
module.exports = initialize;
9+
10+
function initialize(config) {
11+
questions(config).then(function (answers) {
12+
config = Object.assign(config, answers);
13+
console.log(JSON.stringify(config, null, 2));
14+
return inquirer.prompt({
15+
type: 'confirm',
16+
name: 'next',
17+
message: 'About to start the file transfer. Does the above look correct?',
18+
default: true,
19+
});
20+
}).then(function(answers) {
21+
if (!answers.next) {
22+
console.log('Aborted!');
23+
process.exit();
24+
}
25+
Parse.initialize(config.applicationId, null, config.masterKey);
26+
Parse.serverURL = config.serverURL;
27+
return transfer.init(config);
28+
}).then(function() {
29+
return getAllFileObjects();
30+
}).then(function(objects) {
31+
return transfer.run(objects);
32+
}).then(function() {
33+
console.log('Complete!');
34+
process.exit();
35+
}).catch(function(error) {
36+
console.log(error);
37+
process.exit(1);
38+
});
39+
}
40+
41+
function getAllFileObjects() {
42+
console.log("Fetching schema...");
43+
return schemas.get().then(function(res){
44+
console.log("Fetching all objects with files...");
45+
var schemasWithFiles = onlyFiles(res);
46+
return Promise.all(schemasWithFiles.map(getObjectsWithFilesFromSchema));
47+
}).then(function(results) {
48+
var files = results.reduce(function(c, r) {
49+
return c.concat(r);
50+
}, []);
51+
return Promise.resolve(files);
52+
});
53+
}
354

455
function onlyFiles(schemas) {
556
return schemas.map(function(schema) {
@@ -18,53 +69,43 @@ function onlyFiles(schemas) {
1869

1970
function getAllObjects(baseQuery) {
2071
var allObjects = [];
21-
var next = function(startIndex) {
22-
baseQuery.skip(startIndex);
72+
var next = function() {
73+
if (allObjects.length) {
74+
baseQuery.greaterThan('createdAt', allObjects[allObjects.length-1].createdAt);
75+
}
2376
return baseQuery.find({useMasterKey: true}).then(function(r){
2477
allObjects = allObjects.concat(r);
2578
if (r.length == 0) {
2679
return Promise.resolve(allObjects);
2780
} else {
28-
return next(startIndex+r.length);
81+
return next();
2982
}
3083
});
3184
}
32-
return next(0);
85+
return next();
3386
}
3487

35-
function getFilesFromSchema(schema) {
88+
function getObjectsWithFilesFromSchema(schema) {
3689
var query = new Parse.Query(schema.className);
37-
query.select(schema.fields);
90+
query.select(schema.fields.concat('createdAt'));
91+
query.ascending('createdAt');
92+
query.limit(1000);
3893
schema.fields.forEach(function(field) {
3994
query.exists(field);
40-
})
95+
});
4196
return getAllObjects(query).then(function(results) {
4297
return results.reduce(function(current, result){
43-
return current.concat(schema.fields.map(function(field){
44-
return result.get(field).url();
45-
}))
98+
return current.concat(
99+
schema.fields.map(function(field){
100+
return {
101+
className: schema.className,
102+
objectId: result.id,
103+
fieldName: field,
104+
fileName: result.get(field).name(),
105+
url: result.get(field).url()
106+
}
107+
})
108+
);
46109
}, []);
47110
});
48-
}
49-
50-
module.exports = function(applicationId, masterKey, serverURL) {
51-
Parse.initialize(applicationId, null, masterKey);
52-
Parse.serverURL = serverURL || "https://api.parse.com/1";
53-
schemas.get().then(function(res){
54-
var schemasWithFiles = onlyFiles(res);
55-
return Promise.all(schemasWithFiles.map(getFilesFromSchema));
56-
}).then(function(results) {
57-
var files = results.reduce(function(c, r) {
58-
return c.concat(r);
59-
}, []);
60-
files.forEach(function(file) {
61-
process.stdout.write(file);
62-
process.stdout.write("\n");
63-
});
64-
process.exit(0);
65-
}).catch(function(err){
66-
process.stderr.write(err);
67-
process.stderr.write("\n");
68-
process.exit(1);
69-
})
70-
}
111+
}

0 commit comments

Comments
 (0)