Merge pull request #2 from justinmchase/feature/auto-create-bucket

Create bucket in default region if it doesnt exist

Author: Jacob Heun

README.md

@@ -26,14 +26,15 @@ $ npm install datastore-s3
 ```
 
 ## Usage
-A bucket must be created prior to using datastore-s3. Please see the AWS docs for information on how to configure the S3 instance. A bucket name is required to be set at the s3 instance level, see the below example.
+If the flag `createIfMissing` is not set or is false, then the bucket must be created prior to using datastore-s3. Please see the AWS docs for information on how to configure the S3 instance. A bucket name is required to be set at the s3 instance level, see the below example.
 
 ```js
 const S3 = require('aws-sdk').S3
 const s3Instance = new S3({ params: { Bucket: 'my-ipfs-bucket' } })
 const S3Store = require('datastore-s3')
 const store = new S3Store('.ipfs/datastore', {
   s3: s3Instance
+  createIfMissing: false
 })
 ```
 

src/index.js

@@ -2,6 +2,7 @@
 'use strict'
 
 /* :: import type {Batch, Query, QueryResult, Callback} from 'interface-datastore' */
+const assert = require('assert')
 const path = require('path')
 const setImmediate = require('async/setImmediate')
 const each = require('async/each')
@@ -14,7 +15,8 @@ const Deferred = require('pull-defer')
 const pull = require('pull-stream')
 
 /* :: export type S3DSInputOptions = {
-  s3: S3Instance
+  s3: S3Instance,
+  createIfMissing: ?boolean
 }
 
 declare type S3Instance = {
@@ -42,20 +44,26 @@ class S3Datastore {
   /* :: path: string */
   /* :: opts: S3DSInputOptions */
   /* :: bucket: string */
+  /* :: createIfMissing: boolean */
 
   constructor (path /* : string */, opts /* : S3DSInputOptions */) {
     this.path = path
     this.opts = opts
-
-    try {
-      if (typeof this.opts.s3.config.params.Bucket !== 'string') {
-        throw new Error()
-      }
-    } catch (err) {
-      throw new Error('An S3 instance with a predefined Bucket must be supplied. See the datastore-s3 README for examples')
-    }
-
-    this.bucket = this.opts.s3.config.params.Bucket
+    const {
+      createIfMissing = false,
+      s3: {
+        config: {
+          params: {
+            Bucket
+          } = {}
+        } = {}
+      } = {}
+    } = opts
+
+    assert(typeof Bucket === 'string', 'An S3 instance with a predefined Bucket must be supplied. See the datastore-s3 README for examples.')
+    assert(typeof createIfMissing === 'boolean', `createIfMissing must be a boolean but was (${typeof createIfMissing}) ${createIfMissing}`)
+    this.bucket = Bucket
+    this.createIfMissing = createIfMissing
   }
 
   /**
@@ -81,7 +89,14 @@ class S3Datastore {
       Key: this._getFullKey(key),
       Body: val
     }, (err, data) => {
-      callback(err)
+      if (err && err.code === 'NoSuchBucket' && this.createIfMissing) {
+        this.opts.s3.createBucket({}, (err) => {
+          if (err) return callback(err)
+          setImmediate(() => this.put(key, val, callback))
+        })
+      } else {
+        callback(err)
+      }
     })
   }
 

test/index.spec.js

@@ -22,6 +22,16 @@ describe('S3Datastore', () => {
         () => new S3Store('.ipfs/datastore', { s3 })
       ).to.throw()
     })
+    it('createIfMissing defaults to false', () => {
+      const s3 = new S3({ params: { Bucket: 'test' } })
+      const store = new S3Store('.ipfs', { s3 })
+      expect(store.createIfMissing).to.equal(false)
+    })
+    it('createIfMissing can be set to true', () => {
+      const s3 = new S3({ params: { Bucket: 'test' } })
+      const store = new S3Store('.ipfs', { s3, createIfMissing: true })
+      expect(store.createIfMissing).to.equal(true)
+    })
   })
 
   describe('put', () => {
@@ -37,6 +47,54 @@ describe('S3Datastore', () => {
 
       store.put(new Key('/z/key'), Buffer.from('test data'), done)
     })
+    it('should create the bucket when missing if createIfMissing is true', (done) => {
+      const s3 = new S3({ params: { Bucket: 'my-ipfs-bucket' } })
+      const store = new S3Store('.ipfs/datastore', { s3, createIfMissing: true })
+
+      // 1. On the first call upload will fail with a NoSuchBucket error
+      // 2. This should result in the `createBucket` standin being called
+      // 3. upload is then called a second time and it passes
+
+      let bucketCreated = false
+      standin.replace(s3, 'upload', (stand, params, callback) => {
+        if (!bucketCreated) return callback({ code: 'NoSuchBucket' })
+        stand.restore()
+        return callback(null)
+      })
+
+      standin.replace(s3, 'createBucket', (stand, params, callback) => {
+        bucketCreated = true
+        stand.restore()
+        return callback()
+      })
+
+      store.put(new Key('/z/key'), Buffer.from('test data'), done)
+    })
+    it('should create the bucket when missing if createIfMissing is true', (done) => {
+      const s3 = new S3({ params: { Bucket: 'my-ipfs-bucket' } })
+      const store = new S3Store('.ipfs/datastore', { s3, createIfMissing: false })
+
+      let bucketCreated = false
+      standin.replace(s3, 'upload', (stand, params, callback) => {
+        if (!bucketCreated) return callback({ code: 'NoSuchBucket' })
+        stand.restore()
+        return callback(null)
+      })
+
+      standin.replace(s3, 'createBucket', (stand, params, callback) => {
+        bucketCreated = true
+        stand.restore()
+        return callback()
+      })
+
+      store.put(new Key('/z/key'), Buffer.from('test data'), (err) => {
+        expect(bucketCreated).to.equal(false)
+        expect(err).to.deep.equal({
+          code: 'NoSuchBucket'
+        })
+        done()
+      })
+    })
   })
 
   describe('get', () => {