diff --git a/.gitignore b/.gitignore index 57aa6debf..b62f417f0 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,8 @@ *.lo *.Makefile *.target.gyp.mk +stage +lib/binding build out Release diff --git a/.travis.yml b/.travis.yml index c248402f5..0e04fc2dd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,17 @@ language: node_js + node_js: - - "0.10" - - "0.8" - - "0.6" -before_script: "npm install mocha" + - "0.10" + - "0.8" + - "0.6" + +install: + - npm install + - npm install mocha + - npm test + +before_script: "make clean" + +script: + - npm install --stage + - npm test diff --git a/Makefile b/Makefile index 7924cddbd..c986b98ec 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,11 @@ build: - node-gyp build + npm install --build-from-source clean: - rm -f test/support/big.db* - rm -f test/tmp/* + rm -f ./lib/node_sqlite3.node + rm -rf ./lib/binding/ + #rm -f ./test/support/big.db* + rm -f ./test/tmp/* rm -rf ./deps/sqlite-autoconf-*/ rm -rf ./build rm -rf ./out diff --git a/binding.gyp b/binding.gyp index 12a37049e..4a4547e32 100644 --- a/binding.gyp +++ b/binding.gyp @@ -3,17 +3,6 @@ 'variables': { 'sqlite%':'internal', }, - 'conditions': [ - ['OS=="win"', { - 'variables': { - 'copy_command%': 'copy', - }, - },{ - 'variables': { - 'copy_command%': 'cp', - }, - }] - ], 'targets': [ { 'target_name': 'node_sqlite3', @@ -37,23 +26,6 @@ 'src/node_sqlite3.cc', 'src/statement.cc' ], - }, - { - 'target_name': 'action_after_build', - 'type': 'none', - 'dependencies': [ 'node_sqlite3' ], - 'actions': [ - { - 'action_name': 'move_node_module', - 'inputs': [ - '<@(PRODUCT_DIR)/node_sqlite3.node' - ], - 'outputs': [ - 'lib/node_sqlite3.node' - ], - 'action': ['<@(copy_command)', '<@(PRODUCT_DIR)/node_sqlite3.node', 'lib/node_sqlite3.node'] - } - ] } ] } diff --git a/build-util/remake.sh b/build-util/remake.sh new file mode 100755 index 000000000..2ff4e66b9 --- /dev/null +++ b/build-util/remake.sh @@ -0,0 +1,34 @@ +export ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +export UNAME=$(uname -s); + +cd $ROOTDIR +cd ../ + +if [ ${UNAME} = 'Darwin' ]; then + # note: requires FAT (duel-arch) node installed via .pkg + npm install --stage --target_arch=ia32 + npm install --stage --target_arch=ia32 --debug + npm install --stage --target_arch=x64 + npm install --stage --target_arch=x64 --debug + +elif [ ${UNAME} = 'Linux' ]; then + rm -rf ./bin/linux-* + apt-get -y update + apt-get -y install git make build-essential + git clone https://github.com/creationix/nvm.git ~/.nvm + source ~/.nvm/nvm.sh + nvm install 0.10 + npm install -g node-gyp + node ./build.js --target_arch=x64 + # now do 32 bit + NVER=`node -v` + wget http://nodejs.org/dist/${NVER}/node-${NVER}-linux-x86.tar.gz + tar xf node-${NVER}-linux-x86.tar.gz + export PATH=$(pwd)/node-${NVER}-linux-x86/bin:$PATH + # ignore: + # dependency problems - leaving unconfigure gcc-4.6:i386 g++-4.6:i386 libstdc++6-4.6-dev:i386 + # E: Sub-process /usr/bin/dpkg returned an error code (1) + apt-get -y install binutils:i386 cpp:i386 gcc-4.6:i386 g++-4.6:i386 libstdc++6-4.6-dev:i386 | true + CC=gcc-4.6 CXX=g++-4.6 node ./build.js --target_arch=ia32 +fi \ No newline at end of file diff --git a/build-util/tools.js b/build-util/tools.js new file mode 100644 index 000000000..c9ff84606 --- /dev/null +++ b/build-util/tools.js @@ -0,0 +1,102 @@ +var ProgressBar = require('progress'); +var http = require('http'); +var url = require('url'); +var fs = require('fs'); + +function download(from,to,callback) { + var uri = url.parse(from); + var req = http.request(uri); + req.on('response', function(res){ + // needed for end to be called + res.resume(); + if (res.statusCode !== 200) { + return callback(new Error('Server returned '+ res.statusCode),false); + } + var len = parseInt(res.headers['content-length'], 10); + console.log(); + var bar = new ProgressBar(' downloading [:bar] :percent :etas', { + complete: '=' + , incomplete: ' ' + , width: 40 + , total: len + }); + function returnBuffer() { + for (var length = 0, i = 0; i < out.length; i++) { + length += out[i].length; + } + var result = new Buffer(length); + for (var pos = 0, j = 0; j < out.length; j++) { + out[j].copy(result, pos); + pos += out[j].length; + } + fs.writeFile(to,result,function(err) { + if (err) return callback(err,true); + return callback(null,true); + }); + } + var out = []; + res.on('data', function(chunk) { + bar.tick(chunk.length); + out.push(chunk); + found_remote = true; + }); + res.on('end', function(){ + console.log('\n'); + returnBuffer(); + }); + res.on('close', function(){ + returnBuffer(); + }); + }); + req.on('error', function(err){ + callback(err,false); + }); + req.end(); +} + + +function parse_args(_args, opts) { + // first split them like npm returns + var args = []; + _args.forEach(function(a) { + var parts = a.split('='); + parts.forEach(function(p) { + args.push(p); + }) + }) + // respect flags passed to npm install + if (process.env.npm_config_argv) { + var argv_obj = JSON.parse(process.env.npm_config_argv); + args = args.concat(argv_obj.cooked.slice(1)) + } + var debug = (args.indexOf('--debug') > -1); + if (debug) opts.configuration = 'Debug'; + + opts.stage = (args.indexOf('--stage') > -1); + if (opts.stage) { + opts.force = true; + } else { + var from_source = args.indexOf('--build-from-source'); + if ( from_source > -1) { + // no specific module name passed + var next_arg = args[from_source+1]; + if (!next_arg || next_arg.indexOf('--') <= 0) { + opts.force = true; + } else if (next_arg == 'sqlite3'){ + opts.force = true; + } + } + } + var target_arch = args.indexOf('--target_arch'); + if (target_arch > -1) { + var next_arg = args[target_arch+1]; + if (next_arg && next_arg.indexOf('--') < 0) { + opts.target_arch = next_arg; + } + } + opts.args = args; + return opts; +} + +module.exports.parse_args = parse_args; +module.exports.download = download; \ No newline at end of file diff --git a/build-util/upload.sh b/build-util/upload.sh new file mode 100755 index 000000000..624bbf06b --- /dev/null +++ b/build-util/upload.sh @@ -0,0 +1,18 @@ +export ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +export DRY_RUN="--dry-run" + +cd ${ROOTDIR}/../stage/ +if [ -d Debug ]; then +cd Debug +../../../s3cmd/s3cmd sync --no-check-md5 --acl-public ./*.tar.gz s3://node-sqlite3/Debug/ ${DRY_RUN} +cd ../ +fi + +if [ -d Release ]; then +cd Release +../../../s3cmd/s3cmd sync --no-check-md5 --acl-public ./*.tar.gz s3://node-sqlite3/Release/ ${DRY_RUN} +cd ../ +fi + +#../../s3cmd/s3cmd ls s3://node-sqlite3/ diff --git a/build.js b/build.js new file mode 100755 index 000000000..f0732d4ed --- /dev/null +++ b/build.js @@ -0,0 +1,233 @@ +#!/usr/bin/env node + +/* + +TODO + +Really should do: + - checksumming + +Future: + - travis/nvm/32bit auto-build and post to s3 for linux + - script to check for acl-public + - use require() to support node_modules location of binary? +*/ + +var package_json = require('./package.json'); +var Binary = require('./lib/binary_name.js').Binary; +var util = require('./build-util/tools.js'); +var mkdirp = require('mkdirp'); +// https://github.com/isaacs/node-tar/issues/11 +//var tar = require('tar'); +var targz = require('tar.gz'); +var cp = require('child_process'); +var fs = require('fs'); +var path = require('path'); +var os = require('os'); + +var opts = { + name: 'node_sqlite3', + force: false, + stage: false, + configuration: 'Release', + target_arch: process.arch, + platform: process.platform, + uri: 'http://dei9kzb8scfgo.cloudfront.net/', + paths: {} +} + +function log(msg) { + console.log('['+package_json.name+']: ' + msg); +} + +// only for dev +function log_debug(msg) { + //log(msg); +} + +function done(err) { + if (err) { + log(err); + process.exit(1); + } + process.exit(0); +} + +function test(opts,try_build,callback) { + fs.statSync(opts.paths.runtime_module_path); + var args = []; + var shell_cmd; + var arch_names = { + 'ia32':'-i386', + 'x64':'-x86_64' + } + if (process.platform === 'darwin' && arch_names[opts.target_arch]) { + shell_cmd = 'arch'; + args.push(arch_names[opts.target_arch]); + args.push(process.execPath); + } else if (process.arch == opts.target_arch) { + shell_cmd = process.execPath; + } + if (!shell_cmd) { + // system we cannot test on - likely since we are cross compiling + log("Skipping testing binary for " + process.target_arch); + return callback(); + } + args.push('lib/sqlite3'); + cp.execFile(shell_cmd, args, function(err, stdout, stderr) { + if (err || stderr) { + var output = err.message || stderr; + log('Testing the binary failed: "' + output + '"'); + if (try_build) { + log('Attempting source compile...'); + build(opts,callback); + } + } else { + log('Sweet: "' + opts.binary.filename() + '" is valid, node-sqlite3 is now installed!'); + return callback(); + } + }); +} + +function build(opts,callback) { + var shell_cmd = process.platform === 'win32' ? 'node-gyp.cmd' : 'node-gyp'; + var shell_args = ['rebuild'].concat(opts.args); + var cmd = cp.spawn(shell_cmd,shell_args); + cmd.on('error', function(err) { + if (err) { + return callback(new Error("Failed to execute '" + shell_cmd + ' ' + shell_args.join(' ') + "' (" + err + ")")); + } + }); + cmd.stdout.on('data',function(data) { + console.log(data.slice(0,data.length-1).toString()); + }) + // TODO - this node-gyp output comes through formatted poorly, hence disabled + /* + cmd.stderr.on('data',function(data) { + console.error(data.slice(0,data.length-1).toString()); + }) + */ + cmd.on('exit', function(err) { + if (err) { + if (err === 127) { + console.error( + 'node-gyp not found! Please upgrade your install of npm! You need at least 1.1.5 (I think) '+ + 'and preferably 1.1.30.' + ); + } else { + console.error('Build failed'); + } + return callback(err); + } + move(opts,callback); + }); +} + +function tarball(opts,callback) { + var source = path.dirname(opts.paths.staged_module_file_name); + log('compressing: ' + source + ' to ' + opts.paths.tarball_path); + new targz(9).compress(source, opts.paths.tarball_path, function(err) { + if (err) return callback(err); + log('Versioned binary staged for upload at ' + opts.paths.tarball_path); + return callback(); + }); +} + +function move(opts,callback) { + try { + fs.statSync(opts.paths.build_module_path); + } catch (ex) { + return callback(new Error('Build succeeded but target not found at ' + opts.paths.build_module_path)); + } + try { + mkdirp.sync(path.dirname(opts.paths.runtime_module_path)); + log('Created: ' + path.dirname(opts.paths.runtime_module_path)); + } catch (err) { + log_debug(err); + } + fs.renameSync(opts.paths.build_module_path,opts.paths.runtime_module_path); + if (opts.stage) { + try { + mkdirp.sync(path.dirname(opts.paths.staged_module_file_name)); + log('Created: ' + path.dirname(opts.paths.staged_module_file_name)) + } catch (err) { + log_debug(err); + } + fs.writeFileSync(opts.paths.staged_module_file_name,fs.readFileSync(opts.paths.runtime_module_path)); + // drop build metadata into build folder + var metapath = path.join(path.dirname(opts.paths.staged_module_file_name),'build-info.json'); + // more build info + opts.date = new Date(); + opts.node_features = process.features; + opts.versions = process.versions; + opts.config = process.config; + opts.execPath = process.execPath; + fs.writeFileSync(metapath,JSON.stringify(opts,null,2)); + tarball(opts,callback); + } else { + log('Installed in ' + opts.paths.runtime_module_path + ''); + test(opts,false,callback); + } +} + +function rel(p) { + return path.relative(process.cwd(),p); +} + +var opts = util.parse_args(process.argv.slice(2),opts); +opts.binary = new Binary(opts); +var versioned = opts.binary.getRequirePath(); +opts.paths.runtime_module_path = rel(path.join(__dirname, 'lib', versioned)); +opts.paths.runtime_folder = rel(path.join(__dirname, 'lib', 'binding',opts.binary.configuration)); +var staged_module_path = path.join(__dirname, 'stage', opts.binary.getModuleAbi(), opts.binary.getBasePath()); +opts.paths.staged_module_file_name = rel(path.join(staged_module_path,opts.binary.filename())); +opts.paths.build_module_path = rel(path.join(__dirname, 'build', opts.binary.configuration, opts.binary.filename())); +opts.paths.tarball_path = rel(path.join(__dirname, 'stage', opts.binary.configuration, opts.binary.getArchivePath())); + +if (!{ia32: true, x64: true, arm: true}.hasOwnProperty(opts.target_arch)) { + return done(new Error('Unsupported (?) architecture: '+ opts.target_arch+ '')); +} + +if (opts.force) { + build(opts,done); +} else { + try { + test(opts,true,done); + } catch (ex) { + var from = opts.binary.getRemotePath(); + var tmpdir; + if (os.tmpdir) { + tmpdir = os.tmpdir(); + } else { + var tmpdir = '/tmp/node-sqlite3-' + opts.binary.configuration; + try { + mkdirp.sync(tmpdir); + } catch (err) { + log_debug(err); + } + } + var tmpfile = path.join(tmpdir,path.basename(from)); + util.download(from,tmpfile,function(err,found_remote) { + if (err) { + if (!found_remote) { + log(from + ' not found, falling back to source compile (' + err + ')'); + build(opts,done); + } else { + return done(err); + } + } else { + log('downloaded to temp location: '+ tmpfile); + new targz().extract(tmpfile, opts.paths.runtime_folder, function(err) { + if (err) return done(err); + try { + test(opts,true,done); + } catch (ex) { + // Stat failed + log(opts.paths.runtime_folder + ' not found, falling back to source compile'); + build(opts,done); + } + }); + } + }); + } +} diff --git a/lib/binary_name.js b/lib/binary_name.js new file mode 100644 index 000000000..2ca6b6d24 --- /dev/null +++ b/lib/binary_name.js @@ -0,0 +1,57 @@ + +var path = require('path'); + +var Binary = function(options) { + var options = options || {}; + var package_json = options.package_json || require('../package.json'); + this.name = options.name || 'binding'; + this.configuration = options.configuration || 'Release'; + this.uri = options.uri || 'http://'+this.name+'.s3.amazonaws.com/'; + this.module_maj_min = package_json.version.split('.').slice(0,2).join('.'); + this.module_abi = package_json.abi; + this.platform = options.platform || process.platform; + this.target_arch = options.target_arch || process.arch; + if (process.versions.modules) { + this.node_abi = 'node-v' + process.versions.modules + } else { + this.node_abi = 'v8-' + process.versions.v8.split('.').slice(0,2).join('.'); + } +} + +Binary.prototype.filename = function() { + return this.name + '.node'; +} + +Binary.prototype.compression = function() { + return '.tar.gz'; +} + +Binary.prototype.getBasePath = function() { + return this.node_abi + + '-' + this.platform + + '-' + this.target_arch; +} + +Binary.prototype.getRequirePath = function(configuration) { + return './' + path.join('binding', + configuration || this.configuration, + this.getBasePath(), + this.filename()); +} + +Binary.prototype.getModuleAbi = function() { + return this.name + '-v' + this.module_maj_min + '.' + this.module_abi; +} + +Binary.prototype.getArchivePath = function() { + return this.getModuleAbi() + + '-' + + this.getBasePath() + + this.compression(); +} + +Binary.prototype.getRemotePath = function() { + return this.uri+this.configuration+'/'+this.getArchivePath(); +} + +module.exports.Binary = Binary; \ No newline at end of file diff --git a/lib/sqlite3.js b/lib/sqlite3.js index 62541cf37..f7506c271 100644 --- a/lib/sqlite3.js +++ b/lib/sqlite3.js @@ -1,4 +1,12 @@ -var sqlite3 = module.exports = exports = require('./node_sqlite3.node'); +var Binary = require('./binary_name.js').Binary; +var binary = new Binary({name:'node_sqlite3'}); +var binding; +try { + binding = require(binary.getRequirePath('Debug')); +} catch (err) { + binding = require(binary.getRequirePath('Release')); +} +var sqlite3 = module.exports = exports = binding; var path = require('path'); var util = require('util'); var EventEmitter = require('events').EventEmitter; diff --git a/package.json b/package.json index f020a15f5..fb7745a3f 100644 --- a/package.json +++ b/package.json @@ -2,6 +2,7 @@ "name": "sqlite3", "description": "Asynchronous, non-blocking SQLite3 bindings", "version": "2.1.15", + "abi":"a", "homepage": "http://github.com/developmentseed/node-sqlite3", "author": { "name": "Development Seed", @@ -29,10 +30,17 @@ "type": "git", "url": "git://github.com/developmentseed/node-sqlite3.git" }, + "dependencies": { + "progress":"~1.0.1", + "mkdirp":"~0.3.5", + "tar.gz": "~0.1.1" + }, + "bundledDependencies":["mkdirp","tar.gz","progress"], "engines": { "node": ">= 0.6.13 < 0.11.0" }, "scripts": { + "install": "node build.js", "pretest": "node test/support/createdb.js", "test": "mocha -R spec --timeout 200000" },