diff --git a/.gitignore b/.gitignore index e84f2b2c4f..86b420e9ec 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,4 @@ lib/objects/lua /test/redis-sentinel/*.config /lib/.greenlockrc /.greenlockrc +/.dev-server \ No newline at end of file diff --git a/lib/setup.js b/lib/setup.js index df17a86230..ae3ce14715 100644 --- a/lib/setup.js +++ b/lib/setup.js @@ -16,7 +16,6 @@ // TODO need info about progress of stopping const fs = require('fs-extra'); -const pathLib = require('path'); const tools = require('./tools.js'); const cli = require('./cli/index.js'); const EXIT_CODES = require('./exitCodes'); @@ -120,12 +119,7 @@ function initYargs() { } }) .command(['install ', 'i '], 'Installs a specified adapter', {}) - .command('rebuild |self', 'Rebuilds a specified adapter', { - install: { - describe: 'Install', - type: 'boolean' - } - }) + .command('rebuild', 'Rebuild all native modules', {}) .command('url []', 'Install adapter from specified url, e.g. GitHub', {}) .command(['del ', 'delete '], 'Remove adapter from system', { custom: { @@ -388,7 +382,7 @@ let states; // instance * @param {object} params - object with parsed params by yargs, e. g. --force is params.force * @param {(exitCode?: number) => void} callback */ -function processCommand(command, args, params, callback) { +async function processCommand(command, args, params, callback) { if (typeof args === 'function') { callback = args; args = null; @@ -502,7 +496,6 @@ function processCommand(command, args, params, callback) { const install = new Install({ objects, states, - installNpm, getRepository, processExit: callback, params @@ -608,7 +601,6 @@ function processCommand(command, args, params, callback) { const install = new Install({ objects, states, - installNpm, getRepository, processExit: callback, params @@ -703,7 +695,6 @@ function processCommand(command, args, params, callback) { const install = new Install({ objects, states, - installNpm, getRepository, processExit: callback, params @@ -749,30 +740,18 @@ function processCommand(command, args, params, callback) { } case 'rebuild': { - let name = args[0]; - - // If user accidentally wrote tools.appName.adapter => remove adapter - name = cli.tools.normalizeAdapterName(name); - - if (name.indexOf('@') !== -1) { - name = name.split('@')[0]; - } + console.log(`Rebuilding native modules...`); + const result = await tools.rebuildNodeModules({ + debug: process.argv.includes('--debug') + }); - if (!name) { - console.log('Please provide the name of the adapter to rebuild'); - return void callback(EXIT_CODES.INVALID_ADAPTER_ID); + if (result.success) { + console.log(); + console.log(`Rebuilding native modules done`); + return void callback(); + } else { + processExit(`Rebuilding native modules failed with exit code ${result.exitCode}`); } - - const rebuildCommand = params.install ? 'install' : 'rebuild'; - installNpm(name, rebuildCommand, (err, _adapter) => { - if (err) { - processExit(err); - } else { - console.log(); - console.log('Rebuild ' + name + ' done'); - return void callback(); - } - }); break; } @@ -875,7 +854,6 @@ function processCommand(command, args, params, callback) { const install = new Install({ objects, states, - installNpm, getRepository, processExit: callback, params @@ -891,7 +869,6 @@ function processCommand(command, args, params, callback) { const install = new Install({ objects, states, - installNpm, getRepository, processExit: callback, params @@ -973,7 +950,6 @@ function processCommand(command, args, params, callback) { const upgrade = new Upgrade({ objects, states, - installNpm, getRepository, params, processExit: callback, @@ -2507,78 +2483,6 @@ function restartController(callback) { } } -function installNpm(adapter, rebuildCommand, callback) { - if (typeof rebuildCommand === 'function') { - callback = rebuildCommand; - rebuildCommand = false; - } - - let path = __dirname; - if (typeof adapter === 'function') { - callback = adapter; - adapter = undefined; - } - - if (adapter) { - if (rebuildCommand && adapter === 'self') { - path = pathLib.join(__dirname, '..'); - } else { - path = tools.getAdapterDir(adapter); - } - } - - let debug = false; - for (let i = 0; i < process.argv.length; i++) { - if (process.argv[i] === '--debug') { - debug = true; - break; - } - } - - if (!path) { - console.log(`Cannot install ${tools.appName}.${adapter}: adapter path not found`); - return (callback || processExit)(EXIT_CODES.CANNOT_INSTALL_NPM_PACKET); - } - const npmCommand = typeof rebuildCommand === 'string' ? rebuildCommand : 'install'; - - // iob_npm.done file was created if "npm i" yet called there - if (fs.existsSync(pathLib.join(path, 'package.json')) && (rebuildCommand || !fs.existsSync(pathLib.join(path, 'iob_npm.done')))) { - let cmd = `npm ${npmCommand} ${debug ? '' : '--loglevel error'}`; - if (npmCommand === 'install') { - cmd += ' --production'; - } - console.log(`${cmd} (System call1) in "${path}"`); - // Install node modules as system call - - // System call used for update of js-controller itself, - // because during installation npm packet will be deleted too, but some files must be loaded even during the install process. - const exec = require('child_process').exec; - const child = exec(cmd, { - cwd: path, - windowsHide: true - }); - tools.pipeLinewise(child.stderr, process.stdout); - - debug && tools.pipeLinewise(child.stdout, process.stdout); - - child.on('exit', (code, _signal) => { - // code 1 is strange error that cannot be explained. Everything is installed but error :( - if (code && code !== 1) { - console.log(`Cannot install ${tools.appName}.${adapter}: ${code}`); - (callback || processExit)(EXIT_CODES.CANNOT_INSTALL_NPM_PACKET); - return; - } - // command succeeded - if (!rebuildCommand || rebuildCommand === 'install') { - fs.writeFileSync(path + '/iob_npm.done', ' '); - } - typeof callback === 'function' && callback(null, adapter); - }); - } else if (typeof callback === 'function') { - callback(null, adapter); - } -} - function getRepository(repoUrl, params, callback) { if (typeof params === 'function') { callback = params; diff --git a/lib/setup/setupInstall.js b/lib/setup/setupInstall.js index ad24e4db0e..587ee3d0c2 100644 --- a/lib/setup/setupInstall.js +++ b/lib/setup/setupInstall.js @@ -23,12 +23,11 @@ function Install(options) { const PacketManager = require('./setupPacketManager'); const osPlatform = require('os').platform(); const deepClone = require('deep-clone'); - const {URL} = require('url'); + const { URL } = require('url'); // todo solve it somehow const unsafePermAlways = [tools.appName.toLowerCase() + '.zwave', tools.appName.toLowerCase() + '.amazon-dash', tools.appName.toLowerCase() + '.xbox']; const isRootOnUnix = typeof process.getuid === 'function' && process.getuid() === 0; - let JSZip; /** @type {Install} */ const that = this; @@ -44,9 +43,6 @@ function Install(options) { if (!options.processExit) { throw new Error('Invalid arguments: processExit is missing'); } - if (!options.installNpm) { - throw new Error('Invalid arguments: installNpm is missing'); - } if (!options.getRepository) { throw new Error('Invalid arguments: getRepository is missing'); } @@ -54,7 +50,6 @@ function Install(options) { const objects = options.objects; const states = options.states; const processExit = options.processExit; - const installNpm = options.installNpm; const getRepository = options.getRepository; const params = options.params || {}; let mime; @@ -119,38 +114,8 @@ function Install(options) { } } - function _writeOneFile(zip, targetName, fileName, callback) { - zip.files[fileName].async('nodebuffer').then(data => { - fs.writeFileSync(path.join(targetName, fileName), data); - callback(); - }, err => callback(err)); - } - - function extractFiles(fileName, targetName, callback) { - JSZip = JSZip || require('jszip'); - const zip = new JSZip(); - zip.loadAsync(fs.readFileSync(fileName)).then(() => { - let count = 0; - for (const fName of Object.keys(zip.files)) { - if (!fName || fName[fName.length - 1] === '/') { - continue; - } - count++; - _writeOneFile(zip, targetName, fName, err => { - if (!--count) { - callback(err); - } - }); - } - if (!count) { - callback(); - } - }); - } - this.downloadPacket = function (repoUrl, packetName, options, stoppedList, callback) { let url; - let name; if (!options || typeof options !== 'object') { options = {}; } @@ -245,10 +210,11 @@ function Install(options) { } if (url && url.match(tarballRegex)) { // Install node modules - return that.npmInstallWithCheck(url, options, debug, () => { + that.npmInstallWithCheck(url, options, debug, () => { // command succeeded typeof callback === 'function' && callback(_callback => enableAdapters(stoppedList, true, _callback), packetName); }); + return; } // Adapter if (!url) { @@ -256,92 +222,10 @@ function Install(options) { return typeof callback === 'function' && callback(_callback => typeof _callback === 'function' && _callback(), packetName); } - name = packetName.replace(/[/ $&*\\]/g, '_'); - } else { - url = packetName; - if (!url.includes('http://') && !url.includes('https://') && !url.includes('file://')) { - console.error('host.' + hostname + ' Unknown packetName ' + packetName); - processExit(EXIT_CODES.UNKNOWN_PACKET_NAME); - } - name = Math.floor(Math.random() * 0xFFFFFFE).toString(); } - const {ncp} = require('ncp'); - ncp.limit = 16; - - console.log(`host.${hostname} download ${url}`); - - tools.getFile(url, name + '.zip', tmpFile => { - tmpFile = path.normalize(tmpFile); - console.log(`host.${hostname} unzip ${tmpFile}`); - - // Extract files into tmp/ - extractFiles(tmpFile, path.join(__dirname + '/../../tmp/', name), error => { - if (error) { - console.error(error); - processExit(EXIT_CODES.CANNOT_EXTRACT_FROM_ZIP); - } - // Find out the first directory - const dirs = fs.readdirSync(__dirname + '/../../tmp/' + name); - if (dirs.length) { - const source = __dirname + '/../../tmp/' + name + ((dirs.length === 1) ? '/' + dirs[0] : ''); - // Copy files into adapter or controller - if (fs.existsSync(source + '/io-package.json')) { - let packetIo; - try { - packetIo = fs.readJSONSync(source + '/io-package.json'); - } catch { - console.error('host.' + hostname + ' io-package.json has invalid format! Installation terminated.'); - typeof callback === 'function' && callback(_callback => _callback && _callback(), name, 'Invalid io-package.json!'); - processExit(EXIT_CODES.INVALID_IO_PACKAGE_JSON); - } - packetIo.common = packetIo.common || {}; - packetIo.common.installedFrom = url; - fs.writeFileSync(source + '/io-package.json', JSON.stringify(packetIo, null, 2), 'utf8'); - - let destination = __dirname + '/../..'; - if (!packetIo.common.controller) { - if (fs.existsSync(destination + '/../../node_modules')) { - destination += '/../' + tools.appName + '.' + packetIo.common.name; - } else { - destination += '/node_modules/' + tools.appName + '.' + packetIo.common.name; - } - } - - destination = path.normalize(destination); - - console.log(`host.${hostname} copying ${source} to ${destination}(Version: ${packetIo.common.version})`); - - ncp(source, destination, err => { - if (err) { - console.error(`host.${hostname} ncp error: ${err}`); - processExit(EXIT_CODES.CANNOT_COPY_DIR); - } - if (tmpFile.substring(0, (path.normalize(__dirname + '/../../tmp/')).length) === path.normalize(__dirname + '/../../tmp/')) { - console.log(`host.${hostname} delete ${tmpFile}`); - fs.unlinkSync(tmpFile); - } - console.log(`host.${hostname} delete ${path.normalize(__dirname + '/../../tmp/' + name)}`); - tools.rmdirRecursiveSync(__dirname + '/../../tmp/' + name); - - // Call npm install - if (typeof callback === 'function') { - typeof callback === 'function' && callback(_callback => enableAdapters(stoppedList, true, _callback), name, packetIo); - } - - }); - } else { - console.error(`host.${hostname} io-package.json not found in ${source}/io-package.json. Invalid packet! Installation terminated.`); - typeof callback === 'function' && callback(_callback => _callback && _callback(), name, 'Invalid packet!'); - processExit(EXIT_CODES.INVALID_IO_PACKAGE_JSON); - } - } else { - console.error(`host.${hostname} Packet is empty! Installation terminated.`); - typeof callback === 'function' && callback(_callback => _callback && _callback(), name, 'Packet is empty'); - processExit(EXIT_CODES.MISSING_ADAPTER_FILES); - } - }); - }); + console.error(`host.${hostname} Unknown packetName ${packetName}. Please install packages from outside the repository using npm!`); + processExit(EXIT_CODES.UNKNOWN_PACKET_NAME); }; this.npmInstallWithCheck = function (npmUrl, options, debug, callback) { @@ -387,27 +271,11 @@ function Install(options) { } }; - this.npmInstall = function (npmUrl, options, debug, callback) { + this.npmInstall = async function (npmUrl, options, debug, callback) { if (typeof options !== 'object') { options = {}; } - // Install node modules - /** @type {string|string[]} */ - let cwd = __dirname.replace(/\\/g, '/'); - if (fs.existsSync(__dirname + '/../../../../node_modules/' + tools.appName + '.js-controller')) { - // js-controller installed as npm - cwd = cwd.split('/'); - cwd.splice(cwd.length - 4, 4); - cwd = cwd.join('/'); - } else { - // remove lib - cwd = cwd.split('/'); - cwd.pop(); - cwd.pop(); - cwd = cwd.join('/'); - } - // zwave for example requires always unsafe-perm option if (unsafePermAlways.some(adapter => npmUrl.indexOf(adapter) > -1)) { options.unsafePerm = true; @@ -418,59 +286,36 @@ function Install(options) { options.unsafePerm = true; } - // We don't need --production and --save here. - // --production doesn't do anything when installing a specific package (which we do here) - // --save is the default since npm 3 - // Don't use --prefix on Windows, because that has ugly bugs - const cmd = [ - 'npm install', - npmUrl, - debug ? '' : '--loglevel error', - options.unsafePerm ? '--unsafe-perm' : '', - osPlatform !== 'win32' ? `--prefix "${cwd}"` : '' - ].filter(arg => !!arg).join(' '); - - console.log(`${cmd} (System call)`); - // Install node modules as system call - - // System call used for update of js-controller itself, - // because during installation npm packet will be deleted too, but some files must be loaded even during the install process. - const exec = require('child_process').exec; - const child = exec(cmd, { - windowsHide: true, - cwd - }); - tools.pipeLinewise(child.stderr, process.stdout); - if (debug || params.debug) { - tools.pipeLinewise(child.stdout, process.stdout); - } + console.log(`Installing ${npmUrl}... (System call)`); - // Determine where the packet would be installed if npm succeeds - /** @type {string} */ - let packetDirName; - if (options.packetName) { - packetDirName = tools.appName.toLowerCase() + '.' + options.packetName; - } else { - packetDirName = npmUrl.toLowerCase(); - // If the user installed a git commit-ish, the url contains stuff that doesn't belong in a folder name - // e.g. iobroker/iobroker.javascript#branch-name - if (packetDirName.indexOf('#') > -1) { - packetDirName = packetDirName.substr(0, packetDirName.indexOf('#')); - } - if (packetDirName.indexOf('/') > -1 && !packetDirName.startsWith('@')) { - // only scoped packages (e.g. @types/node ) may have a slash in their path - packetDirName = packetDirName.substr(packetDirName.lastIndexOf('/') + 1); - } - } - const installDir = path.join(cwd, 'node_modules', packetDirName); + const result = await tools.installNodeModule(npmUrl, { + debug: !!debug, + unsafePerm: !!options.unsafePerm + }); - child.on('exit', code => { + if (result.success || result.exitCode === 1) { // code 1 is strange error that cannot be explained. Everything is installed but error :( - if (code && code !== 1) { - console.error('host.' + hostname + ' Cannot install ' + npmUrl + ': ' + code); - processExit(EXIT_CODES.CANNOT_INSTALL_NPM_PACKET); - return; + + // Determine where the packet would be installed if npm succeeds + // TODO: There's probably a better way to figure this out + /** @type {string} */ + let packetDirName; + if (options.packetName) { + packetDirName = tools.appName.toLowerCase() + '.' + options.packetName; + } else { + packetDirName = npmUrl.toLowerCase(); + // If the user installed a git commit-ish, the url contains stuff that doesn't belong in a folder name + // e.g. iobroker/iobroker.javascript#branch-name + if (packetDirName.indexOf('#') > -1) { + packetDirName = packetDirName.substr(0, packetDirName.indexOf('#')); + } + if (packetDirName.indexOf('/') > -1 && !packetDirName.startsWith('@')) { + // only scoped packages (e.g. @types/node ) may have a slash in their path + packetDirName = packetDirName.substr(packetDirName.lastIndexOf('/') + 1); + } } + const installDir = tools.getAdapterDir(packetDirName); + // inject the installedFrom information in io-package if (fs.existsSync(installDir)) { const ioPackPath = path.join(installDir, 'io-package.json'); @@ -490,76 +335,35 @@ function Install(options) { } } } else { - console.error('host.' + hostname + ' Cannot install ' + npmUrl + ': ' + code); + // TODO: revisit this - this should not happen + console.error(`host.${hostname} Cannot install ${npmUrl}: ${result.exitCode}`); processExit(EXIT_CODES.CANNOT_INSTALL_NPM_PACKET); return; } // create file that indicates, that npm was called there fs.writeFileSync(path.join(installDir, 'iob_npm.done'), ' '); // command succeeded - typeof callback === 'function' && callback(npmUrl, cwd + '/node_modules'); - }); - }; - - this.npmUninstall = function (packageName, options, debug, callback) { - // TODO: find a nicer way to find the root directory - - // Install node modules - /** @type {string|string[]} */ - let cwd = __dirname.replace(/\\/g, '/'); - if (fs.existsSync(`${__dirname}/../../../../node_modules/${tools.appName}.js-controller`)) { - // js-controller installed as npm - cwd = cwd.split('/'); - cwd.splice(cwd.length - 4, 4); - cwd = cwd.join('/'); + typeof callback === 'function' && callback(npmUrl, path.dirname(installDir)); } else { - // remove lib - cwd = cwd.split('/'); - cwd.pop(); - cwd.pop(); - cwd = cwd.join('/'); + console.error(`host.${hostname} Cannot install ${npmUrl}: ${result.exitCode}`); + processExit(EXIT_CODES.CANNOT_INSTALL_NPM_PACKET); + return; } - // Don't use --prefix on Windows, because that has ugly bugs - // Instead set the working directory (cwd) of the process - const cmd = [ - 'npm uninstall', - packageName, - debug ? '' : '--loglevel error', - osPlatform !== 'win32' ? `--prefix "${cwd}"` : '' - ].filter(arg => !!arg).join(' '); - - console.log(`${cmd} (System call)`); - // Install node modules as system call - - // System call used for update of js-controller itself, - // because during installation npm packet will be deleted too, but some files must be loaded even during the install process. - const exec = require('child_process').exec; - const child = exec(cmd, { - windowsHide: true, - cwd + }; + + /** @type {(packageName: string, options: any, debug: boolean, callback?: (err?: Error) => void) => Promise} */ + this.npmUninstall = async function (packageName, options, debug, callback) { + const result = await tools.uninstallNodeModule(packageName, { + debug: !!debug }); - tools.pipeLinewise(child.stderr, process.stdout); - if (debug || params.debug) { - tools.pipeLinewise(child.stdout, process.stdout); + if (result.success) { + return tools.maybeCallback(callback); + } else { + return tools.maybeCallbackWithError(callback, `host.${hostname}: Cannot uninstall ${packageName}: ${result.exitCode}`); } - child.on('exit', code => { - // code 1 is strange error that cannot be explained. Everything is installed but error :( - if (code) { - if (typeof callback === 'function') { - callback(`host.${hostname}: Cannot uninstall ${packageName}: ${code}`); - } - } - // command succeeded - if (callback) { - callback(); - } - }); }; - /** @type {(packageName: string, options: any, debug: boolean) => Promise} */ - this.npmUninstallAsync = tools.promisify(this.npmUninstall, this); - // this command is executed always on THIS host function checkDependencies(adapter, deps, globalDeps, _options, callback) { if (!deps && !globalDeps) { @@ -788,28 +592,12 @@ function Install(options) { } } - if (!fs.existsSync(adapterDir + '/node_modules')) { - // Install node modules - installNpm(adapter, (err, _adapter) => { - if (err) { - processExit(err); - } else { - upload.uploadAdapter(_adapter, true, true, null, null, () => - upload.uploadAdapter(_adapter, false, true, null, null, () => - callInstallOfAdapter(_adapter, adapterConf, () => - that.uploadStaticObjects(adapter, _err => - upload.upgradeAdapterObjects(adapter, () => - callback(adapter)))))); - } - }); - } else { - upload.uploadAdapter(adapter, true, true, () => - upload.uploadAdapter(adapter, false, true, () => - callInstallOfAdapter(adapter, adapterConf, () => - that.uploadStaticObjects(adapter, _err => - upload.upgradeAdapterObjects(adapter, () => - callback(adapter)))))); - } + upload.uploadAdapter(adapter, true, true, () => + upload.uploadAdapter(adapter, false, true, () => + callInstallOfAdapter(adapter, adapterConf, () => + that.uploadStaticObjects(adapter, _err => + upload.upgradeAdapterObjects(adapter, () => + callback(adapter)))))); }; async function callInstallOfAdapter(adapter, config, callback) { @@ -1497,7 +1285,7 @@ function Install(options) { const ioPack = require(`${adapterNpm}/io-package.json`); // yep, it's that easy if (!ioPack.common || !ioPack.common.nondeletable) { - await that.npmUninstallAsync(adapterNpm, null, false); + await that.npmUninstall(adapterNpm, null, false); // after uninstalling we have to restart the defined adapters if (ioPack.common.restartAdapters) { if (!Array.isArray(ioPack.common.restartAdapters)) { diff --git a/lib/setup/setupUpgrade.js b/lib/setup/setupUpgrade.js index fd7a1dfd36..de4e06ea47 100644 --- a/lib/setup/setupUpgrade.js +++ b/lib/setup/setupUpgrade.js @@ -20,9 +20,6 @@ function Upgrade(options) { if (!options.processExit) { throw new Error('Invalid arguments: processExit is missing'); } - if (!options.installNpm) { - throw new Error('Invalid arguments: installNpm is missing'); - } if (!options.restartController) { throw new Error('Invalid arguments: restartController is missing'); } @@ -31,7 +28,6 @@ function Upgrade(options) { } const processExit = options.processExit; - const installNpm = options.installNpm; const getRepository = options.getRepository; const params = options.params; const objects = options.objects; @@ -291,35 +287,29 @@ function Upgrade(options) { } let count = 0; - installNpm(name, (err, _name) => { - if (err) { - processExit(err); - } else { - // Upload www and admin files of adapter into CouchDB - count++; - upload.uploadAdapter(name, false, true, () => { - // extend all adapter instance default configs with current config - // (introduce potentially new attributes while keeping current settings) - upload.upgradeAdapterObjects(name, iopack, () => { - count--; - if (!count) { - console.log(`Adapter "${name}" updated`); - if (callback) { - callback(name); - } - } - }); - }); - count++; - upload.uploadAdapter(name, true, true, () => { - count--; - if (!count) { - console.log(`Adapter "${name}" updated`); - if (callback) { - callback(name); - } + // Upload www and admin files of adapter into CouchDB + count++; + upload.uploadAdapter(name, false, true, () => { + // extend all adapter instance default configs with current config + // (introduce potentially new attributes while keeping current settings) + upload.upgradeAdapterObjects(name, iopack, () => { + count--; + if (!count) { + console.log(`Adapter "${name}" updated`); + if (callback) { + callback(name); } - }); + } + }); + }); + count++; + upload.uploadAdapter(name, true, true, () => { + count--; + if (!count) { + console.log(`Adapter "${name}" updated`); + if (callback) { + callback(name); + } } }); }; @@ -628,13 +618,7 @@ function Upgrade(options) { console.log(`Update ${installed.common.name} from @${installed.common.version} to @${repoUrl[installed.common.name].version}`); // Get the controller from web site install.downloadPacket(repoUrl, `${installed.common.name}@${repoUrl[installed.common.name].version}`, null, (enableAdapterCallback, _name) => { - installNpm((err, _name) => { - if (err) { - processExit(err); - } else { - enableAdapterCallback(callback); - } - }); + enableAdapterCallback(callback); }); } } else { @@ -667,13 +651,7 @@ function Upgrade(options) { console.log(`Update ${name} from @${installed.common.version} to ${version}`); // Get the controller from web site install.downloadPacket(repoUrl, name + version, null, (enableAdapterCallback, _name) => { - installNpm((err, _name) => { - if (err) { - processExit(err); - } else { - enableAdapterCallback(callback); - } - }); + enableAdapterCallback(callback); }); } }); diff --git a/lib/tools.js b/lib/tools.js index acbc04a335..f7c551c8d7 100644 --- a/lib/tools.js +++ b/lib/tools.js @@ -8,6 +8,8 @@ const forge = require('node-forge'); const deepClone = require('deep-clone'); const cpPromise = require('promisify-child-process'); const { createInterface } = require('readline'); +const { PassThrough } = require('stream'); +const { detectPackageManager } = require('@alcalzone/pak'); // @ts-ignore require('events').EventEmitter.prototype._maxListeners = 100; @@ -1143,6 +1145,142 @@ function getSystemNpmVersion(callback) { } } +/** + * @typedef {object} InstallNodeModuleOptions + * @property {boolean} [unsafePerm] Whether the `--unsafe-perm` flag should be used + * @property {boolean} [debug] Whether to include `stderr` in the output and increase the loglevel to include more than errors + * @property {string} [cwd] Which directory to work in. If none is given, this defaults to ioBroker's root directory. + */ + +/** + * Installs a node module using npm or a similar package manager + * @param {string} npmUrl Which node module to install + * @param {InstallNodeModuleOptions} options Options for the installation + * @returns {Promise} + */ +async function installNodeModule(npmUrl, options = {}) { + // Figure out which package manager is in charge (probably npm at this point) + const pak = await detectPackageManager( + typeof options.cwd === 'string' + // If a cwd was provided, use it + ? { cwd: options.cwd } + // Otherwise find the ioBroker root dir + : { + cwd: __dirname, + setCwdToPackageRoot: true + } + ); + // By default, don't print all the stuff the package manager spits out + if (!options.debug) { + pak.loglevel = 'error'; + } + + // Set up streams to pass the command output through + if (options.debug) { + const stdall = new PassThrough(); + pak.stdall = stdall; + pipeLinewise(stdall, process.stdout); + } else { + const stdout = new PassThrough(); + pak.stdout = stdout; + pipeLinewise(stdout, process.stdout); + } + + // And install the module + /** @type {import("@alcalzone/pak").InstallOptions} */ + const installOpts = {}; + if (options.unsafePerm) { + installOpts.additionalArgs = ['--unsafe-perm']; + } + return pak.install([npmUrl], installOpts); +} + +/** + * @typedef {object} UninstallNodeModuleOptions + * @property {boolean} [debug] Whether to include `stderr` in the output and increase the loglevel to include more than errors + * @property {string} [cwd] Which directory to work in. If none is given, this defaults to ioBroker's root directory. + */ + +/** + * Uninstalls a node module using npm or a similar package manager + * @param {string} packageName Which node module to uninstall + * @param {UninstallNodeModuleOptions} options Options for the installation + * @returns {Promise} + */ +async function uninstallNodeModule(packageName, options = {}) { + // Figure out which package manager is in charge (probably npm at this point) + const pak = await detectPackageManager( + typeof options.cwd === 'string' + // If a cwd was provided, use it + ? { cwd: options.cwd } + // Otherwise find the ioBroker root dir + : { + cwd: __dirname, + setCwdToPackageRoot: true + } + ); + // By default, don't print all the stuff the package manager spits out + if (!options.debug) { + pak.loglevel = 'error'; + } + + // Set up streams to pass the command output through + if (options.debug) { + const stdall = new PassThrough(); + pak.stdall = stdall; + pipeLinewise(stdall, process.stdout); + } else { + const stdout = new PassThrough(); + pak.stdout = stdout; + pipeLinewise(stdout, process.stdout); + } + + return pak.uninstall([packageName]); +} + +/** + * @typedef {object} RebuildNodeModulesOptions + * @property {boolean} [debug] Whether to include `stderr` in the output and increase the loglevel to include more than errors + * @property {string} [cwd] Which directory to work in. If none is given, this defaults to ioBroker's root directory. + */ + +/** + * Rebuilds all native node_modules that are dependencies of the project in the current working directory / project root. + * If `options.cwd` is given, the directory must contain a lockfile. + * @param {RebuildNodeModulesOptions} options Options for the rebuild + * @returns {Promise} + */ +async function rebuildNodeModules(options = {}) { + // Figure out which package manager is in charge (probably npm at this point) + const pak = await detectPackageManager( + typeof options.cwd === 'string' + // If a cwd was provided, use it + ? { cwd: options.cwd } + // Otherwise find the ioBroker root dir + : { + cwd: __dirname, + setCwdToPackageRoot: true + } + ); + // By default, don't print all the stuff the package manager spits out + if (!options.debug) { + pak.loglevel = 'error'; + } + + // Set up streams to pass the command output through + if (options.debug) { + const stdall = new PassThrough(); + pak.stdall = stdall; + pipeLinewise(stdall, process.stdout); + } else { + const stdout = new PassThrough(); + pak.stdout = stdout; + pipeLinewise(stdout, process.stdout); + } + + return pak.rebuild(); +} + /** * Read disk free space * @@ -2870,6 +3008,9 @@ module.exports = { getInstancesOrderedByStartPrio, getRepositoryFile, getSystemNpmVersion, + installNodeModule, + uninstallNodeModule, + rebuildNodeModules, isObject, isArray, maybeCallback, diff --git a/main.js b/main.js index c605815f98..61f0d6bc94 100644 --- a/main.js +++ b/main.js @@ -79,7 +79,7 @@ let isStopping = null; let allInstancesStopped = true; let stopTimeout = 10000; let uncaughtExceptionCount = 0; -const installQueue = []; +let installQueue = []; let started = false; let inputCount = 0; let outputCount = 0; @@ -2561,7 +2561,7 @@ async function processMessage(msg) { case 'rebuildAdapter': if (!installQueue.some(entry => entry.id === msg.message.id)) { logger.info(hostLogPrefix + ' ' + msg.message.id + ' will be rebuilt'); - installQueue.push({id: msg.message.id, rebuild: true, rebuildViaInstall: msg.message.rebuildViaInstall}); + installQueue.push({id: msg.message.id, rebuild: true}); // start install queue if not started installQueue.length === 1 && installAdapters(); @@ -3087,9 +3087,8 @@ function installAdapters() { } } else { installArgs.push(commandScope); - installArgs.push(name); - if (task.rebuildViaInstall) { - installArgs.push('--install'); + if (!task.rebuild) { + installArgs.push(name); } } logger.info(`${hostLogPrefix} ${tools.appName} ${installArgs.join(' ')}${task.rebuild ? '' : ' using ' + ((procs[task.id].downloadRetry < 3 && task.installedFrom) ? 'installedFrom' : 'installedVersion')}`); @@ -3113,24 +3112,40 @@ function installAdapters() { child.on('exit', exitCode => { logger.info(`${hostLogPrefix} ${tools.appName} npm-${commandScope}: exit ${exitCode}`); - installQueue.shift(); if (exitCode === EXIT_CODES.CANNOT_INSTALL_NPM_PACKET) { task.inProgress = false; - installQueue.push(task); // We add at the end again to try three times - } else if (procs[task.id]) { - procs[task.id].needsRebuild = false; - if (!task.disabled) { - if (!procs[task.id].config.common.enabled) { - logger.info(`${hostLogPrefix} startInstance ${task.id}: instance is disabled but should be started, re-enabling it`); - states.setState(task.id + '.alive', {val: true, ack: false, from: hostObjectPrefix}); - } else if (task.rebuild) { - // on rebuild we send a restart signal via object change to also reach compact group processes - objects.extendObject(task.id, {}); - } else { - startInstance(task.id, task.wakeUp); + // Move task to the end of the queue to try again (up to 3 times) + installQueue.shift(); + installQueue.push(task); + } else { + const finishTask = task => { + if (procs[task.id]) { + procs[task.id].needsRebuild = false; + if (!task.disabled) { + if (!procs[task.id].config.common.enabled) { + logger.info(`${hostLogPrefix} startInstance ${task.id}: instance is disabled but should be started, re-enabling it`); + states.setState(task.id + '.alive', {val: true, ack: false, from: hostObjectPrefix}); + } else if (task.rebuild) { + // on rebuild we send a restart signal via object change to also reach compact group processes + objects.extendObject(task.id, {}); + } else { + startInstance(task.id, task.wakeUp); + } + } else { + logger.debug(`${hostLogPrefix} ${tools.appName} ${commandScope} successful but the instance is disabled`); + } } + }; + if (task.rebuild) { + // This was a rebuild - find all tasks that required a rebuild and "finish" them (including the current one) + // Since we rebuild globally now, they should all be done too. + const rebuildTasks = installQueue.filter(t => t.rebuild); + // Remove all rebuild tasks from the queue + installQueue = installQueue.filter(t => !t.rebuild); + rebuildTasks.forEach(t => finishTask(t)); } else { - logger.debug(`${hostLogPrefix} ${tools.appName} ${commandScope} successful but the instance is disabled`); + installQueue.shift(); + finishTask(task); } } @@ -3595,10 +3610,7 @@ async function startInstance(id, wakeUp) { logger.info(`${hostLogPrefix} Adapter ${id} needs rebuild and will be restarted afterwards.`); const msg = { command: 'rebuildAdapter', - message: { - id: instance._id, - rebuildViaInstall: procs[id].rebuildCounter > 1 - } + message: { id: instance._id } }; if (!compactGroupController) { // execute directly processMessage(msg); diff --git a/package-lock.json b/package-lock.json index b3ce0ad088..7245d8126d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -41,6 +41,34 @@ } } }, + "@alcalzone/pak": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/@alcalzone/pak/-/pak-0.6.0.tgz", + "integrity": "sha512-uX5Yl7GFFX5auw9ZLCuXZ7vuWYzHaOtXTvKWKWs8VSUqTIHtWe6Vdy+WAA3SbH9ZnjT8gcXW5Rvt4mGmjFmliQ==", + "requires": { + "axios": "^0.21.1", + "execa": "^5.0.0", + "fs-extra": "^9.1.0" + }, + "dependencies": { + "fs-extra": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.1.0.tgz", + "integrity": "sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==", + "requires": { + "at-least-node": "^1.0.0", + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + } + }, + "graceful-fs": { + "version": "4.2.8", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.8.tgz", + "integrity": "sha512-qkIilPUYcNhJpd33n0GBXTB1MMPp14TxEsEs0pTrsSVucApsYzW5V+Q8Qxhik6KU3evy+qkAAowTByymK0avdg==" + } + } + }, "@alcalzone/release-script": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/@alcalzone/release-script/-/release-script-2.2.1.tgz", @@ -71,6 +99,23 @@ "integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==", "dev": true }, + "execa": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", + "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", + "dev": true, + "requires": { + "cross-spawn": "^7.0.3", + "get-stream": "^6.0.0", + "human-signals": "^2.1.0", + "is-stream": "^2.0.0", + "merge-stream": "^2.0.0", + "npm-run-path": "^4.0.1", + "onetime": "^5.1.2", + "signal-exit": "^3.0.3", + "strip-final-newline": "^2.0.0" + } + }, "yargs": { "version": "17.0.1", "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.0.1.tgz", @@ -991,7 +1036,6 @@ "version": "0.21.1", "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.1.tgz", "integrity": "sha512-dKQiRHxGD9PPRIUNIWvZhPTPpl1rf/OxTYKsqKUDjBwYylTvV7SjSHJb9ratfyzM6wCdLCOYLzs73qpg5c4iGA==", - "dev": true, "requires": { "follow-redirects": "^1.10.0" } @@ -1603,7 +1647,6 @@ "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", - "dev": true, "requires": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", @@ -1614,7 +1657,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "dev": true, "requires": { "isexe": "^2.0.0" } @@ -2346,10 +2388,9 @@ } }, "execa": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", - "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", - "dev": true, + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/execa/-/execa-5.0.0.tgz", + "integrity": "sha512-ov6w/2LCiuyO4RLYGdpFGjkcs0wMTgGE8PrkTHikeUy5iJekXyPIKUjifk5CsE0pt7sMCrMZ3YNqoCj6idQOnQ==", "requires": { "cross-spawn": "^7.0.3", "get-stream": "^6.0.0", @@ -2728,10 +2769,9 @@ "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==" }, "follow-redirects": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.1.tgz", - "integrity": "sha512-HWqDgT7ZEkqRzBvc2s64vSZ/hfOceEol3ac/7tKwzuvEyWx3/4UegXh5oBOIotkGsObyk3xznnSRVADBgWSQVg==", - "dev": true + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.0.tgz", + "integrity": "sha512-0vRwd7RKQBTt+mgu87mtYeofLFZpTas2S9zY+jIeuLJMNvudIgF52nr19q40HOwH5RrhWIPuj9puybzSJiRrVg==" }, "for-in": { "version": "1.0.2", @@ -2875,8 +2915,7 @@ "get-stream": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", - "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", - "dev": true + "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==" }, "get-value": { "version": "2.0.6", @@ -4121,8 +4160,7 @@ "human-signals": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", - "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", - "dev": true + "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==" }, "i": { "version": "0.3.6", @@ -4451,8 +4489,7 @@ "isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", - "dev": true + "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=" }, "isobject": { "version": "3.0.1", @@ -5250,8 +5287,7 @@ "merge-stream": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", - "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", - "dev": true + "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==" }, "micromatch": { "version": "3.1.10", @@ -5295,8 +5331,7 @@ "mimic-fn": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", - "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", - "dev": true + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==" }, "minimatch": { "version": "3.0.4", @@ -5551,11 +5586,6 @@ "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=", "dev": true }, - "ncp": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz", - "integrity": "sha1-GVoh1sRuNh0vsSgbo4uR6d9727M=" - }, "neo-async": { "version": "2.6.1", "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.1.tgz", @@ -5672,7 +5702,6 @@ "version": "4.0.1", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", - "dev": true, "requires": { "path-key": "^3.0.0" } @@ -5817,7 +5846,6 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", - "dev": true, "requires": { "mimic-fn": "^2.1.0" } @@ -5955,8 +5983,7 @@ "path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", - "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", - "dev": true + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==" }, "path-parse": { "version": "1.0.7", @@ -6694,7 +6721,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "dev": true, "requires": { "shebang-regex": "^3.0.0" } @@ -6702,8 +6728,7 @@ "shebang-regex": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "dev": true + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==" }, "signal-exit": { "version": "3.0.3", @@ -7133,8 +7158,7 @@ "strip-final-newline": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", - "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", - "dev": true + "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==" }, "strip-json-comments": { "version": "3.1.1", diff --git a/package.json b/package.json index 047a49517f..43d9a31c61 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ }, "dependencies": { "@alcalzone/esbuild-register": "^2.5.1-1", + "@alcalzone/pak": "^0.6.0", "@iobroker/db-objects-file": "~1.2.7", "@iobroker/db-objects-jsonl": "~1.2.7", "@iobroker/db-objects-redis": "~1.2.7", @@ -38,7 +39,6 @@ "jszip": "^3.7.0", "loadavg-windows": "^1.1.1", "mime": "^2.5.2", - "ncp": "^2.0.0", "node-forge": "^0.10.0", "node-schedule": "^2.0.0", "node.extend": "^2.0.2",