v0.3.0

.gitignore

@@ -1,2 +1,3 @@
 node_modules
 .npmrc
+logs

.npmignore

@@ -1,2 +1,3 @@
 .npmrc
-node_modules
+node_modules
+logs

README.md

@@ -19,7 +19,7 @@ CodeQL Agent CLI is a tool that automates the process of using CodeQL, a semanti
 ## Features
 
 - Automated CodeQL from detect language, create database and scan.
-- Scan remote target (e.g. GitHub repository) or local target (e.g. source code folder).
+- Scan remote target (e.g. GitHub repository) or local target (e.g. source code folder). Support scan list of target.
 - Support running on Docker which prepackaged and precompiled CodeQL for running code scanning (*under development*).
 - Send results to Discord webhook.
 
@@ -65,23 +65,33 @@ codeql-agent scan -h
 This will display help for the tool. Here are all the switches of `scan` command supports.
 
 ```console
+   ____          _       ___  _          _                    _   
+  / ___|___   __| | ___ / _ \| |        / \   __ _  ___ _ __ | |_ 
+ | |   / _ \ / _` |/ _ \ | | | |       / _ \ / _` |/ _ \ '_ \| __|
+ | |__| (_) | (_| |  __/ |_| | |___   / ___ \ (_| |  __/ | | | |_ 
+  \____\___/ \__,_|\___|\__\_\_____| /_/   \_\__, |\___|_| |_|\__|
+                                             |___/                
+        Author: doublevkay - Version: 0.3.0
+
 Usage: codeql-agent scan [options] <target>
 
-scan a source code folder or remote repository (e.g. GitHub repository)
+scan a target. Target could be source code folder, remote repository (e.g. GitHub repository) or a list of target.
 
 Arguments:
-  target                          source code folder or remote repository.
+  target                          source code folder, remote repository or list of target.
 
   Examples:
         codeql-agent scan src/sammple
-        codeql-agent scan src/sammple --use-docker
+        codeql-agent scan targets.txt
         codeql-agent scan https://github.com/OWASP/NodeGoat
 
 Options:
-  -l, --language <language>       language of source code. Supported languages: go, java, cpp, csharp, cpp, javascript, ruby. Omitting this option to auto-detect the language.
+  -l, --language <language>       language of source code. Supported languages: go, java, cpp, csharp, cpp, javascript, ruby. Omitting this option to auto-detect the
+                                  language.
   -o, --output <output>           output folder. Default: <target>-codeql-results
-  -c, --command <command>         command to create database for compiled languages, omit if the only languages requested are Python and JavaScript. This specifies the build commands needed to invoke the compiler. If
-                                  you don't set this variable, CodeQL will attempt to detect the build system automatically, using a built-in autobuilder
+  -c, --command <command>         command to create database for compiled languages, omit if the only languages requested are Python and JavaScript. This specifies
+                                  the build commands needed to invoke the compiler. If you don't set this variable, CodeQL will attempt to detect the build system
+                                  automatically, using a built-in autobuilder
   -t, --threads <number>          number of threads to use. Pass 0 to use one threads per core on the machine. Default: 1 (default: 1)
   --query <query>                 CodeQL query to run. Default: <language>-security-extended.qls
   --format <format>               output format. Default: sarif-latest (default: "sarif-latest")

cli.js

@@ -26,8 +26,8 @@ program
     .version(config.version);
 
 program.command('scan')
-    .description('scan a source code folder or remote repository (e.g. GitHub repository)')
-    .argument('<target>', 'source code folder or remote repository.\n\nExamples:\n\tcodeql-agent scan src/sammple\n\tcodeql-agent scan src/sammple --use-docker\n\tcodeql-agent scan https://github.com/OWASP/NodeGoat')
+    .description('scan a target. Target could be source code folder, remote repository (e.g. GitHub repository) or a list of target.')
+    .argument('<target>', 'source code folder, remote repository or list of target.\n\nExamples:\n\tcodeql-agent scan src/sammple\n\tcodeql-agent scan targets.txt\n\tcodeql-agent scan https://github.com/OWASP/NodeGoat')
     .option('-l, --language <language>', 'language of source code. Supported languages: go, java, cpp, csharp, cpp, javascript, ruby. Omitting this option to auto-detect the language.',)
     .option('-o, --output <output>', 'output folder. Default: <target>-codeql-results')
     .option('-c, --command <command>', 'command to create database for compiled languages, omit if the only languages requested are Python and JavaScript. This specifies the build commands needed to invoke the compiler. If you don\'t set this variable, CodeQL will attempt to detect the build system automatically, using a built-in autobuilder')

index.js

@@ -3,62 +3,69 @@ const utils = require('./utils');
 const fs = require('fs');
 const config = require('./config');
 const path = require('path');
+const pLimit = require('p-limit');
 
 module.exports = {
     scan: scanAction
 }
 
 async function scanAction(sourceTarget, options) {
     if (options.verbose) { defaultLogger.setLevel('verbose') }
+    if (options.enableFileLogging) { defaultLogger.enableFileTransport() }
+    if (options.discordWebhook) { bugLogger.enableDiscordTransport(options.discordWebhook) }
+    options.enableFileLogging = undefined
+    options.discordWebhook = undefined
     if (options.useDocker) {
         await utils.isCommandExist('docker', defaultLogger);
         defaultLogger.error('Docker is not supported yet.');
         return;
     }
-    if (options.enableFileLogging) { defaultLogger.enableFileTransport() }
-    if (options.discordWebhook) { bugLogger.enableDiscordTransport(options.discordWebhook) }
     await utils.isCommandExist('codeql', defaultLogger);
-    // Create Database
-    var createDbOptions = { ...options };
-    createDbOptions.output = options.dbOutput;
+    if (fs.existsSync(sourceTarget) && fs.statSync(sourceTarget).isFile()) {
+        const targets = fs.readFileSync(sourceTarget, 'utf8').split(/\r?\n/);
+        const limit = pLimit(1);
+        return await Promise.all(targets.map((target) => limit(() => scanAction(target, options))));
+    }
     var isRemoteRepository = utils.isRemoteRepository(sourceTarget);
+    var sourceFolderPath = '';
     if (isRemoteRepository) {
         defaultLogger.info(`Cloning remote repository ${sourceTarget}`)
         sourceFolderPath = await utils.cloneRemoteRepository(sourceTarget, defaultLogger);
     } else sourceFolderPath = sourceTarget;
+    if (!fs.existsSync(sourceFolderPath)) {
+        defaultLogger.error(`Source folder \`${sourceFolderPath}\` does not exist.`);
+        return [];
+    }
     sourceFolderPath = fs.realpathSync(sourceFolderPath);
     defaultLogger.info(`Creating CodeQL database for ${sourceFolderPath}...`)
+
+    // Create Database
+    var createDbOptions = { ...options };
+    createDbOptions.output = options.dbOutput;
     var { args: createDbArgs, databasePath } = await utils.setupCreateDatabaseCommandArgs(sourceFolderPath, createDbOptions, defaultLogger);
     defaultLogger.verbose(`Options:`);
     for (const key in options) {
         const element = options[key];
         defaultLogger.verbose(`[+] ${key}: ${element}`);
     }
     createDbExitCode = await utils.executeCommand('codeql', createDbArgs, 'Create CodeQL database', defaultLogger);
-    defaultLogger.info(`CodeQL database created at ${databasePath}.`)
-    if (isRemoteRepository && options.removeRemoteRepository) {
-        defaultLogger.info(`Removing remote repository ${sourceFolderPath}`)
-        await utils.removeFolder(sourceFolderPath, defaultLogger);
-    }
     if (options.createDbOnly) {
         return databasePath;
     }
+
     // Scan Database
     const outputFolderPath = options.output ? options.output : path.resolve(`${databasePath.endsWith('-codeql-database') ? databasePath.slice(0, -16) : databasePath}-codeql-results`);
     if (!fs.existsSync(outputFolderPath)) {
         fs.mkdirSync(outputFolderPath);
     }
     const languages = await utils.getDatabaseLanguages(databasePath, defaultLogger);
-    if (!languages) {
-        defaultLogger.error('Can not detect languages. Please specify the language using --language option');
-        return;
-    }
     for (const language of languages) {
         options.language = language;
         languageDatabasePath = path.resolve(`${databasePath}${path.sep}${language}`);
-        options.output = path.resolve(outputFolderPath, `${language}-codeql-result.sarif`)
+        const scanOption = { ...options };
+        scanOption.output = path.resolve(outputFolderPath, `${language}-codeql-result.sarif`)
         defaultLogger.info(`Scanning ${language} code in ${databasePath}...`)
-        var { args: scanArgs } = await utils.setupScanCommandArgs(languageDatabasePath, options, defaultLogger);
+        var { args: scanArgs } = await utils.setupScanCommandArgs(languageDatabasePath, scanOption, defaultLogger);
         await utils.executeCommand('codeql', scanArgs, 'Scan CodeQL database', defaultLogger);
     }
     defaultLogger.info(`CodeQL scan results saved at ${outputFolderPath}.`)
@@ -78,9 +85,14 @@ async function scanAction(sourceTarget, options) {
             meta: alert
         });
     }
+    if (isRemoteRepository && options.removeRemoteRepository) {
+        defaultLogger.verbose(`Removing remote repository ${sourceFolderPath}`)
+        utils.removeFolder(sourceFolderPath, defaultLogger);
+    }
     if (options.removeDatabase) {
-        defaultLogger.info(`Removing database folder ${databasePath}`)
-        await utils.removeFolder(databasePath, defaultLogger);
+        defaultLogger.verbose(`Removing database folder ${databasePath}`)
+        utils.removeFolder(databasePath, defaultLogger);
     }
     return alerts;
 }
+

package.json

@@ -1,6 +1,6 @@
 {
   "name": "codeql-agent",
-  "version": "0.2.2",
+  "version": "0.3.0",
   "description": "A CodeQL tool to automatically execute code scanning.",
   "main": "index.js",
   "scripts": {
@@ -36,6 +36,7 @@
     "commander": "^10.0.0",
     "figlet": "^1.5.2",
     "linguist-js": "^2.5.4",
+    "p-limit": "^3.1.0",
     "path": "^0.12.7",
     "which": "^3.0.0",
     "winston": "^3.8.2",
@@ -47,4 +48,4 @@
     "ts-node": "^10.9.1",
     "typescript": "^4.9.4"
   }
-}
+}

utils.js

@@ -84,7 +84,7 @@ async function executeCommand(commandPath, commandArgs, description, logger) {
         result.stderr?.on('data', function (data) {
             if (data.includes('A fatal error occurred')) {
                 logger.error(`${description} failed: ${data} `);
-                process.exit(1);
+                return -1;
             }
             logger.verbose(`${description}: ${data.toString().trim()}`);
         });
@@ -103,7 +103,7 @@ async function executeCommand(commandPath, commandArgs, description, logger) {
         return exitCode;
     } catch (err) {
         logger.error(`${description} failed: ${err.stderr || err} `);
-        process.exit(1);
+        return -1;
     }
 }
 
@@ -271,13 +271,21 @@ async function cloneRemoteRepository(target, logger) {
  * @param {object} logger - A logger instance
  * @return {void}
 */
-async function removeFolder(folderPath, logger) {
+function removeFolder(folderPath, logger) {
     const fs = require('fs');
+    // check if exists
     try {
-        await fs.rmSync(folderPath, { recursive: true, force: true });
-    }
-    catch (error) {
-        logger.warning(`${error.message}`);
+        if (fs.existsSync(folderPath)) {
+            fs.rm(folderPath, { recursive: true, force: true, maxRetries: 10 }, (err) => {
+                if (err) {
+                    logger.warn(`Failed to remove folder ${folderPath}.`);
+                }
+            });
+        }
+
+
+    } catch (error) {
+        logger.warn(`Failed to remove folder ${folderPath}.`);
     }
 }