Fix #11750 (refactor ctu-info to generate fewer artifacts on disk) (#6778)

Author: danmar

addons/cppcheckdata.py

@@ -1688,11 +1688,14 @@ def reportError(location, severity, message, addon, errorId, extra='', columnOve
         EXIT_CODE = 1
 
 def reportSummary(dumpfile, summary_type, summary_data):
-    # dumpfile ends with ".dump"
-    ctu_info_file = dumpfile[:-4] + "ctu-info"
-    with open(ctu_info_file, 'at') as f:
-        msg = {'summary': summary_type, 'data': summary_data}
-        f.write(json.dumps(msg) + '\n')
+    msg = {'summary': summary_type, 'data': summary_data}
+    if '--cli' in sys.argv:
+        sys.stdout.write(json.dumps(msg) + '\n')
+    else:
+        # dumpfile ends with ".dump"
+        ctu_info_file = dumpfile[:-4] + "ctu-info"
+        with open(ctu_info_file, 'at') as f:
+            f.write(json.dumps(msg) + '\n')
 
 
 def get_path_premium_addon():

addons/test/util.py

@@ -38,6 +38,8 @@ def convert_json_output(raw_json_strings):
     """Convert raw stdout/stderr cppcheck JSON output to python dict."""
     json_output = {}
     for line in raw_json_strings:
+        if line.startswith('{"summary":'):
+            continue
         try:
             json_line = json.loads(line)
             #  json_output[json_line['errorId']] = json_line

cli/cppcheckexecutor.cpp

@@ -129,6 +129,10 @@ namespace {
             return !mCriticalErrors.empty();
         }
 
+        const std::string& getCtuInfo() const {
+            return mCtuInfo;
+        }
+
     private:
         /**
          * Information about progress is directed here. This should be
@@ -170,9 +174,14 @@ namespace {
         std::set<std::string> mActiveCheckers;
 
         /**
-         * True if there are critical errors
+         * List of critical errors
          */
         std::string mCriticalErrors;
+
+        /**
+         * CTU information
+         */
+        std::string mCtuInfo;
     };
 }
 
@@ -286,7 +295,7 @@ int CppCheckExecutor::check_internal(const Settings& settings) const
 #endif
     }
 
-    returnValue |= cppcheck.analyseWholeProgram(settings.buildDir, mFiles, mFileSettings);
+    returnValue |= cppcheck.analyseWholeProgram(settings.buildDir, mFiles, mFileSettings, stdLogger.getCtuInfo());
 
     if (settings.severity.isEnabled(Severity::information) || settings.checkConfiguration) {
         const bool err = reportSuppressions(settings, suppressions, settings.checks.isEnabled(Checks::unusedFunction), mFiles, mFileSettings, stdLogger);
@@ -424,6 +433,11 @@ void StdLogger::reportErr(const ErrorMessage &msg)
         return;
     }
 
+    if (msg.severity == Severity::internal && msg.id == "ctuinfo") {
+        mCtuInfo += msg.shortMessage() + "\n";
+        return;
+    }
+
     if (ErrorLogger::isCriticalErrorId(msg.id) && mCriticalErrors.find(msg.id) == std::string::npos) {
         if (!mCriticalErrors.empty())
             mCriticalErrors += ", ";

gui/checkthread.cpp

@@ -117,10 +117,11 @@ void CheckThread::check(const Settings &settings)
     start();
 }
 
-void CheckThread::analyseWholeProgram(const QStringList &files)
+void CheckThread::analyseWholeProgram(const QStringList &files, const std::string& ctuInfo)
 {
     mFiles = files;
     mAnalyseWholeProgram = true;
+    mCtuInfo = ctuInfo;
     start();
 }
 
@@ -131,12 +132,14 @@ void CheckThread::run()
 
     if (!mFiles.isEmpty() || mAnalyseWholeProgram) {
         mAnalyseWholeProgram = false;
+        std::string ctuInfo;
+        ctuInfo.swap(mCtuInfo);
         qDebug() << "Whole program analysis";
         std::list<FileWithDetails> files2;
         std::transform(mFiles.cbegin(), mFiles.cend(), std::back_inserter(files2), [&](const QString& file) {
             return FileWithDetails{file.toStdString(), 0};
         });
-        mCppcheck.analyseWholeProgram(mCppcheck.settings().buildDir, files2, {});
+        mCppcheck.analyseWholeProgram(mCppcheck.settings().buildDir, files2, {}, ctuInfo);
         mFiles.clear();
         emit done();
         return;

gui/checkthread.h

@@ -60,8 +60,9 @@ class CheckThread : public QThread {
     /**
      * @brief Run whole program analysis
      * @param files    All files
+     * @param ctuInfo  Ctu info for addons
      */
-    void analyseWholeProgram(const QStringList &files);
+    void analyseWholeProgram(const QStringList &files, const std::string& ctuInfo);
 
     void setAddonsAndTools(const QStringList &addonsAndTools) {
         mAddonsAndTools = addonsAndTools;
@@ -143,6 +144,7 @@ class CheckThread : public QThread {
 
     QStringList mFiles;
     bool mAnalyseWholeProgram{};
+    std::string mCtuInfo;
     QStringList mAddonsAndTools;
     QStringList mClangIncludePaths;
     QList<SuppressionList::Suppression> mSuppressions;

gui/threadhandler.cpp

@@ -52,6 +52,7 @@ void ThreadHandler::clearFiles()
     mLastFiles.clear();
     mResults.clearFiles();
     mAnalyseWholeProgram = false;
+    mCtuInfo.clear();
     mAddonsAndTools.clear();
     mSuppressions.clear();
 }
@@ -102,6 +103,8 @@ void ThreadHandler::check(const Settings &settings)
             addonsAndTools << s;
     }
 
+    mCtuInfo.clear();
+
     for (int i = 0; i < mRunningThreadCount; i++) {
         mThreads[i]->setAddonsAndTools(addonsAndTools);
         mThreads[i]->setSuppressions(mSuppressions);
@@ -164,8 +167,9 @@ void ThreadHandler::removeThreads()
 void ThreadHandler::threadDone()
 {
     if (mRunningThreadCount == 1 && mAnalyseWholeProgram) {
-        mThreads[0]->analyseWholeProgram(mLastFiles);
+        mThreads[0]->analyseWholeProgram(mLastFiles, mCtuInfo);
         mAnalyseWholeProgram = false;
+        mCtuInfo.clear();
         return;
     }
 
@@ -187,6 +191,7 @@ void ThreadHandler::stop()
 {
     mCheckStartTime = QDateTime();
     mAnalyseWholeProgram = false;
+    mCtuInfo.clear();
     for (CheckThread* thread : mThreads) {
         thread->stop();
     }

gui/threadhandler.h

@@ -257,6 +257,7 @@ protected slots:
     int mRunningThreadCount{};
 
     bool mAnalyseWholeProgram{};
+    std::string mCtuInfo;
 
     QStringList mAddonsAndTools;
     QList<SuppressionList::Suppression> mSuppressions;

lib/cppcheck.cpp

@@ -139,7 +139,7 @@ static std::string getDumpFileName(const Settings& settings, const std::string&
         return settings.dumpFile;
 
     std::string extension;
-    if (settings.dump)
+    if (settings.dump || !settings.buildDir.empty())
         extension = ".dump";
     else
         extension = "." + std::to_string(settings.pid) + ".dump";
@@ -167,7 +167,7 @@ static void createDumpFile(const Settings& settings,
     if (!fdump.is_open())
         return;
 
-    {
+    if (!settings.buildDir.empty()) {
         std::ofstream fout(getCtuInfoFileName(dumpFile));
     }
 
@@ -1419,8 +1419,7 @@ void CppCheck::executeAddons(const std::vector<std::string>& files, const std::s
     std::string fileList;
 
     if (files.size() >= 2 || endsWith(files[0], ".ctu-info")) {
-        fileList = Path::getPathFromFilename(files[0]) + FILELIST + std::to_string(mSettings.pid);
-        filesDeleter.addFile(fileList);
+        fileList = Path::getPathFromFilename(files[0]) + FILELIST;
         std::ofstream fout(fileList);
         for (const std::string& f: files)
             fout << f << std::endl;
@@ -1429,6 +1428,8 @@ void CppCheck::executeAddons(const std::vector<std::string>& files, const std::s
     // ensure all addons have already been resolved - TODO: remove when settings are const after creation
     assert(mSettings.addonInfos.size() == mSettings.addons.size());
 
+    std::string ctuInfo;
+
     for (const AddonInfo &addonInfo : mSettings.addonInfos) {
         if (addonInfo.name != "misra" && !addonInfo.ctu && endsWith(files.back(), ".ctu-info"))
             continue;
@@ -1445,6 +1446,18 @@ void CppCheck::executeAddons(const std::vector<std::string>& files, const std::s
 
             ErrorMessage errmsg;
 
+            if (obj.count("summary") > 0) {
+                if (!mSettings.buildDir.empty()) {
+                    ctuInfo += res.serialize() + "\n";
+                } else {
+                    errmsg.severity = Severity::internal;
+                    errmsg.id = "ctuinfo";
+                    errmsg.setmsg(res.serialize());
+                    reportErr(errmsg);
+                }
+                continue;
+            }
+
             if (obj.count("file") > 0) {
                 std::string fileName = obj["file"].get<std::string>();
                 const int64_t lineNumber = obj["linenr"].get<int64_t>();
@@ -1483,13 +1496,30 @@ void CppCheck::executeAddons(const std::vector<std::string>& files, const std::s
             reportErr(errmsg);
         }
     }
+
+    if (!mSettings.buildDir.empty() && fileList.empty()) {
+        const std::string& ctuInfoFile = getCtuInfoFileName(files[0]);
+        std::ofstream fout(ctuInfoFile);
+        fout << ctuInfo;
+    }
 }
 
-void CppCheck::executeAddonsWholeProgram(const std::list<FileWithDetails> &files, const std::list<FileSettings>& fileSettings)
+void CppCheck::executeAddonsWholeProgram(const std::list<FileWithDetails> &files, const std::list<FileSettings>& fileSettings, const std::string& ctuInfo)
 {
     if (mSettings.addons.empty())
         return;
 
+    if (mSettings.buildDir.empty()) {
+        const std::string fileName = std::to_string(mSettings.pid) + ".ctu-info";
+        FilesDeleter filesDeleter;
+        filesDeleter.addFile(fileName);
+        std::ofstream fout(fileName);
+        fout << ctuInfo;
+        fout.close();
+        executeAddons({fileName}, "");
+        return;
+    }
+
     std::vector<std::string> ctuInfoFiles;
     for (const auto &f: files) {
         const std::string &dumpFileName = getDumpFileName(mSettings, f.path());
@@ -1507,11 +1537,6 @@ void CppCheck::executeAddonsWholeProgram(const std::list<FileWithDetails> &files
         const ErrorMessage errmsg = ErrorMessage::fromInternalError(e, nullptr, "", "Bailing out from analysis: Whole program analysis failed");
         reportErr(errmsg);
     }
-
-    if (mSettings.buildDir.empty()) {
-        for (const std::string &f: ctuInfoFiles)
-            std::remove(f.c_str());
-    }
 }
 
 Settings &CppCheck::settings()
@@ -1807,13 +1832,9 @@ bool CppCheck::analyseWholeProgram()
     return errors && (mExitCode > 0);
 }
 
-unsigned int CppCheck::analyseWholeProgram(const std::string &buildDir, const std::list<FileWithDetails> &files, const std::list<FileSettings>& fileSettings)
+unsigned int CppCheck::analyseWholeProgram(const std::string &buildDir, const std::list<FileWithDetails> &files, const std::list<FileSettings>& fileSettings, const std::string& ctuInfo)
 {
-    executeAddonsWholeProgram(files, fileSettings);
-    if (buildDir.empty()) {
-        removeCtuInfoFiles(files, fileSettings);
-        return mExitCode;
-    }
+    executeAddonsWholeProgram(files, fileSettings, ctuInfo);
     if (mSettings.checks.isEnabled(Checks::unusedFunction))
         CheckUnusedFunctions::analyseWholeProgram(mSettings, *this, buildDir);
     std::list<Check::FileInfo*> fileInfoList;
@@ -1877,22 +1898,6 @@ unsigned int CppCheck::analyseWholeProgram(const std::string &buildDir, const st
     return mExitCode;
 }
 
-void CppCheck::removeCtuInfoFiles(const std::list<FileWithDetails> &files, const std::list<FileSettings>& fileSettings)
-{
-    if (mSettings.buildDir.empty()) {
-        for (const auto& f: files) {
-            const std::string &dumpFileName = getDumpFileName(mSettings, f.path());
-            const std::string &ctuInfoFileName = getCtuInfoFileName(dumpFileName);
-            std::remove(ctuInfoFileName.c_str());
-        }
-        for (const auto& fs: fileSettings) {
-            const std::string &dumpFileName = getDumpFileName(mSettings, fs.filename());
-            const std::string &ctuInfoFileName = getCtuInfoFileName(dumpFileName);
-            std::remove(ctuInfoFileName.c_str());
-        }
-    }
-}
-
 // cppcheck-suppress unusedFunction - only used in tests
 void CppCheck::resetTimerResults()
 {

lib/cppcheck.h

@@ -145,11 +145,8 @@ class CPPCHECKLIB CppCheck : ErrorLogger {
     /** Analyze all files using clang-tidy */
     void analyseClangTidy(const FileSettings &fileSettings);
 
-    /** analyse whole program use .analyzeinfo files */
-    unsigned int analyseWholeProgram(const std::string &buildDir, const std::list<FileWithDetails> &files, const std::list<FileSettings>& fileSettings);
-
-    /** Remove *.ctu-info files */
-    void removeCtuInfoFiles(const std::list<FileWithDetails>& files, const std::list<FileSettings>& fileSettings); // cppcheck-suppress functionConst // has side effects
+    /** analyse whole program use .analyzeinfo files or ctuinfo string */
+    unsigned int analyseWholeProgram(const std::string &buildDir, const std::list<FileWithDetails> &files, const std::list<FileSettings>& fileSettings, const std::string& ctuInfo);
 
     static void resetTimerResults();
     static void printTimerResults(SHOWTIME_MODES mode);
@@ -196,7 +193,7 @@ class CPPCHECKLIB CppCheck : ErrorLogger {
     /**
      * Execute addons
      */
-    void executeAddonsWholeProgram(const std::list<FileWithDetails> &files, const std::list<FileSettings>& fileSettings);
+    void executeAddonsWholeProgram(const std::list<FileWithDetails> &files, const std::list<FileSettings>& fileSettings, const std::string& ctuInfo);
 
 #ifdef HAVE_RULES
     /**