Merge remote-tracking branch 'origin/develop' into hpce/merge

Author: Simon Moll

.github/lockdown.yml

@@ -0,0 +1,8 @@
+# Configuration for Repo Lockdown - https://github.com/dessant/repo-lockdown
+
+skipCreatedBefore: "2020-03-21"
+
+pulls:
+  comment: >
+    This repository does not accept pull requests.
+    Please follow http://llvm.org/docs/Contributing.html#how-to-submit-a-patch for contribution to LLVM.

.gitignore

@@ -63,3 +63,5 @@ pythonenv*
 /clang/utils/analyzer/projects/*/PatchedSource
 /clang/utils/analyzer/projects/*/ScanBuildResults
 /clang/utils/analyzer/projects/*/RefScanBuildResults
+# automodapi puts generated documentation files here.
+/lldb/docs/python_api/

Jenkinsfile

@@ -12,6 +12,11 @@ pipeline {
         REPO_TOP_URL = sh(
             returnStdout: true,
             script: "echo ${env.GIT_URL} | sed -e 's:/[^/]*/[^/]*\$::'").trim()
+
+        // Use VE 0 or 2.
+        VE_NODE_NUMBER = sh(
+            returnStdout: true,
+            script: "echo ${env.EXECUTOR_NUMBER} | sed -e 's:1:2:'").trim()
     }
 
     stages {
@@ -71,19 +76,19 @@ pipeline {
                         }
                     }
                 }
-                stage('Prepare vetfkernel') {
+                stage('Prepare vml') {
                     steps {
-                        dir('vetfkernel') {
+                        dir('vml') {
                             git branch: 'master',
                                 credentialsId: 'marukawa-token',
-                                url: "${REPO_TOP_URL}/ve-tensorflow/vetfkernel.git"
+                                url: "${REPO_TOP_URL}/ve-tensorflow/vml.git"
                         }
-                        dir('vetfkernel/libs/vednn') {
-                            git branch: 'vetfkernel',
+                        dir('vml/libs/vednn') {
+                            git branch: 'vml',
                                 credentialsId: 'marukawa-token',
                                 url: "${REPO_TOP_URL}/ve-tensorflow/vednn.git"
                         }
-                        dir('vetfkernel/build') {
+                        dir('vml/build') {
                             sh """
                                 ${CMAKE} -DCMAKE_BUILD_TYPE="Debug" \
                                     -DLLVM_DIR=${TOP}/llvm-dev/install/lib/cmake/llvm \
@@ -95,12 +100,12 @@ pipeline {
                 }
             }
         }
-        stage('Check vetfkernel') {
+        stage('Check vml') {
             steps {
-                dir('vetfkernel') {
+                dir('vml/build') {
                     sh """
-                        ./build/test/test01
-                        ${PYTHON} perf.py -e build/test/bench -d perfdb/10B \
+                        make test
+                        ${PYTHON} ../perf.py -e bench/bench -d ../perfdb/10B \
                             test
                     """
                 }

clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.cpp

@@ -128,7 +128,7 @@ void UpgradeDurationConversionsCheck::check(
 
   if (!match(isInTemplateInstantiation(), *OuterExpr, *Result.Context)
            .empty()) {
-    if (MatchedTemplateLocations.count(Loc.getRawEncoding()) == 0) {
+    if (MatchedTemplateLocations.count(Loc) == 0) {
       // For each location matched in a template instantiation, we check if the
       // location can also be found in `MatchedTemplateLocations`. If it is not
       // found, that means the expression did not create a match without the
@@ -144,7 +144,7 @@ void UpgradeDurationConversionsCheck::check(
   internal::Matcher<Stmt> IsInsideTemplate =
       hasAncestor(decl(anyOf(classTemplateDecl(), functionTemplateDecl())));
   if (!match(IsInsideTemplate, *ArgExpr, *Result.Context).empty())
-    MatchedTemplateLocations.insert(Loc.getRawEncoding());
+    MatchedTemplateLocations.insert(Loc);
 
   DiagnosticBuilder Diag = diag(Loc, Message);
   CharSourceRange SourceRange = Lexer::makeFileCharRange(

clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.h

@@ -11,7 +11,8 @@
 
 #include "../ClangTidyCheck.h"
 
-#include <unordered_set>
+#include "clang/Basic/SourceLocation.h"
+#include "llvm/ADT/DenseSet.h"
 
 namespace clang {
 namespace tidy {
@@ -32,7 +33,7 @@ class UpgradeDurationConversionsCheck : public ClangTidyCheck {
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
 
 private:
-  std::unordered_set<unsigned> MatchedTemplateLocations;
+  llvm::DenseSet<SourceLocation> MatchedTemplateLocations;
 };
 
 } // namespace abseil

clang-tools-extra/clangd/AST.cpp

@@ -116,6 +116,7 @@ getQualification(ASTContext &Context, const DeclContext *DestContext,
     if (auto *TD = llvm::dyn_cast<TagDecl>(CurContext)) {
       // There can't be any more tag parents after hitting a namespace.
       assert(!ReachedNS);
+      (void)ReachedNS;
       NNS = NestedNameSpecifier::Create(Context, nullptr, false,
                                         TD->getTypeForDecl());
     } else {
@@ -299,19 +300,27 @@ SymbolID getSymbolID(const llvm::StringRef MacroName, const MacroInfo *MI,
   return SymbolID(USR);
 }
 
-// FIXME: This should be handled while printing underlying decls instead.
 std::string printType(const QualType QT, const DeclContext &CurContext) {
   std::string Result;
   llvm::raw_string_ostream OS(Result);
-  auto Decls =
-      explicitReferenceTargets(DynTypedNode::create(QT), DeclRelation::Alias);
-  if (!Decls.empty())
-    OS << getQualification(CurContext.getParentASTContext(), &CurContext,
-                           Decls.front(),
-                           /*VisibleNamespaces=*/llvm::ArrayRef<std::string>{});
   PrintingPolicy PP(CurContext.getParentASTContext().getPrintingPolicy());
-  PP.SuppressScope = true;
   PP.SuppressTagKeyword = true;
+  PP.SuppressUnwrittenScope = true;
+
+  class PrintCB : public PrintingCallbacks {
+  public:
+    PrintCB(const DeclContext *CurContext) : CurContext(CurContext) {}
+    virtual ~PrintCB() {}
+    virtual bool isScopeVisible(const DeclContext *DC) const override {
+      return DC->Encloses(CurContext);
+    }
+
+  private:
+    const DeclContext *CurContext;
+  };
+  PrintCB PCB(&CurContext);
+  PP.Callbacks = &PCB;
+
   QT.print(OS, PP);
   return OS.str();
 }

clang-tools-extra/clangd/ASTSignals.cpp

@@ -0,0 +1,42 @@
+//===--- ASTSignals.cpp ------------------------------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "ASTSignals.h"
+#include "AST.h"
+#include "FindTarget.h"
+
+namespace clang {
+namespace clangd {
+ASTSignals ASTSignals::derive(const ParsedAST &AST) {
+  ASTSignals Signals;
+  const SourceManager &SM = AST.getSourceManager();
+  findExplicitReferences(AST.getASTContext(), [&](ReferenceLoc Ref) {
+    for (const NamedDecl *ND : Ref.Targets) {
+      if (!isInsideMainFile(Ref.NameLoc, SM))
+        continue;
+      SymbolID ID = getSymbolID(ND);
+      if (!ID)
+        continue;
+      unsigned &SymbolCount = Signals.ReferencedSymbols[ID];
+      SymbolCount++;
+      // Process namespace only when we see the symbol for the first time.
+      if (SymbolCount != 1)
+        continue;
+      if (const auto *NSD = dyn_cast<NamespaceDecl>(ND->getDeclContext())) {
+        if (NSD->isAnonymousNamespace())
+          continue;
+        std::string NS = printNamespaceScope(*NSD);
+        if (!NS.empty())
+          Signals.RelatedNamespaces[NS]++;
+      }
+    }
+  });
+  return Signals;
+}
+} // namespace clangd
+} // namespace clang

clang-tools-extra/clangd/ASTSignals.h

@@ -0,0 +1,39 @@
+//===--- ASTSignals.h --------------------------------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_ASTSIGNALS_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_ASTSIGNALS_H
+
+#include "ParsedAST.h"
+#include "index/SymbolID.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/StringMap.h"
+
+namespace clang {
+namespace clangd {
+
+/// Signals derived from a valid AST of a file.
+/// Provides information that can only be extracted from the AST to actions that
+/// can't access an AST. The signals are computed and updated asynchronously by
+/// the ASTWorker and thus they are always stale and also can be absent.
+/// Example usage: Information about the declarations used in a file affects
+/// code-completion ranking in that file.
+struct ASTSignals {
+  /// Number of occurrences of each symbol present in the file.
+  llvm::DenseMap<SymbolID, unsigned> ReferencedSymbols;
+  /// Namespaces whose symbols are used in the file, and the number of such
+  /// distinct symbols.
+  llvm::StringMap<unsigned> RelatedNamespaces;
+
+  static ASTSignals derive(const ParsedAST &AST);
+};
+
+} // namespace clangd
+} // namespace clang
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_ASTSIGNALS_H

clang-tools-extra/clangd/CMakeLists.txt

@@ -46,6 +46,7 @@ include_directories(BEFORE "${CMAKE_CURRENT_BINARY_DIR}/../clang-tidy")
 
 add_clang_library(clangDaemon
   AST.cpp
+  ASTSignals.cpp
   ClangdLSPServer.cpp
   ClangdServer.cpp
   CodeComplete.cpp

clang-tools-extra/clangd/ClangdLSPServer.cpp

@@ -620,15 +620,17 @@ void ClangdLSPServer::onInitialize(const InitializeParams &Params,
             {"documentSymbolProvider", true},
             {"workspaceSymbolProvider", true},
             {"referencesProvider", true},
-            {"astProvider", true},
+            {"astProvider", true}, // clangd extension
             {"executeCommandProvider",
              llvm::json::Object{
                  {"commands",
                   {ExecuteCommandParams::CLANGD_APPLY_FIX_COMMAND,
                    ExecuteCommandParams::CLANGD_APPLY_TWEAK}},
              }},
             {"typeHierarchyProvider", true},
-            {"memoryUsageProvider", true}, // clangd extension.
+            {"memoryUsageProvider", true}, // clangd extension
+            {"compilationDatabase",        // clangd extension
+             llvm::json::Object{{"automaticReload", true}}},
             {"callHierarchyProvider", true},
         }}}};
   if (Opts.Encoding)

clang-tools-extra/clangd/ClangdServer.cpp

@@ -243,22 +243,19 @@ void ClangdServer::codeComplete(PathRef File, Position Pos,
       // No speculation in Fallback mode, as it's supposed to be much faster
       // without compiling.
       vlog("Build for file {0} is not ready. Enter fallback mode.", File);
-    } else {
-      if (CodeCompleteOpts.Index && CodeCompleteOpts.SpeculativeIndexRequest) {
-        SpecFuzzyFind.emplace();
-        {
-          std::lock_guard<std::mutex> Lock(
-              CachedCompletionFuzzyFindRequestMutex);
-          SpecFuzzyFind->CachedReq =
-              CachedCompletionFuzzyFindRequestByFile[File];
-        }
+    } else if (CodeCompleteOpts.Index) {
+      SpecFuzzyFind.emplace();
+      {
+        std::lock_guard<std::mutex> Lock(CachedCompletionFuzzyFindRequestMutex);
+        SpecFuzzyFind->CachedReq = CachedCompletionFuzzyFindRequestByFile[File];
       }
     }
     ParseInputs ParseInput{IP->Command, &TFS, IP->Contents.str()};
     ParseInput.Index = Index;
     ParseInput.Opts.BuildRecoveryAST = BuildRecoveryAST;
     ParseInput.Opts.PreserveRecoveryASTType = PreserveRecoveryASTType;
 
+    CodeCompleteOpts.MainFileSignals = IP->Signals;
     // FIXME(ibiryukov): even if Preamble is non-null, we may want to check
     // both the old and the new version in case only one of them matches.
     CodeCompleteResult Result = clangd::codeComplete(
@@ -549,8 +546,8 @@ void ClangdServer::switchSourceHeader(
   //     the same directory.
   //  2) if 1) fails, we use the AST&Index approach, it is slower but supports
   //     different code layout.
-  if (auto CorrespondingFile = getCorrespondingHeaderOrSource(
-          std::string(Path), TFS.view(llvm::None)))
+  if (auto CorrespondingFile =
+          getCorrespondingHeaderOrSource(Path, TFS.view(llvm::None)))
     return CB(std::move(CorrespondingFile));
   auto Action = [Path = Path.str(), CB = std::move(CB),
                  this](llvm::Expected<InputsAndAST> InpAST) mutable {

clang-tools-extra/clangd/CodeComplete.cpp

@@ -277,7 +277,7 @@ struct CodeCompletionBuilder {
                         CodeCompletionContext::Kind ContextKind,
                         const CodeCompleteOptions &Opts,
                         bool IsUsingDeclaration, tok::TokenKind NextTokenKind)
-      : ASTCtx(ASTCtx), ExtractDocumentation(Opts.IncludeComments),
+      : ASTCtx(ASTCtx),
         EnableFunctionArgSnippets(Opts.EnableFunctionArgSnippets),
         IsUsingDeclaration(IsUsingDeclaration), NextTokenKind(NextTokenKind) {
     add(C, SemaCCS);
@@ -393,7 +393,7 @@ struct CodeCompletionBuilder {
       S.SnippetSuffix = std::string(C.IndexResult->CompletionSnippetSuffix);
       S.ReturnType = std::string(C.IndexResult->ReturnType);
     }
-    if (ExtractDocumentation && !Completion.Documentation) {
+    if (!Completion.Documentation) {
       auto SetDoc = [&](llvm::StringRef Doc) {
         if (!Doc.empty()) {
           Completion.Documentation.emplace();
@@ -512,7 +512,6 @@ struct CodeCompletionBuilder {
   ASTContext *ASTCtx;
   CodeCompletion Completion;
   llvm::SmallVector<BundledEntry, 1> Bundled;
-  bool ExtractDocumentation;
   bool EnableFunctionArgSnippets;
   // No snippets will be generated for using declarations and when the function
   // arguments are already present.
@@ -828,9 +827,9 @@ struct CompletionRecorder : public CodeCompleteConsumer {
 };
 
 struct ScoredSignature {
-  // When set, requires documentation to be requested from the index with this
-  // ID.
-  llvm::Optional<SymbolID> IDForDoc;
+  // When not null, requires documentation to be requested from the index with
+  // this ID.
+  SymbolID IDForDoc;
   SignatureInformation Signature;
   SignatureQualitySignals Quality;
 };
@@ -894,7 +893,7 @@ class SignatureHelpCollector final : public CodeCompleteConsumer {
       for (const auto &S : ScoredSignatures) {
         if (!S.IDForDoc)
           continue;
-        IndexRequest.IDs.insert(*S.IDForDoc);
+        IndexRequest.IDs.insert(S.IDForDoc);
       }
       Index->lookup(IndexRequest, [&](const Symbol &S) {
         if (!S.Documentation.empty())
@@ -939,7 +938,7 @@ class SignatureHelpCollector final : public CodeCompleteConsumer {
 
     for (auto &SS : ScoredSignatures) {
       auto IndexDocIt =
-          SS.IDForDoc ? FetchedDocs.find(*SS.IDForDoc) : FetchedDocs.end();
+          SS.IDForDoc ? FetchedDocs.find(SS.IDForDoc) : FetchedDocs.end();
       if (IndexDocIt != FetchedDocs.end())
         SS.Signature.documentation = IndexDocIt->second;
 
@@ -1111,8 +1110,8 @@ bool semaCodeComplete(std::unique_ptr<CodeCompleteConsumer> Consumer,
       offsetToClangLineColumn(Input.ParseInput.Contents, Input.Offset);
 
   std::unique_ptr<llvm::MemoryBuffer> ContentsBuffer =
-      llvm::MemoryBuffer::getMemBufferCopy(Input.ParseInput.Contents,
-                                           Input.FileName);
+      llvm::MemoryBuffer::getMemBuffer(Input.ParseInput.Contents,
+                                       Input.FileName);
   // The diagnostic options must be set before creating a CompilerInstance.
   CI->getDiagnosticOpts().IgnoreWarnings = true;
   // We reuse the preamble whether it's valid or not. This is a
@@ -1686,6 +1685,7 @@ class CodeCompleteFlow {
     if (PreferredType)
       Relevance.HadContextType = true;
     Relevance.ContextWords = &ContextWords;
+    Relevance.MainFileSignals = Opts.MainFileSignals;
 
     auto &First = Bundle.front();
     if (auto FuzzyScore = fuzzyScore(First))
@@ -1765,8 +1765,8 @@ class CodeCompleteFlow {
 
 clang::CodeCompleteOptions CodeCompleteOptions::getClangCompleteOpts() const {
   clang::CodeCompleteOptions Result;
-  Result.IncludeCodePatterns = EnableSnippets && IncludeCodePatterns;
-  Result.IncludeMacros = IncludeMacros;
+  Result.IncludeCodePatterns = EnableSnippets;
+  Result.IncludeMacros = true;
   Result.IncludeGlobals = true;
   // We choose to include full comments and not do doxygen parsing in
   // completion.