diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs index f55abb0f6..be2bfdd9d 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs @@ -36,6 +36,7 @@ internal sealed partial class ExpressionEval : IExpressionEvaluator { private readonly Stack _openScopes = new Stack(); private readonly object _lock = new object(); private readonly List _diagnostics = new List(); + private readonly AnalysisOptions _analysisOptions; public ExpressionEval(IServiceContainer services, IPythonModule module, PythonAst ast) { Services = services ?? throw new ArgumentNullException(nameof(services)); @@ -46,6 +47,18 @@ public ExpressionEval(IServiceContainer services, IPythonModule module, PythonAs CurrentScope = GlobalScope; DefaultLocation = new Location(module); //Log = services.GetService(); + + var optionsOptovider = services.GetService(); + if (optionsOptovider != null) { + _analysisOptions = optionsOptovider.Options; + } else { + _analysisOptions = new AnalysisOptions { + AnalysisCachingLevel = AnalysisCachingLevel.None, + StubOnlyAnalysis = true, + KeepLibraryAst = false, + LintingEnabled = true + }; + } } public GlobalScope GlobalScope { get; } @@ -56,7 +69,11 @@ public ExpressionEval(IServiceContainer services, IPythonModule module, PythonAs public IPythonType UnknownType => Interpreter.UnknownType; public Location DefaultLocation { get; } public IPythonModule BuiltinsModule => Interpreter.ModuleResolution.BuiltinsModule; - + + public bool StubOnlyAnalysis + => _analysisOptions.StubOnlyAnalysis + && Module.Stub != null && Module.ModuleType != ModuleType.User && Module.Stub.IsTypeshed; + public LocationInfo GetLocationInfo(Node node) => node?.GetLocation(this) ?? LocationInfo.Empty; public Location GetLocationOfName(Node node) { diff --git a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs index 201f7660c..d0f05732c 100644 --- a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs +++ b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs @@ -79,9 +79,7 @@ private void HandleAugmentedAllAssign(AugmentedAssignStatement node) { } var rightVar = Eval.GetValueFromExpression(node.Right); - var right = rightVar as IPythonCollection; - - if (right == null) { + if (!(rightVar is IPythonCollection right)) { _allIsUsable = false; return; } @@ -202,7 +200,6 @@ public void Complete() { _cancellationToken.ThrowIfCancellationRequested(); SymbolTable.EvaluateAll(); - SymbolTable.ReplacedByStubs.Clear(); new StubMerger(Eval).MergeStub(_stubAnalysis, _cancellationToken); if (_allIsUsable && _allReferencesCount >= 1 && GlobalScope.Variables.TryGetVariable(AllVariableName, out var variable) diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/ClassEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/ClassEvaluator.cs index cfc515212..422494537 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/ClassEvaluator.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/ClassEvaluator.cs @@ -72,36 +72,41 @@ private void ProcessClassBody() { SymbolTable.Evaluate(b.ClassDefinition); } - // Process imports - foreach (var s in GetStatements(_classDef)) { - ImportHandler.HandleFromImport(s); - } - foreach (var s in GetStatements(_classDef)) { - ImportHandler.HandleImport(s); - } - UpdateClassMembers(); + if (!Eval.StubOnlyAnalysis) { + // Process imports + foreach (var s in GetStatements(_classDef)) { + ImportHandler.HandleFromImport(s); + } - // Process assignments so we get class variables declared. - // Note that annotated definitions and assignments can be intermixed - // and must be processed in order. Consider - // class A: - // x: int - // x = 1 - foreach (var s in GetStatements(_classDef)) { - switch (s) { - case AssignmentStatement assignment: - AssignmentHandler.HandleAssignment(assignment, LookupOptions.All); - break; - case ExpressionStatement e: - AssignmentHandler.HandleAnnotatedExpression(e.Expression as ExpressionWithAnnotation, null, LookupOptions.All); - break; + foreach (var s in GetStatements(_classDef)) { + ImportHandler.HandleImport(s); } - } - UpdateClassMembers(); - // Ensure constructors are processed so class members are initialized. - EvaluateConstructors(_classDef); - UpdateClassMembers(); + UpdateClassMembers(); + + // Process assignments so we get class variables declared. + // Note that annotated definitions and assignments can be intermixed + // and must be processed in order. Consider + // class A: + // x: int + // x = 1 + foreach (var s in GetStatements(_classDef)) { + switch (s) { + case AssignmentStatement assignment: + AssignmentHandler.HandleAssignment(assignment, LookupOptions.All); + break; + case ExpressionStatement e: + AssignmentHandler.HandleAnnotatedExpression(e.Expression as ExpressionWithAnnotation, null, LookupOptions.All); + break; + } + } + + UpdateClassMembers(); + + // Ensure constructors are processed so class members are initialized. + EvaluateConstructors(_classDef); + UpdateClassMembers(); + } // Process remaining methods. SymbolTable.EvaluateScope(_classDef); diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs index 39c7b128a..eeb7bfcad 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs @@ -46,9 +46,11 @@ public FunctionEvaluator(ExpressionEval eval, PythonFunctionOverload overload) private FunctionDefinition FunctionDefinition { get; } public override void Evaluate() { - var stub = SymbolTable.ReplacedByStubs.Contains(Target) - || _function.DeclaringModule.ModuleType == ModuleType.Stub - || Module.ModuleType == ModuleType.Specialized; + if(Eval.StubOnlyAnalysis) { + return; + } + + var stub = _function.DeclaringModule.ModuleType == ModuleType.Stub; using (Eval.OpenScope(_function.DeclaringModule, FunctionDefinition, out _)) { var returnType = TryDetermineReturnValue(); diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/ModuleSymbolTable.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/ModuleSymbolTable.cs index 31a3b4212..97976d271 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/ModuleSymbolTable.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/ModuleSymbolTable.cs @@ -30,8 +30,6 @@ internal sealed class ModuleSymbolTable { private readonly Dictionary _evaluators = new Dictionary(); private readonly HashSet _processed = new HashSet(); - public HashSet ReplacedByStubs { get; } = new HashSet(); - public IEnumerable> Evaluators => _evaluators.ToArray(); public void Add(MemberEvaluator e) => _evaluators[e.Target] = e; public bool Contains(ScopeStatement node) => _evaluators.ContainsKey(node) || _processed.Contains(node); diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs index b383758d1..8aacf01e1 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs @@ -15,7 +15,6 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.Linq; using Microsoft.Python.Analysis.Analyzer.Evaluation; using Microsoft.Python.Analysis.Types; @@ -70,8 +69,7 @@ public override bool Walk(ClassDefinition cd) { } public override void PostWalk(ClassDefinition cd) { - if (!string.IsNullOrEmpty(cd.NameExpression?.Name) && - _typeMap.ContainsKey(cd)) { + if (!string.IsNullOrEmpty(cd.NameExpression?.Name) && _typeMap.ContainsKey(cd)) { _scopes.Pop().Dispose(); } base.PostWalk(cd); @@ -136,43 +134,20 @@ private void AddFunction(FunctionDefinition fd, PythonType declaringType) { } private void AddOverload(FunctionDefinition fd, IPythonClassMember function, Action addOverload) { - // Check if function exists in stubs. If so, take overload from stub - // and the documentation from this actual module. - if (!_table.ReplacedByStubs.Contains(fd)) { - var stubOverload = GetOverloadFromStub(fd); - if (stubOverload != null) { - var documentation = fd.GetDocumentation(); - if (!string.IsNullOrEmpty(documentation)) { - stubOverload.SetDocumentation(documentation); - } - addOverload(stubOverload); - _table.ReplacedByStubs.Add(fd); - return; - } - } - if (!_table.Contains(fd)) { // Do not evaluate parameter types just yet. During light-weight top-level information // collection types cannot be determined as imports haven't been processed. var overload = new PythonFunctionOverload(function, fd, _eval.GetLocationOfName(fd), fd.ReturnAnnotation?.ToCodeString(_eval.Ast)); addOverload(overload); - _table.Add(new FunctionEvaluator(_eval, overload)); - } - } - - private PythonFunctionOverload GetOverloadFromStub(FunctionDefinition node) { - var t = GetMemberFromStub(node.Name).GetPythonType(); - if (t is IPythonFunctionType f) { - return f.Overloads - .OfType() - .FirstOrDefault(o => o.Parameters.Count == node.Parameters.Count(p => !p.IsPositionalOnlyMarker)); + if (!_eval.StubOnlyAnalysis) { + _table.Add(new FunctionEvaluator(_eval, overload)); + } } - return null; } private bool TryAddProperty(FunctionDefinition node, PythonType declaringType) { // We can't add a property to an unknown type. Fallback to a regular function for now. - // TOOD: Decouple declaring types from the property. + // TODO: Decouple declaring types from the property. if (declaringType.IsUnknown()) { return false; } diff --git a/src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs b/src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs index e04965a97..5e2258cf9 100644 --- a/src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs +++ b/src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs @@ -36,15 +36,5 @@ internal interface IModuleDatabaseService: IModuleDatabaseCache { /// /// Cancellation token Task StoreModuleAnalysisAsync(IDocumentAnalysis analysis, bool immediate = false, CancellationToken cancellationToken = default); - - /// - /// Determines if module analysis exists in the storage. - /// - bool ModuleExistsInStorage(string name, string filePath, ModuleType moduleType); - } - - internal static class ModuleDatabaseExtensions { - public static bool ModuleExistsInStorage(this IModuleDatabaseService dbs, IPythonModule module) - => dbs.ModuleExistsInStorage(module.Name, module.FilePath, module.ModuleType); } } diff --git a/src/Analysis/Ast/Impl/Definitions/AnalysisOptions.cs b/src/Analysis/Ast/Impl/Definitions/AnalysisOptions.cs index 52a8f2106..3d99d59e4 100644 --- a/src/Analysis/Ast/Impl/Definitions/AnalysisOptions.cs +++ b/src/Analysis/Ast/Impl/Definitions/AnalysisOptions.cs @@ -46,5 +46,12 @@ public class AnalysisOptions { /// Defines level of caching analysis engine will maintain. /// public AnalysisCachingLevel AnalysisCachingLevel { get; set; } + + /// + /// Tells if source module should be analyzed or, if stub is present, + /// the stub becomes primary source of information on types and source + /// modules would be used only as documentation provider. + /// + public bool StubOnlyAnalysis { get; set; } } } diff --git a/src/Analysis/Ast/Test/AnalysisTestBase.cs b/src/Analysis/Ast/Test/AnalysisTestBase.cs index 5ad5426de..a2d9bf6a7 100644 --- a/src/Analysis/Ast/Test/AnalysisTestBase.cs +++ b/src/Analysis/Ast/Test/AnalysisTestBase.cs @@ -93,6 +93,14 @@ protected async Task CreateServicesAsync(string root, Interpret sm.AddService(ds); } + var ap = Substitute.For(); + ap.Options.Returns(x => new AnalysisOptions { + AnalysisCachingLevel = AnalysisCachingLevel.None, + StubOnlyAnalysis = false, + LintingEnabled = true + }); + sm.AddService(ap); + TestLogger.Log(TraceEventType.Information, "Create PythonAnalyzer"); CacheService.Register(sm, stubCacheFolderPath, pathCheck: false); diff --git a/src/Analysis/Ast/Test/BasicTests.cs b/src/Analysis/Ast/Test/BasicTests.cs index a469fdddc..f704f285a 100644 --- a/src/Analysis/Ast/Test/BasicTests.cs +++ b/src/Analysis/Ast/Test/BasicTests.cs @@ -18,7 +18,6 @@ using Microsoft.Python.Analysis.Tests.FluentAssertions; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; -using Microsoft.Python.Parsing.Tests; using Microsoft.VisualStudio.TestTools.UnitTesting; using TestUtilities; diff --git a/src/Caching/Impl/ModuleDatabase.cs b/src/Caching/Impl/ModuleDatabase.cs index 2e51aedde..6bfea9572 100644 --- a/src/Caching/Impl/ModuleDatabase.cs +++ b/src/Caching/Impl/ModuleDatabase.cs @@ -47,7 +47,6 @@ private readonly ConcurrentDictionary _searchResults private readonly IFileSystem _fs; private readonly AnalysisCachingLevel _defaultCachingLevel; private readonly CacheWriter _cacheWriter; - private AnalysisCachingLevel? _cachingLevel; public ModuleDatabase(IServiceManager sm, string cacheFolder = null) { _services = sm; @@ -76,27 +75,6 @@ public IPythonModule RestoreModule(string moduleName, string modulePath, ModuleT ? RestoreModule(model) : null; } - /// - /// Determines if module analysis exists in the storage. - /// - public bool ModuleExistsInStorage(string name, string filePath, ModuleType moduleType) { - if (GetCachingLevel() == AnalysisCachingLevel.None) { - return false; - } - - var key = new AnalysisModuleKey(name, filePath); - if (_searchResults.TryGetValue(key, out var result)) { - return result; - } - - WithRetries.Execute(() => { - var dbPath = FindDatabaseFile(name, filePath, moduleType); - _searchResults[key] = result = !string.IsNullOrEmpty(dbPath); - return result; - }, "Unable to find database file for {name}.", _log); - return false; - } - public async Task StoreModuleAnalysisAsync(IDocumentAnalysis analysis, bool immediate = false, CancellationToken cancellationToken = default) { var cachingLevel = GetCachingLevel(); if (cachingLevel == AnalysisCachingLevel.None) { @@ -192,9 +170,7 @@ private bool TryGetModuleModel(string moduleName, string dbPath, out ModuleModel } private AnalysisCachingLevel GetCachingLevel() - => _cachingLevel - ?? (_cachingLevel = _services.GetService()?.Options.AnalysisCachingLevel) - ?? _defaultCachingLevel; + => _services.GetService()?.Options.AnalysisCachingLevel ?? _defaultCachingLevel; public void Dispose() => _cacheWriter.Dispose(); } diff --git a/src/Caching/Test/AnalysisCachingTestBase.cs b/src/Caching/Test/AnalysisCachingTestBase.cs index 9ebca86f6..a8faa5dc0 100644 --- a/src/Caching/Test/AnalysisCachingTestBase.cs +++ b/src/Caching/Test/AnalysisCachingTestBase.cs @@ -20,7 +20,6 @@ using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Caching.Models; using Microsoft.Python.Analysis.Caching.Tests.FluentAssertions; -using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Tests; using Microsoft.Python.Analysis.Types; using Newtonsoft.Json; diff --git a/src/Caching/Test/RestoreTests.cs b/src/Caching/Test/RestoreTests.cs index b7883c0c4..1bb746f57 100644 --- a/src/Caching/Test/RestoreTests.cs +++ b/src/Caching/Test/RestoreTests.cs @@ -22,6 +22,7 @@ using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Types; using Microsoft.VisualStudio.TestTools.UnitTesting; +using NSubstitute; using TestUtilities; namespace Microsoft.Python.Analysis.Caching.Tests { @@ -57,6 +58,8 @@ def func2() -> C2: ... var dbs = new ModuleDatabase(Services, Path.GetDirectoryName(TestData.GetDefaultModulePath())); Services.AddService(dbs); + + EnableCaching(); await dbs.StoreModuleAnalysisAsync(analysis2, immediate: true, CancellationToken.None); await Services.GetService().ResetAnalyzer(); @@ -101,15 +104,28 @@ private async Task CreateDatabaseAsync(IPythonInterpreter interpreter) { var dbs = new ModuleDatabase(Services, Path.GetDirectoryName(TestData.GetDefaultModulePath())); Services.AddService(dbs); + EnableCaching(); + var importedModules = interpreter.ModuleResolution.GetImportedModules(); foreach (var m in importedModules.Where(m => m.Analysis is LibraryAnalysis)) { await dbs.StoreModuleAnalysisAsync(m.Analysis, immediate: true); } - + importedModules = interpreter.TypeshedResolution.GetImportedModules(); foreach (var m in importedModules.Where(m => m.Analysis is LibraryAnalysis)) { await dbs.StoreModuleAnalysisAsync(m.Analysis, immediate: true); } } + + private void EnableCaching() { + var a = Services.GetService(); + Services.RemoveService(a); + var aop = Substitute.For(); + aop.Options.Returns(x => new AnalysisOptions { + AnalysisCachingLevel = AnalysisCachingLevel.Library, + StubOnlyAnalysis = a.Options.StubOnlyAnalysis + }); + Services.AddService(aop); + } } } diff --git a/src/LanguageServer/Impl/LanguageServer.Configuration.cs b/src/LanguageServer/Impl/LanguageServer.Configuration.cs index 73a4c2fe0..d4e8e0821 100644 --- a/src/LanguageServer/Impl/LanguageServer.Configuration.cs +++ b/src/LanguageServer/Impl/LanguageServer.Configuration.cs @@ -132,6 +132,7 @@ private void HandleDiagnosticsChanges(JToken pythonSection, LanguageServerSettin var optionsProvider = _services.GetService(); optionsProvider.Options.KeepLibraryAst = GetSetting(memory, "keepLibraryAst", false); optionsProvider.Options.AnalysisCachingLevel = GetAnalysisCachingLevel(analysis); + optionsProvider.Options.StubOnlyAnalysis = GetSetting(analysis, "stubsOnly", true); _logger?.Log(TraceEventType.Information, Resources.AnalysisCacheLevel.FormatInvariant(optionsProvider.Options.AnalysisCachingLevel)); } diff --git a/src/LanguageServer/Impl/Program.cs b/src/LanguageServer/Impl/Program.cs index 25528cdb5..29ed9f88b 100644 --- a/src/LanguageServer/Impl/Program.cs +++ b/src/LanguageServer/Impl/Program.cs @@ -41,34 +41,34 @@ public static void Main(string[] args) { messageFormatter.JsonSerializer.ConstructorHandling = ConstructorHandling.AllowNonPublicDefaultConstructor; messageFormatter.JsonSerializer.Converters.Add(new UriConverter()); - using (var cin = Console.OpenStandardInput()) - using (var cout = Console.OpenStandardOutput()) - using (var server = new Implementation.LanguageServer()) - using (var rpc = new LanguageServerJsonRpc(cout, cin, messageFormatter, server)) { - rpc.TraceSource.Switch.Level = SourceLevels.Error; - rpc.SynchronizationContext = new SingleThreadSynchronizationContext(); - var clientApp = new ClientApplication(rpc); - - var osp = new OSPlatform(); - services - .AddService(clientApp) - .AddService(new Logger(clientApp)) - .AddService(new UIService(clientApp)) - .AddService(new ProgressService(clientApp)) - .AddService(new TelemetryService(clientApp)) - .AddService(new IdleTimeService()) - .AddService(osp) - .AddService(new ProcessServices()) - .AddService(new FileSystem(osp)); - - services.AddService(messageFormatter.JsonSerializer); - - var token = server.Start(services, rpc); - rpc.StartListening(); - - // Wait for the "exit" request, it will terminate the process. - token.WaitHandle.WaitOne(); - } + using var cin = Console.OpenStandardInput(); + using var cout = Console.OpenStandardOutput(); + using var server = new Implementation.LanguageServer(); + using var rpc = new LanguageServerJsonRpc(cout, cin, messageFormatter, server); + + rpc.TraceSource.Switch.Level = SourceLevels.Error; + rpc.SynchronizationContext = new SingleThreadSynchronizationContext(); + var clientApp = new ClientApplication(rpc); + + var osp = new OSPlatform(); + services + .AddService(clientApp) + .AddService(new Logger(clientApp)) + .AddService(new UIService(clientApp)) + .AddService(new ProgressService(clientApp)) + .AddService(new TelemetryService(clientApp)) + .AddService(new IdleTimeService()) + .AddService(osp) + .AddService(new ProcessServices()) + .AddService(new FileSystem(osp)); + + services.AddService(messageFormatter.JsonSerializer); + + var token = server.Start(services, rpc); + rpc.StartListening(); + + // Wait for the "exit" request, it will terminate the process. + token.WaitHandle.WaitOne(); } } diff --git a/src/LanguageServer/Test/LinterTests.cs b/src/LanguageServer/Test/LinterTests.cs index 678b12c46..b996762fb 100644 --- a/src/LanguageServer/Test/LinterTests.cs +++ b/src/LanguageServer/Test/LinterTests.cs @@ -45,6 +45,12 @@ public async Task LinterOnOff() { var d = a.LintModule(analysis.Document); d.Should().HaveCount(1); + + var aop = Services.GetService(); + if (aop != null) { + Services.RemoveService(aop); + } + var provider = Substitute.For(); Services.AddService(provider);