Rewrite gen.py for new LS (#1141)

Author: jakebailey

src/gen.py

@@ -15,7 +15,7 @@ def main():
         script_dir, "UnitTests", "TestData", "gen", "completion"
     )
     default_output = os.path.join(
-        script_dir, "Analysis", "Engine", "Test", "GenTests.cs"
+        script_dir, "LanguageServer", "Test", "GenTests.cs"
     )
 
     parser = argparse.ArgumentParser(
@@ -202,62 +202,72 @@ def create_tests(name, filename, ignored_lines):
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
-using AnalysisTests;
 using FluentAssertions;
-using Microsoft.Python.LanguageServer.Implementation;
-using Microsoft.PythonTools.Analysis;
-using Microsoft.PythonTools.Analysis.FluentAssertions;
-using Microsoft.PythonTools.Interpreter;
-using Microsoft.PythonTools.Parsing;
+using Microsoft.Python.Analysis;
+using Microsoft.Python.Analysis.Analyzer;
+using Microsoft.Python.Analysis.Core.Interpreter;
+using Microsoft.Python.Analysis.Documents;
+using Microsoft.Python.Core.Idle;
+using Microsoft.Python.Core.Services;
+using Microsoft.Python.Core.Text;
+using Microsoft.Python.LanguageServer;
+using Microsoft.Python.LanguageServer.Completion;
+using Microsoft.Python.LanguageServer.Sources;
+using Microsoft.Python.LanguageServer.Tests;
+using Microsoft.Python.Parsing;
+using Microsoft.Python.Parsing.Tests;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
+using NSubstitute;
 using TestUtilities;
 
 namespace GenTests {"""
 
 POSTAMBLE = """
-    public class GenTest : ServerBasedTest {
-        private static Server _server;
-        private static readonly SemaphoreSlim _sem = new SemaphoreSlim(1, 1);
+    public class GenTest : LanguageServerTestBase {
+        private static readonly Dictionary<string, Task<IDocumentAnalysis>> _analysis = new Dictionary<string, Task<IDocumentAnalysis>>();
+
         private static readonly InterpreterConfiguration _interpreter = PythonVersions.LatestAvailable3X;
         private static readonly PythonLanguageVersion _version = _interpreter.Version.ToLanguageVersion();
-        private static readonly ConcurrentDictionary<string, Task> _opened = new ConcurrentDictionary<string, Task>();
 
-        private async Task<Server> SharedServer() {
-            if (_server != null) {
-                return _server;
-            }
+        private static readonly CompletionSource _cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion);
+        private static readonly HoverSource _hs = new HoverSource(new PlainTextDocumentationSource());
 
-            await _sem.WaitAsync();
-            try {
-                var root = new Uri(TestData.GetPath("TestData", "gen", "completion"));
-                _server = await CreateServerAsync(_interpreter, root);
-            } finally {
-                _sem.Release();
+        static GenTest() {
+            _interpreter.TypeshedPath = TestData.GetDefaultTypeshedPath();
+        }
+
+        protected async Task<IDocumentAnalysis> GetGenAnalysisAsync(string module) {
+            // Create an analyzer per module. This is slower than creating a single
+            // analyzer shared between all GenTest instances, but sharing them makes
+            // the "types" module fail (due to a bug where the name "types" shadows
+            // a builtin module name).
+            if (_analysis.TryGetValue(module, out var analysis)) {
+                return await analysis;
             }
 
-            return _server;
-        }
+            var root = TestData.GetPath("TestData", "gen", "completion");
 
-        protected async Task<Uri> OpenAndWait(string module) {
-            var server = await SharedServer();
+            var sm = CreateServiceManager();
+            sm.AddService(new PythonAnalyzer(sm));
+            sm.AddService(await PythonInterpreter.CreateAsync(_interpreter, root, sm));
+            sm.AddService(new RunningDocumentTable(sm));
 
             var src = TestData.GetPath("TestData", "gen", "completion", module + ".py");
-            var uri = new Uri(src);
+            analysis = GetAnalysisAsync(File.ReadAllText(src), sm, modulePath: src);
+            _analysis[module] = analysis;
 
-            await _opened.GetOrAdd(src, f => server.SendDidOpenTextDocument(uri, File.ReadAllText(f)));
-            await server.WaitForCompleteAnalysisAsync(CancellationToken.None);
-
-            return uri;
+            return await analysis;
         }
 
         protected async Task DoCompletionTest(string module, int lineNum, int col, string args, string filter) {
-            var server = await SharedServer();
-
             var tests = string.IsNullOrWhiteSpace(args) ? new List<string>() : ParseStringList(args);
-            var uri = await OpenAndWait(module);
 
-            var res = await server.SendCompletion(uri, lineNum, col);
-            var items = res.items?.Select(item => item.insertText).Where(t => t.Contains(filter)).ToList() ?? new List<string>();
+            var analysis = await GetGenAnalysisAsync(module);
+
+            var res = _cs.GetCompletions(analysis, new SourceLocation(lineNum + 1, col + 1));
+            res.Should().NotBeNull();
+
+            var items = res.Completions?.Select(item => item.insertText).Where(t => t.Contains(filter)).ToList() ?? new List<string>();
 
             if (tests.Count == 0) {
                 items.Should().BeEmpty();
@@ -267,15 +277,14 @@ def create_tests(name, filename, ignored_lines):
         }
 
         protected async Task DoHoverTest(string module, int lineNum, int col, string args) {
-            var server = await SharedServer();
-
             var tests = string.IsNullOrWhiteSpace(args)
                 ? new List<string>()
                 : args.Split(' ', options: StringSplitOptions.RemoveEmptyEntries).Select(s => s.EndsWith("()") ? s.Substring(0, s.Length - 2) : s).ToList();
 
-            var uri = await OpenAndWait(module);
+            var analysis = await GetGenAnalysisAsync(module);
 
-            var res = await server.SendHover(uri, lineNum, col);
+            var res = _hs.GetHover(analysis, new SourceLocation(lineNum + 1, col + 1));
+            res.Should().NotBeNull();
 
             if (tests.Count == 0) {
                 res.contents.value.Should().BeEmpty();
@@ -309,7 +318,8 @@ def create_tests(name, filename, ignored_lines):
             return list;
         }
     }
-}"""
+}
+"""
 
 CLASS_PREAMBLE = """    [TestClass]
     public class {name}Tests : GenTest {{