Skip to content
This repository was archived by the owner on Apr 14, 2022. It is now read-only.

Commit 53eb588

Browse files
AlexanderSherAlexanderSher
authored
Fix #975: Analysis continues on old content if file content is replaced by unrelated code (#1124)
* Fix #975: Analysis continues on old content if file content is replaced by unrelated code * Add comment
1 parent a6e31c8 commit 53eb588

8 files changed

Lines changed: 168 additions & 74 deletions

File tree

‎src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,11 @@ private bool TryCreateSession(int graphVersion, PythonAnalyzerEntry entry, out P
233233
}
234234

235235
if (!_dependencyResolver.TryCreateWalker(graphVersion, 2, out var walker)) {
236+
if (entry.Module.ModuleType == ModuleType.Builtins) {
237+
session = CreateSession(null, entry);
238+
return true;
239+
}
240+
236241
session = null;
237242
return false;
238243
}

‎src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs‎

Lines changed: 27 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -280,17 +280,15 @@ private void Analyze(IDependencyChainNode<PythonAnalyzerEntry> node, AsyncCountd
280280
AnalyzeEntry(entry, module, ast, _walker.Version);
281281
node.Commit();
282282

283-
_log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) completed in {(stopWatch.Elapsed - startTime).TotalMilliseconds} ms.");
283+
LogCompleted(module, stopWatch, startTime);
284284
} catch (OperationCanceledException oce) {
285285
node.Value.TryCancel(oce, _walker.Version);
286286
node.Skip();
287-
module = node.Value.Module;
288-
_log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled.");
287+
LogCanceled(node.Value.Module);
289288
} catch (Exception exception) {
290-
module = node.Value.Module;
291289
node.Value.TrySetException(exception, _walker.Version);
292290
node.Commit();
293-
_log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) failed. Exception message: {exception.Message}.");
291+
LogException(node.Value.Module, exception);
294292
} finally {
295293
bool isCanceled;
296294
lock (_syncObj) {
@@ -307,7 +305,7 @@ private void Analyze(IDependencyChainNode<PythonAnalyzerEntry> node, AsyncCountd
307305
}
308306

309307
private void AnalyzeEntry() {
310-
var stopWatch = Stopwatch.StartNew();
308+
var stopWatch = _log != null ? Stopwatch.StartNew() : null;
311309
try {
312310
if (!_entry.IsValidVersion(Version, out var module, out var ast)) {
313311
if (ast == null) {
@@ -319,20 +317,18 @@ private void AnalyzeEntry() {
319317
return;
320318
}
321319

322-
var startTime = stopWatch.Elapsed;
320+
var startTime = stopWatch?.Elapsed ?? TimeSpan.Zero;
323321
AnalyzeEntry(_entry, module, ast, Version);
324322

325-
_log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) completed in {(stopWatch.Elapsed - startTime).TotalMilliseconds} ms.");
323+
LogCompleted(module, stopWatch, startTime);
326324
} catch (OperationCanceledException oce) {
327325
_entry.TryCancel(oce, Version);
328-
var module = _entry.Module;
329-
_log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled.");
326+
LogCanceled(_entry.Module);
330327
} catch (Exception exception) {
331-
var module = _entry.Module;
332328
_entry.TrySetException(exception, Version);
333-
_log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) failed. Exception message: {exception.Message}.");
329+
LogException(_entry.Module, exception);
334330
} finally {
335-
stopWatch.Stop();
331+
stopWatch?.Stop();
336332
Interlocked.Decrement(ref _runningTasks);
337333
}
338334
}
@@ -360,6 +356,24 @@ private void AnalyzeEntry(PythonAnalyzerEntry entry, IPythonModule module, Pytho
360356
}
361357
}
362358

359+
private void LogCompleted(IPythonModule module, Stopwatch stopWatch, TimeSpan startTime) {
360+
if (_log != null) {
361+
_log.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) completed in {(stopWatch.Elapsed - startTime).TotalMilliseconds} ms.");
362+
}
363+
}
364+
365+
private void LogCanceled(IPythonModule module) {
366+
if (_log != null) {
367+
_log.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled.");
368+
}
369+
}
370+
371+
private void LogException(IPythonModule module, Exception exception) {
372+
if (_log != null) {
373+
_log.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) failed. Exception message: {exception.Message}.");
374+
}
375+
}
376+
363377
private enum State {
364378
NotStarted = 0,
365379
Started = 1,

‎src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs‎

Lines changed: 81 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -192,7 +192,9 @@ public bool TryCreateWalker(int version, int walkerDepthLimit, out IDependencyCh
192192
return false;
193193
}
194194

195-
if (!TryCreateWalkingGraph(vertices, version, out var walkingGraph)) {
195+
var depths = CalculateDepths(vertices);
196+
197+
if (!TryCreateWalkingGraph(vertices, depths, version, out var walkingGraph)) {
196198
walker = default;
197199
return false;
198200
}
@@ -202,7 +204,14 @@ public bool TryCreateWalker(int version, int walkerDepthLimit, out IDependencyCh
202204
return false;
203205
}
204206

205-
if (!TryResolveLoops(walkingGraph, loopsCount, version, out var startingVertices, out var totalNodesCount)) {
207+
if (!TryResolveLoops(walkingGraph, loopsCount, version, out var totalNodesCount)) {
208+
walker = default;
209+
return false;
210+
}
211+
212+
// Iterate original graph to get starting vertices
213+
var startingVertices = walkingGraph.Where(v => v.IncomingCount == 0);
214+
if (!TryFindMissingDependencies(vertices, walkingGraph, version)) {
206215
walker = default;
207216
return false;
208217
}
@@ -212,27 +221,27 @@ public bool TryCreateWalker(int version, int walkerDepthLimit, out IDependencyCh
212221
vertex.SecondPass?.Seal();
213222
}
214223

215-
var depths = CalculateDepths(vertices);
224+
var affectedValues = walkingGraph.Select(v => v.DependencyVertex.Value);
225+
var missingKeys = ImmutableArray<TKey>.Empty;
226+
216227
lock (_syncObj) {
217228
if (version != _version) {
218229
walker = default;
219230
return false;
220231
}
221-
222-
var affectedValues = walkingGraph.Select(v => v.DependencyVertex.Value);
223-
var missingKeys = ImmutableArray<TKey>.Empty;
232+
224233
foreach (var (key, index) in _keys) {
225-
if (_vertices[index] == default/* && depths[index] <= walkerDepthLimit*/) {
234+
if (vertices[index] == default/* && depths[index] <= walkerDepthLimit*/) {
226235
missingKeys = missingKeys.Add(key);
227236
}
228237
}
229-
230-
walker = new DependencyChainWalker(startingVertices, affectedValues, depths, missingKeys, totalNodesCount, version);
231-
return true;
232238
}
239+
240+
walker = new DependencyChainWalker(startingVertices, affectedValues, depths, missingKeys, totalNodesCount, version);
241+
return true;
233242
}
234243

235-
private bool TryBuildReverseGraph(ImmutableArray<DependencyVertex<TKey, TValue>> vertices, int version) {
244+
private bool TryBuildReverseGraph(in ImmutableArray<DependencyVertex<TKey, TValue>> vertices, int version) {
236245
var reverseGraphIsBuilt = true;
237246
foreach (var vertex in vertices) {
238247
if (vertex != null && !vertex.IsSealed) {
@@ -283,12 +292,17 @@ private bool TryBuildReverseGraph(ImmutableArray<DependencyVertex<TKey, TValue>>
283292
}
284293
}
285294

286-
private bool TryCreateWalkingGraph(in ImmutableArray<DependencyVertex<TKey, TValue>> vertices, int version, out ImmutableArray<WalkingVertex<TKey, TValue>> analysisGraph) {
295+
private bool TryCreateWalkingGraph(in ImmutableArray<DependencyVertex<TKey, TValue>> vertices, ImmutableArray<int> depths, int version, out ImmutableArray<WalkingVertex<TKey, TValue>> analysisGraph) {
296+
if (version != _version) {
297+
analysisGraph = default;
298+
return false;
299+
}
300+
287301
var nodesByVertexIndex = new Dictionary<int, WalkingVertex<TKey, TValue>>();
288302

289303
for (var index = 0; index < vertices.Count; index++) {
290304
var vertex = vertices[index];
291-
if (vertex == null || vertex.IsWalked) {
305+
if (vertex == null || vertex.IsWalked || depths[index] == -1) {
292306
continue;
293307
}
294308

@@ -332,7 +346,7 @@ private bool TryCreateWalkingGraph(in ImmutableArray<DependencyVertex<TKey, TVal
332346
private static ImmutableArray<int> CalculateDepths(in ImmutableArray<DependencyVertex<TKey, TValue>> vertices) {
333347
var depths = new int[vertices.Count];
334348
for (var i = 0; i < depths.Length; i++) {
335-
depths[i] = -1;
349+
depths[i] = -1; // Unreachable vertices will be marked as -1
336350
}
337351

338352
for (var i = 0; i < depths.Length; i++) {
@@ -342,19 +356,11 @@ private static ImmutableArray<int> CalculateDepths(in ImmutableArray<DependencyV
342356
SetDepths(depths, vertices, vertex.Incoming, 1);
343357
}
344358
}
345-
346-
for (var i = 0; i < depths.Length; i++) {
347-
var vertex = vertices[i];
348-
if (vertex != null && depths[i] == -1) {
349-
depths[i] = 1;
350-
SetDepths(depths, vertices, vertex.Incoming, 2);
351-
}
352-
}
353-
359+
354360
return ImmutableArray<int>.Create(depths);
355361
}
356362

357-
private static void SetDepths(int[] depths, ImmutableArray<DependencyVertex<TKey, TValue>> vertices, ImmutableArray<int> indices, int depth) {
363+
private static void SetDepths(in int[] depths, in ImmutableArray<DependencyVertex<TKey, TValue>> vertices, in ImmutableArray<int> indices, int depth) {
358364
foreach (var index in indices) {
359365
if (depths[index] != -1 && depths[index] <= depth) {
360366
continue;
@@ -432,7 +438,12 @@ private static bool SetLoopNumber(WalkingVertex<TKey, TValue> vertex, Stack<Walk
432438
return false;
433439
}
434440

435-
private bool TryResolveLoops(ImmutableArray<WalkingVertex<TKey, TValue>> graph, int loopsCount, int version, out ImmutableArray<WalkingVertex<TKey, TValue>> startingVertices, out int totalNodesCount) {
441+
private bool TryResolveLoops(in ImmutableArray<WalkingVertex<TKey, TValue>> graph, int loopsCount, int version, out int totalNodesCount) {
442+
if (loopsCount == 0) {
443+
totalNodesCount = graph.Count;
444+
return true;
445+
}
446+
436447
// Create vertices for second pass
437448
var inLoopsCount = 0;
438449
var secondPassLoops = new List<WalkingVertex<TKey, TValue>>[loopsCount];
@@ -450,7 +461,6 @@ private bool TryResolveLoops(ImmutableArray<WalkingVertex<TKey, TValue>> graph,
450461
}
451462

452463
if (version != _version) {
453-
startingVertices = default;
454464
totalNodesCount = default;
455465
return false;
456466
}
@@ -471,7 +481,6 @@ private bool TryResolveLoops(ImmutableArray<WalkingVertex<TKey, TValue>> graph,
471481
}
472482

473483
if (version != _version) {
474-
startingVertices = default;
475484
totalNodesCount = default;
476485
return false;
477486
}
@@ -505,18 +514,14 @@ private bool TryResolveLoops(ImmutableArray<WalkingVertex<TKey, TValue>> graph,
505514
}
506515

507516
if (version != _version) {
508-
startingVertices = default;
509517
totalNodesCount = default;
510518
return false;
511519
}
512520

513521
loopsCount++;
514522
}
515523

516-
// Iterate original graph to get starting vertices
517-
startingVertices = graph.Where(v => v.IncomingCount == 0);
518524
totalNodesCount = graph.Count + inLoopsCount;
519-
520525
return true;
521526
}
522527

@@ -536,6 +541,50 @@ private static void RemoveOutgoingLoopEdges(WalkingVertex<TKey, TValue> vertex,
536541
}
537542
}
538543

544+
private bool TryFindMissingDependencies(in ImmutableArray<DependencyVertex<TKey, TValue>> vertices, in ImmutableArray<WalkingVertex<TKey, TValue>> walkingGraph, int version) {
545+
var haveMissingDependencies = new bool[vertices.Count];
546+
var queue = new Queue<DependencyVertex<TKey, TValue>>();
547+
548+
foreach (var vertex in vertices) {
549+
if (vertex == default) {
550+
continue;
551+
}
552+
553+
foreach (var incoming in vertex.Incoming) {
554+
if (vertices[incoming] == default) {
555+
haveMissingDependencies[vertex.Index] = true;
556+
queue.Enqueue(vertex);
557+
break;
558+
}
559+
}
560+
}
561+
562+
while (queue.Count > 0) {
563+
if (version != _version) {
564+
return false;
565+
}
566+
567+
var vertex = queue.Dequeue();
568+
foreach (var outgoing in vertex.Outgoing) {
569+
if (haveMissingDependencies[outgoing]) {
570+
continue; // This vertex has been visited
571+
}
572+
573+
haveMissingDependencies[outgoing] = true;
574+
queue.Enqueue(vertices[outgoing]);
575+
}
576+
}
577+
578+
foreach (var walkingVertex in walkingGraph) {
579+
if (haveMissingDependencies[walkingVertex.DependencyVertex.Index]) {
580+
walkingVertex.MarkHasMissingDependencies();
581+
walkingVertex.SecondPass?.MarkHasMissingDependencies();
582+
}
583+
}
584+
585+
return true;
586+
}
587+
539588
private sealed class DependencyChainWalker : IDependencyChainWalker<TKey, TValue> {
540589
private readonly ImmutableArray<WalkingVertex<TKey, TValue>> _startingVertices;
541590
private readonly ImmutableArray<int> _depths;
@@ -630,6 +679,7 @@ private sealed class DependencyChainNode : IDependencyChainNode<TValue> {
630679
private readonly WalkingVertex<TKey, TValue> _vertex;
631680
private DependencyChainWalker _walker;
632681
public TValue Value => _vertex.DependencyVertex.Value;
682+
public bool HasMissingDependencies => _vertex.HasMissingDependencies;
633683
public int VertexDepth { get; }
634684

635685
public DependencyChainNode(DependencyChainWalker walker, WalkingVertex<TKey, TValue> vertex, int depth) {

‎src/Analysis/Ast/Impl/Dependencies/IDependencyChainNode.cs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,11 @@
1616
namespace Microsoft.Python.Analysis.Dependencies {
1717
internal interface IDependencyChainNode<out TValue> {
1818
int VertexDepth { get; }
19+
20+
/// <summary>
21+
/// Shows if node has any direct or indirect dependencies that aren't added to the graph
22+
/// </summary>
23+
bool HasMissingDependencies { get; }
1924
TValue Value { get; }
2025
void Commit();
2126
void Skip();

‎src/Analysis/Ast/Impl/Dependencies/WalkingVertex.cs‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ internal sealed class WalkingVertex<TKey, TValue> {
3131
public int Index { get; set; }
3232
public int LoopNumber { get; set; }
3333
public int IncomingCount { get; private set; }
34+
public bool HasMissingDependencies { get; private set; }
3435

3536
public WalkingVertex<TKey, TValue> FirstPass { get; }
3637
public WalkingVertex<TKey, TValue> SecondPass { get; private set; }
@@ -47,20 +48,16 @@ public WalkingVertex(DependencyVertex<TKey, TValue> vertex, WalkingVertex<TKey,
4748
_outgoing = new List<WalkingVertex<TKey, TValue>>();
4849
}
4950

50-
public void AddOutgoing(WalkingVertex<TKey, TValue> outgoingVertex) {
51+
public void MarkHasMissingDependencies() {
5152
CheckNotSealed();
52-
53-
_outgoing.Add(outgoingVertex);
54-
outgoingVertex.IncomingCount++;
53+
HasMissingDependencies = true;
5554
}
5655

57-
public void AddOutgoing(List<WalkingVertex<TKey, TValue>> loop) {
56+
public void AddOutgoing(WalkingVertex<TKey, TValue> outgoingVertex) {
5857
CheckNotSealed();
5958

60-
_outgoing.AddRange(loop);
61-
foreach (var outgoingVertex in loop) {
62-
outgoingVertex.IncomingCount++;
63-
}
59+
_outgoing.Add(outgoingVertex);
60+
outgoingVertex.IncomingCount++;
6461
}
6562

6663
public void AddOutgoing(HashSet<WalkingVertex<TKey, TValue>> loop) {
@@ -88,6 +85,7 @@ public WalkingVertex<TKey, TValue> CreateSecondPassVertex() {
8885
}
8986

9087
public void Seal() => _isSealed = true;
88+
9189
public void DecrementIncoming() {
9290
CheckSealed();
9391
IncomingCount--;

0 commit comments

Comments
 (0)