Skip to content
This repository was archived by the owner on Apr 14, 2022. It is now read-only.

Commit 91e912c

Browse files
author
Mikhail Arkhipov
authored
Relocate generated stub cache (#1147)
* Initial * Rename folder * Add test * Port setting for the cache path * Clean settings * Remove unused setting
1 parent 348374f commit 91e912c

19 files changed

Lines changed: 295 additions & 815 deletions

File tree

‎src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
using System.Linq;
2020
using System.Threading;
2121
using System.Threading.Tasks;
22+
using Microsoft.Python.Analysis.Caching;
2223
using Microsoft.Python.Analysis.Dependencies;
2324
using Microsoft.Python.Analysis.Diagnostics;
2425
using Microsoft.Python.Analysis.Documents;
@@ -48,14 +49,15 @@ public sealed class PythonAnalyzer : IPythonAnalyzer, IDisposable {
4849
private PythonAnalyzerSession _currentSession;
4950
private PythonAnalyzerSession _nextSession;
5051

51-
public PythonAnalyzer(IServiceManager services) {
52+
public PythonAnalyzer(IServiceManager services, string cacheFolderPath = null) {
5253
_services = services;
5354
_log = services.GetService<ILogger>();
5455
_dependencyResolver = new DependencyResolver<AnalysisModuleKey, PythonAnalyzerEntry>();
5556
_analysisCompleteEvent.Set();
5657
_startNextSession = StartNextSession;
5758

5859
_progress = new ProgressReporter(services.GetService<IProgressService>());
60+
_services.AddService(new StubCache(_services, cacheFolderPath));
5961
}
6062

6163
public void Dispose() {

‎src/Analysis/Ast/Impl/Caching/CacheFolders.cs‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
// Copyright(c) Microsoft Corporation
2+
// All rights reserved.
3+
//
4+
// Licensed under the Apache License, Version 2.0 (the License); you may not use
5+
// this file except in compliance with the License. You may obtain a copy of the
6+
// License at http://www.apache.org/licenses/LICENSE-2.0
7+
//
8+
// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
9+
// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
10+
// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
11+
// MERCHANTABILITY OR NON-INFRINGEMENT.
12+
//
13+
// See the Apache Version 2.0 License for specific language governing
14+
// permissions and limitations under the License.
15+
16+
using System;
17+
using System.Diagnostics;
18+
using System.IO;
19+
using System.Security.Cryptography;
20+
using System.Text;
21+
using Microsoft.Python.Core;
22+
using Microsoft.Python.Core.IO;
23+
using Microsoft.Python.Core.Logging;
24+
using Microsoft.Python.Core.OS;
25+
26+
namespace Microsoft.Python.Analysis.Caching {
27+
internal static class CacheFolders {
28+
public static string GetCacheFilePath(string root, string moduleName, string content, IFileSystem fs) {
29+
// Folder for all module versions and variants
30+
// {root}/module_name/content_hash.pyi
31+
var folder = Path.Combine(root, moduleName);
32+
33+
var filePath = Path.Combine(root, folder, $"{FileNameFromContent(content)}.pyi");
34+
if (fs.StringComparison == StringComparison.Ordinal) {
35+
filePath = filePath.ToLowerInvariant();
36+
}
37+
38+
return filePath;
39+
}
40+
41+
public static string GetCacheFolder(IServiceContainer services) {
42+
var platform = services.GetService<IOSPlatform>();
43+
var logger = services.GetService<ILogger>();
44+
45+
// Default. Not ideal on all platforms, but used as a fall back.
46+
var localAppData = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData);
47+
var plsSubfolder = $"Microsoft{Path.DirectorySeparatorChar}Python Language Server";
48+
var defaultCachePath = Path.Combine(localAppData, plsSubfolder);
49+
50+
string cachePath = null;
51+
try {
52+
const string homeVarName = "HOME";
53+
var homeFolderPath = Environment.GetEnvironmentVariable(homeVarName);
54+
55+
if(platform.IsWindows) {
56+
cachePath = defaultCachePath;
57+
}
58+
59+
if (platform.IsMac) {
60+
if (CheckVariableSet(homeVarName, homeFolderPath, logger)
61+
&& CheckPathRooted(homeVarName, homeFolderPath, logger)
62+
&& !string.IsNullOrWhiteSpace(homeFolderPath)) {
63+
cachePath = Path.Combine(homeFolderPath, "Library/Caches", plsSubfolder);
64+
} else {
65+
logger?.Log(TraceEventType.Warning, Resources.EnvVariablePathNotRooted.FormatInvariant(homeVarName));
66+
}
67+
}
68+
69+
if (platform.IsLinux) {
70+
const string xdgCacheVarName = "XDG_CACHE_HOME";
71+
var xdgCacheHomePath = Environment.GetEnvironmentVariable(xdgCacheVarName);
72+
73+
if (!string.IsNullOrWhiteSpace(xdgCacheHomePath)
74+
&& CheckPathRooted(xdgCacheVarName, xdgCacheHomePath, logger)) {
75+
cachePath = Path.Combine(xdgCacheVarName, plsSubfolder);
76+
} else if (!string.IsNullOrWhiteSpace(homeFolderPath)
77+
&& CheckVariableSet(homeVarName, homeFolderPath, logger)
78+
&& CheckPathRooted(homeVarName, homeFolderPath, logger)) {
79+
cachePath = Path.Combine(homeFolderPath, ".cache", plsSubfolder);
80+
} else {
81+
logger?.Log(TraceEventType.Warning, Resources.EnvVariablePathNotRooted.FormatInvariant(homeVarName));
82+
}
83+
}
84+
} catch (Exception ex) when (!ex.IsCriticalException()) {
85+
logger?.Log(TraceEventType.Warning, Resources.UnableToDetermineCachePathException.FormatInvariant(ex.Message, defaultCachePath));
86+
}
87+
88+
// Default is same as Windows. Not ideal on all platforms, but it is a fallback anyway.
89+
if (cachePath == null) {
90+
logger?.Log(TraceEventType.Warning, Resources.UnableToDetermineCachePath.FormatInvariant(defaultCachePath));
91+
cachePath = defaultCachePath;
92+
}
93+
94+
logger?.Log(TraceEventType.Information, Resources.AnalysisCachePath.FormatInvariant(cachePath));
95+
return cachePath;
96+
}
97+
98+
public static string FileNameFromContent(string content) {
99+
// File name depends on the content so we can distinguish between different versions.
100+
var hash = SHA256.Create();
101+
return Convert
102+
.ToBase64String(hash.ComputeHash(new UTF8Encoding(false).GetBytes(content)))
103+
.Replace('/', '_').Replace('+', '-');
104+
}
105+
106+
public static string GetAnalysisCacheFilePath(string analysisRootFolder, string moduleName, string content, IFileSystem fs)
107+
=> GetCacheFilePath(analysisRootFolder, moduleName, content, fs);
108+
109+
private static bool CheckPathRooted(string varName, string path, ILogger logger) {
110+
if (!string.IsNullOrWhiteSpace(path) && Path.IsPathRooted(path)) {
111+
return true;
112+
}
113+
114+
logger?.Log(TraceEventType.Warning, Resources.EnvVariablePathNotRooted.FormatInvariant(varName));
115+
return false;
116+
}
117+
118+
private static bool CheckVariableSet(string varName, string value, ILogger logger) {
119+
if (!string.IsNullOrWhiteSpace(value)) {
120+
return true;
121+
}
122+
123+
logger?.Log(TraceEventType.Warning, Resources.EnvVariableNotSet.FormatInvariant(varName));
124+
return false;
125+
}
126+
}
127+
}

‎src/Analysis/Ast/Impl/Caching/Definitions/IStubCache.cs‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
// Copyright(c) Microsoft Corporation
2+
// All rights reserved.
3+
//
4+
// Licensed under the Apache License, Version 2.0 (the License); you may not use
5+
// this file except in compliance with the License. You may obtain a copy of the
6+
// License at http://www.apache.org/licenses/LICENSE-2.0
7+
//
8+
// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
9+
// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
10+
// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
11+
// MERCHANTABILITY OR NON-INFRINGEMENT.
12+
//
13+
// See the Apache Version 2.0 License for specific language governing
14+
// permissions and limitations under the License.
15+
16+
namespace Microsoft.Python.Analysis.Caching {
17+
public interface IStubCache {
18+
string StubCacheFolder { get; }
19+
string GetCacheFilePath(string filePath);
20+
string ReadCachedModule(string filePath);
21+
void WriteCachedModule(string filePath, string code);
22+
}
23+
}

src/Analysis/Ast/Impl/Modules/ModuleCache.cs renamed to src/Analysis/Ast/Impl/Caching/StubCache.cs

Lines changed: 18 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -16,75 +16,54 @@
1616
using System;
1717
using System.Diagnostics;
1818
using System.IO;
19-
using System.Security.Cryptography;
20-
using System.Text;
2119
using System.Threading.Tasks;
2220
using Microsoft.Python.Core;
2321
using Microsoft.Python.Core.IO;
2422
using Microsoft.Python.Core.Logging;
2523

26-
namespace Microsoft.Python.Analysis.Modules {
27-
internal sealed class ModuleCache : IModuleCache {
28-
private readonly IServiceContainer _services;
29-
private readonly IPythonInterpreter _interpreter;
24+
namespace Microsoft.Python.Analysis.Caching {
25+
internal sealed class StubCache : IStubCache {
26+
private const int _stubCacheFormatVersion = 1;
27+
3028
private readonly IFileSystem _fs;
3129
private readonly ILogger _log;
32-
private readonly bool _skipCache;
33-
private bool _loggedBadDbPath;
34-
35-
private string ModuleCachePath => _interpreter.Configuration.DatabasePath;
3630

37-
public ModuleCache(IPythonInterpreter interpreter, IServiceContainer services) {
38-
_interpreter = interpreter;
39-
_services = services;
31+
public StubCache(IServiceContainer services, string cacheRootFolder = null) {
4032
_fs = services.GetService<IFileSystem>();
4133
_log = services.GetService<ILogger>();
42-
_skipCache = string.IsNullOrEmpty(_interpreter.Configuration.DatabasePath);
34+
35+
cacheRootFolder = cacheRootFolder ?? CacheFolders.GetCacheFolder(services);
36+
StubCacheFolder = Path.Combine(cacheRootFolder, $"stubs.v{_stubCacheFormatVersion}");
4337
}
4438

45-
public string GetCacheFilePath(string filePath) {
46-
if (string.IsNullOrEmpty(filePath) || !PathEqualityComparer.IsValidPath(ModuleCachePath)) {
47-
if (!_loggedBadDbPath) {
48-
_loggedBadDbPath = true;
49-
_log?.Log(TraceEventType.Warning, $"Invalid module cache path: {ModuleCachePath}");
50-
}
51-
return null;
52-
}
39+
public string StubCacheFolder { get; }
5340

41+
public string GetCacheFilePath(string filePath) {
5442
var name = PathUtils.GetFileName(filePath);
5543
if (!PathEqualityComparer.IsValidPath(name)) {
5644
_log?.Log(TraceEventType.Warning, $"Invalid cache name: {name}");
5745
return null;
5846
}
5947
try {
60-
var candidate = Path.ChangeExtension(Path.Combine(ModuleCachePath, name), ".pyi");
48+
var candidate = Path.ChangeExtension(Path.Combine(StubCacheFolder, name), ".pyi");
6149
if (_fs.FileExists(candidate)) {
6250
return candidate;
6351
}
6452
} catch (ArgumentException) {
6553
return null;
6654
}
6755

68-
var hash = SHA256.Create();
6956
var dir = Path.GetDirectoryName(filePath) ?? string.Empty;
7057
if (_fs.StringComparison == StringComparison.OrdinalIgnoreCase) {
7158
dir = dir.ToLowerInvariant();
7259
}
7360

74-
var dirHash = Convert.ToBase64String(hash.ComputeHash(new UTF8Encoding(false).GetBytes(dir)))
75-
.Replace('/', '_').Replace('+', '-');
76-
77-
return Path.ChangeExtension(Path.Combine(
78-
ModuleCachePath,
79-
Path.Combine(dirHash, name)
80-
), ".pyi");
61+
var dirHash = CacheFolders.FileNameFromContent(dir);
62+
var stubFile = Path.Combine(StubCacheFolder, Path.Combine(dirHash, name));
63+
return Path.ChangeExtension(stubFile, ".pyi");
8164
}
8265

8366
public string ReadCachedModule(string filePath) {
84-
if (_skipCache) {
85-
return string.Empty;
86-
}
87-
8867
var cachePath = GetCacheFilePath(filePath);
8968
if (string.IsNullOrEmpty(cachePath)) {
9069
return string.Empty;
@@ -117,26 +96,26 @@ public string ReadCachedModule(string filePath) {
11796
exception = ex;
11897
}
11998

120-
var reason = "Unknown";
99+
string reason;
121100
if (!cachedFileExists) {
122101
reason = "Cached file does not exist";
123102
} else if (cachedFileOlderThanAssembly) {
124103
reason = "Cached file is older than the assembly.";
125104
} else if (cachedFileOlderThanSource) {
126105
reason = $"Cached file is older than the source {filePath}.";
127-
} else if (exception != null) {
106+
} else {
128107
reason = $"Exception during cache file check {exception.Message}.";
129108
}
130109

131-
_log?.Log(TraceEventType.Verbose, $"Invalidate cached module {cachePath}. Reason: {reason}");
110+
_log?.Log(TraceEventType.Verbose, $"Invalidated cached module {cachePath}. Reason: {reason}");
132111
_fs.DeleteFileWithRetries(cachePath);
133112
return string.Empty;
134113
}
135114

136115
public void WriteCachedModule(string filePath, string code) {
137116
var cache = GetCacheFilePath(filePath);
138117
if (!string.IsNullOrEmpty(cache)) {
139-
_log?.Log(TraceEventType.Verbose, "Write cached module: ", cache);
118+
_log?.Log(TraceEventType.Verbose, "Writing cached module: ", cache);
140119
// Don't block analysis on cache writes.
141120
CacheWritingTask = Task.Run(() => _fs.WriteTextWithRetry(cache, code));
142121
CacheWritingTask.DoNotWait();

‎src/Analysis/Ast/Impl/Modules/CompiledPythonModule.cs‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,14 @@
1717
using System.Collections.Generic;
1818
using System.Diagnostics;
1919
using System.Text;
20+
using Microsoft.Python.Analysis.Caching;
2021
using Microsoft.Python.Analysis.Types;
2122
using Microsoft.Python.Core;
2223
using Microsoft.Python.Core.IO;
2324

2425
namespace Microsoft.Python.Analysis.Modules {
2526
internal class CompiledPythonModule : PythonModule {
26-
protected IModuleCache ModuleCache => Interpreter.ModuleResolution.ModuleCache;
27+
protected IStubCache StubCache => Interpreter.ModuleResolution.StubCache;
2728

2829
public CompiledPythonModule(string moduleName, ModuleType moduleType, string filePath, IPythonModule stub, IServiceContainer services)
2930
: base(moduleName, filePath, moduleType, stub, services) { }
@@ -53,7 +54,7 @@ protected virtual string[] GetScrapeArguments(IPythonInterpreter interpreter) {
5354

5455
protected override string LoadContent() {
5556
// Exceptions are handled in the base
56-
var code = ModuleCache.ReadCachedModule(FilePath);
57+
var code = StubCache.ReadCachedModule(FilePath);
5758
if (string.IsNullOrEmpty(code)) {
5859
if (!FileSystem.FileExists(Interpreter.Configuration.InterpreterPath)) {
5960
return string.Empty;
@@ -65,7 +66,7 @@ protected override string LoadContent() {
6566
return code;
6667
}
6768

68-
protected virtual void SaveCachedCode(string code) => ModuleCache.WriteCachedModule(FilePath, code);
69+
protected virtual void SaveCachedCode(string code) => StubCache.WriteCachedModule(FilePath, code);
6970

7071
private string ScrapeModule() {
7172
var args = GetScrapeArguments(Interpreter);

‎src/Analysis/Ast/Impl/Modules/Definitions/IModuleManagement.cs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
using System;
1717
using System.Collections.Generic;
1818
using System.Threading;
19+
using Microsoft.Python.Analysis.Caching;
1920
using Microsoft.Python.Analysis.Core.Interpreter;
2021
using Microsoft.Python.Analysis.Types;
2122

@@ -33,7 +34,10 @@ public interface IModuleManagement: IModuleResolution {
3334

3435
IReadOnlyCollection<string> GetPackagesFromDirectory(string searchPath, CancellationToken cancellationToken = default);
3536

36-
IModuleCache ModuleCache { get; }
37+
/// <summary>
38+
/// Cache of module stubs generated from compiled modules.
39+
/// </summary>
40+
IStubCache StubCache { get; }
3741

3842
bool TryAddModulePath(in string path, in bool allowNonRooted, out string fullName);
3943

‎src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
using System.Threading;
2323
using System.Threading.Tasks;
2424
using Microsoft.Python.Analysis.Analyzer;
25+
using Microsoft.Python.Analysis.Caching;
2526
using Microsoft.Python.Analysis.Core.DependencyResolution;
2627
using Microsoft.Python.Analysis.Core.Interpreter;
2728
using Microsoft.Python.Analysis.Documents;
@@ -46,8 +47,8 @@ internal IBuiltinsPythonModule CreateBuiltinsModule() {
4647
// Initialize built-in
4748
var moduleName = BuiltinTypeId.Unknown.GetModuleName(_interpreter.LanguageVersion);
4849

49-
ModuleCache = new ModuleCache(_interpreter, _services);
50-
var modulePath = ModuleCache.GetCacheFilePath(_interpreter.Configuration.InterpreterPath);
50+
StubCache = _services.GetService<IStubCache>();
51+
var modulePath = StubCache.GetCacheFilePath(_interpreter.Configuration.InterpreterPath);
5152

5253
var b = new BuiltinsPythonModule(moduleName, modulePath, _services);
5354
BuiltinsModule = b;

‎src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
using System.IO;
1919
using System.Linq;
2020
using System.Threading;
21+
using Microsoft.Python.Analysis.Caching;
2122
using Microsoft.Python.Analysis.Core.DependencyResolution;
2223
using Microsoft.Python.Analysis.Core.Interpreter;
2324
using Microsoft.Python.Analysis.Documents;
@@ -80,6 +81,8 @@ public IReadOnlyCollection<string> GetPackagesFromDirectory(string searchPath, C
8081
).Select(mp => mp.ModuleName).Where(n => !string.IsNullOrEmpty(n)).TakeWhile(_ => !cancellationToken.IsCancellationRequested).ToList();
8182
}
8283

84+
public IStubCache StubCache { get; protected set; }
85+
8386
public IPythonModule GetImportedModule(string name)
8487
=> Modules.TryGetValue(name, out var moduleRef) ? moduleRef.Value : _interpreter.ModuleResolution.GetSpecializedModule(name);
8588

0 commit comments

Comments
 (0)