Skip to content

Commit ff598b6

Browse files
SteveIvesSteveIves
committed
Codex solution to evening out the cross thread work on initial sync.
1 parent e730189 commit ff598b6

3 files changed

Lines changed: 232 additions & 65 deletions

File tree

‎SFTPSync.chm-keep‎

-1.32 MB
Binary file not shown.

‎SFTPSyncLib/RemoteSync.cs‎

Lines changed: 181 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
using Renci.SshNet;
22
using Renci.SshNet.Sftp;
3+
using System.Collections.Concurrent;
34

45
namespace SFTPSyncLib
56
{
@@ -53,6 +54,94 @@ public RemoteSync(string host, string username, string password,
5354
});
5455
}
5556

57+
public RemoteSync(string host, string username, string password,
58+
string localRootDirectory, string remoteRootDirectory,
59+
string searchPattern, SyncDirector director, List<string>? excludedFolders, Task initialSyncTask)
60+
{
61+
_host = host;
62+
_username = username;
63+
_password = password;
64+
_searchPattern = searchPattern;
65+
_localRootDirectory = localRootDirectory;
66+
_remoteRootDirectory = remoteRootDirectory;
67+
_director = director;
68+
_excludedFolders = excludedFolders ?? new List<string>();
69+
_sftp = new SftpClient(host, username, password);
70+
_sftp.Connect();
71+
72+
DoneMakingFolders = Task.CompletedTask;
73+
DoneInitialSync = initialSyncTask;
74+
75+
DoneInitialSync.ContinueWith((tmp) =>
76+
{
77+
_director.AddCallback(searchPattern, (args) => Fsw_Changed(null, args));
78+
});
79+
}
80+
81+
public static async Task RunSharedInitialSyncAsync(
82+
string host,
83+
string username,
84+
string password,
85+
string localRootDirectory,
86+
string remoteRootDirectory,
87+
string[] searchPatterns,
88+
List<string>? excludedFolders,
89+
int workerCount)
90+
{
91+
if (workerCount <= 0 || searchPatterns.Length == 0)
92+
{
93+
return;
94+
}
95+
96+
try
97+
{
98+
using (var sftp = new SftpClient(host, username, password))
99+
{
100+
sftp.Connect();
101+
await CreateDirectoriesInternal(sftp, localRootDirectory, localRootDirectory, remoteRootDirectory, excludedFolders);
102+
}
103+
}
104+
catch (Exception ex)
105+
{
106+
Logger.LogError($"Failed to create directories. Exception: {ex.Message}");
107+
throw;
108+
}
109+
110+
var workQueue = new ConcurrentQueue<SyncWorkItem>();
111+
foreach (var pair in EnumerateLocalDirectories(localRootDirectory, remoteRootDirectory, excludedFolders))
112+
{
113+
foreach (var pattern in searchPatterns)
114+
{
115+
workQueue.Enqueue(new SyncWorkItem(pair.LocalPath, pair.RemotePath, pattern));
116+
}
117+
}
118+
119+
var workers = new List<Task>();
120+
for (int i = 0; i < workerCount; i++)
121+
{
122+
workers.Add(Task.Run(async () =>
123+
{
124+
using (var sftp = new SftpClient(host, username, password))
125+
{
126+
sftp.Connect();
127+
while (workQueue.TryDequeue(out var item))
128+
{
129+
try
130+
{
131+
await SyncDirectoryAsync(sftp, item.LocalPath, item.RemotePath, item.SearchPattern);
132+
}
133+
catch (Exception ex)
134+
{
135+
Logger.LogError($"Failed to sync {item.LocalPath} ({item.SearchPattern}). Exception: {ex.Message}");
136+
}
137+
}
138+
}
139+
}));
140+
}
141+
142+
await Task.WhenAll(workers);
143+
}
144+
56145
/// <summary>
57146
/// Sync changes for a file. This is only used for changes AFTER the initial sync has completed.
58147
/// </summary>
@@ -120,30 +209,7 @@ public static Task<IEnumerable<FileInfo>> SyncDirectoryAsync(SftpClient sftp, st
120209

121210
private string[] FilteredDirectories(string localPath)
122211
{
123-
return Directory.GetDirectories(localPath).Where(path =>
124-
{
125-
var relativePath = path.Substring(_localRootDirectory.Length);
126-
127-
// Existing exclusions
128-
bool isExcluded = relativePath.EndsWith(".git")
129-
|| relativePath.EndsWith(".vs")
130-
|| relativePath.EndsWith("bin")
131-
|| relativePath.EndsWith("obj")
132-
|| relativePath.Contains(".");
133-
134-
// Check _excludedFolders
135-
if (!isExcluded && _excludedFolders != null && _excludedFolders.Count > 0)
136-
{
137-
string fullPath = Path.GetFullPath(path).TrimEnd(Path.DirectorySeparatorChar);
138-
isExcluded = _excludedFolders.Any(excluded =>
139-
{
140-
var excludedFullPath = Path.GetFullPath(excluded).TrimEnd(Path.DirectorySeparatorChar);
141-
return string.Equals(fullPath, excludedFullPath, StringComparison.OrdinalIgnoreCase);
142-
});
143-
}
144-
145-
return !isExcluded;
146-
}).ToArray();
212+
return FilteredDirectories(_localRootDirectory, localPath, _excludedFolders);
147213
}
148214

149215
public async Task CreateDirectories(string localPath, string remotePath)
@@ -155,30 +221,7 @@ public async Task CreateDirectories(string localPath, string remotePath)
155221
if (!EnsureConnectedSafe())
156222
return;
157223

158-
//Got local directories to sync
159-
var localDirectories = FilteredDirectories(localPath);
160-
161-
//Get remote directories
162-
var remoteDirectories = (await ListDirectoryAsync(_sftp, remotePath)).Where(item => item.IsDirectory).ToDictionary(item =>
163-
{
164-
if (item.Name.Contains(".DIR", StringComparison.OrdinalIgnoreCase))
165-
return item.Name.Remove(item.Name.IndexOf(".DIR", StringComparison.OrdinalIgnoreCase));
166-
else
167-
return item.Name;
168-
});
169-
170-
//Compare local and remote directories, creating missing ones, and recurse for subdirectories
171-
foreach (var item in localDirectories)
172-
{
173-
var directoryName = item.Split(Path.DirectorySeparatorChar).Last();
174-
if (!remoteDirectories.ContainsKey(directoryName))
175-
{
176-
//Create new remote directory
177-
_sftp.CreateDirectory(remotePath + "/" + directoryName);
178-
}
179-
//And recurse for any subdirectories it may need
180-
await CreateDirectories(localPath + "\\" + directoryName, remotePath + "/" + directoryName);
181-
}
224+
await CreateDirectoriesInternal(_sftp, _localRootDirectory, localPath, remotePath, _excludedFolders);
182225
}
183226
catch (Exception)
184227
{
@@ -224,6 +267,82 @@ public async Task InitialSync(string localPath, string remotePath)
224267
await SyncDirectoryAsync(_sftp, localPath, remotePath, _searchPattern);
225268
}
226269

270+
private static string[] FilteredDirectories(string localRootDirectory, string localPath, List<string>? excludedFolders)
271+
{
272+
return Directory.GetDirectories(localPath).Where(path =>
273+
{
274+
var relativePath = path.Substring(localRootDirectory.Length);
275+
276+
bool isExcluded = relativePath.EndsWith(".git")
277+
|| relativePath.EndsWith(".vs")
278+
|| relativePath.EndsWith("bin")
279+
|| relativePath.EndsWith("obj")
280+
|| relativePath.Contains(".");
281+
282+
if (!isExcluded && excludedFolders != null && excludedFolders.Count > 0)
283+
{
284+
string fullPath = Path.GetFullPath(path).TrimEnd(Path.DirectorySeparatorChar);
285+
isExcluded = excludedFolders.Any(excluded =>
286+
{
287+
var excludedFullPath = Path.GetFullPath(excluded).TrimEnd(Path.DirectorySeparatorChar);
288+
return string.Equals(fullPath, excludedFullPath, StringComparison.OrdinalIgnoreCase);
289+
});
290+
}
291+
292+
return !isExcluded;
293+
}).ToArray();
294+
}
295+
296+
private static IEnumerable<(string LocalPath, string RemotePath)> EnumerateLocalDirectories(
297+
string localRootDirectory,
298+
string remoteRootDirectory,
299+
List<string>? excludedFolders)
300+
{
301+
var stack = new Stack<(string LocalPath, string RemotePath)>();
302+
stack.Push((localRootDirectory, remoteRootDirectory));
303+
304+
while (stack.Count > 0)
305+
{
306+
var current = stack.Pop();
307+
yield return current;
308+
309+
foreach (var directory in FilteredDirectories(localRootDirectory, current.LocalPath, excludedFolders))
310+
{
311+
var directoryName = directory.Split(Path.DirectorySeparatorChar).Last();
312+
var remotePath = current.RemotePath + "/" + directoryName;
313+
stack.Push((directory, remotePath));
314+
}
315+
}
316+
}
317+
318+
private static async Task CreateDirectoriesInternal(
319+
SftpClient sftp,
320+
string localRootDirectory,
321+
string localPath,
322+
string remotePath,
323+
List<string>? excludedFolders)
324+
{
325+
var localDirectories = FilteredDirectories(localRootDirectory, localPath, excludedFolders);
326+
327+
var remoteDirectories = (await ListDirectoryAsync(sftp, remotePath)).Where(item => item.IsDirectory).ToDictionary(item =>
328+
{
329+
if (item.Name.Contains(".DIR", StringComparison.OrdinalIgnoreCase))
330+
return item.Name.Remove(item.Name.IndexOf(".DIR", StringComparison.OrdinalIgnoreCase));
331+
else
332+
return item.Name;
333+
});
334+
335+
foreach (var item in localDirectories)
336+
{
337+
var directoryName = item.Split(Path.DirectorySeparatorChar).Last();
338+
if (!remoteDirectories.ContainsKey(directoryName))
339+
{
340+
sftp.CreateDirectory(remotePath + "/" + directoryName);
341+
}
342+
await CreateDirectoriesInternal(sftp, localRootDirectory, localPath + "\\" + directoryName, remotePath + "/" + directoryName, excludedFolders);
343+
}
344+
}
345+
227346

228347
public static Task UploadFileAsync(SftpClient sftp, Stream file, string destination)
229348
{
@@ -412,5 +531,19 @@ public void Dispose()
412531
_sftp.Dispose();
413532
}
414533
}
534+
535+
private sealed class SyncWorkItem
536+
{
537+
public SyncWorkItem(string localPath, string remotePath, string searchPattern)
538+
{
539+
LocalPath = localPath;
540+
RemotePath = remotePath;
541+
SearchPattern = searchPattern;
542+
}
543+
544+
public string LocalPath { get; }
545+
public string RemotePath { get; }
546+
public string SearchPattern { get; }
547+
}
415548
}
416549
}

‎SFTPSyncUI/SFTPSyncUI.cs‎

Lines changed: 51 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
using SFTPSyncLib;
44
using System.IO.Pipes;
55
using System.Reflection;
6+
using System.Linq;
67

78
namespace SFTPSyncUI
89
{
@@ -211,27 +212,51 @@ public static async void StartSync(Action<string> loggerAction)
211212

212213
var director = new SyncDirector(settings.LocalPath);
213214

214-
foreach (var pattern in settings.LocalSearchPattern.Split(';', StringSplitOptions.RemoveEmptyEntries))
215+
var patterns = settings.LocalSearchPattern
216+
.Split(';', StringSplitOptions.RemoveEmptyEntries)
217+
.Select(pattern => pattern.Trim())
218+
.Where(pattern => pattern.Length > 0)
219+
.ToArray();
220+
221+
if (patterns.Length == 0)
222+
{
223+
Logger.LogError("No valid search patterns were configured.");
224+
return;
225+
}
226+
227+
Task initialSyncTask;
228+
try
229+
{
230+
initialSyncTask = RemoteSync.RunSharedInitialSyncAsync(
231+
settings.RemoteHost,
232+
settings.RemoteUsername,
233+
DPAPIEncryption.Decrypt(settings.RemotePassword),
234+
settings.LocalPath,
235+
settings.RemotePath,
236+
patterns,
237+
settings.ExcludedDirectories,
238+
patterns.Length);
239+
}
240+
catch (Exception ex)
241+
{
242+
Logger.LogError($"Failed to start initial sync. Exception: {ex.Message}");
243+
return;
244+
}
245+
246+
foreach (var pattern in patterns)
215247
{
216-
if (RemoteSyncWorkers.Count > 0)
217-
{
218-
//The first sync worker will create all of the remote folders before it starts,
219-
//to sync files matching it's pattern so wait for it to finish before starting
220-
//the remaining sync workers
221-
await RemoteSyncWorkers[0].DoneMakingFolders;
222-
}
223248
try
224249
{
225250
RemoteSyncWorkers.Add(new RemoteSync(
226-
settings.RemoteHost,
227-
settings.RemoteUsername,
228-
DPAPIEncryption.Decrypt(settings.RemotePassword),
229-
settings.LocalPath,
230-
settings.RemotePath,
231-
pattern,
232-
RemoteSyncWorkers.Count == 0, // Only the first worker will create remote folders
251+
settings.RemoteHost,
252+
settings.RemoteUsername,
253+
DPAPIEncryption.Decrypt(settings.RemotePassword),
254+
settings.LocalPath,
255+
settings.RemotePath,
256+
pattern,
233257
director,
234-
settings.ExcludedDirectories));
258+
settings.ExcludedDirectories,
259+
initialSyncTask));
235260

236261
Logger.LogInfo($"Started sync worker {RemoteSyncWorkers.Count} for pattern {pattern}");
237262
}
@@ -242,7 +267,16 @@ public static async void StartSync(Action<string> loggerAction)
242267
}
243268

244269
//Wait for all sync workers to finish initial sync then tell the user
245-
await Task.WhenAll(RemoteSyncWorkers.Select(rsw => rsw.DoneInitialSync));
270+
try
271+
{
272+
await initialSyncTask;
273+
}
274+
catch (Exception ex)
275+
{
276+
Logger.LogError($"Initial sync failed. Exception: {ex.Message}");
277+
mainForm?.SetStatusBarText("Initial sync failed");
278+
return;
279+
}
246280

247281
Logger.LogInfo("Initial sync complete, real-time sync active");
248282
mainForm?.SetStatusBarText("Real time sync active");

0 commit comments

Comments
 (0)