Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions Defs/KeyBindings.xml
Original file line number Diff line number Diff line change
Expand Up @@ -31,4 +31,10 @@
<defaultKeyCodeA>Keypad0</defaultKeyCodeA>
</KeyBindingDef>

<!-- No default key: the menu is reachable from the wheel + in-game UI, so an opt-in hotkey avoids colliding with mod-added bindings. -->
<KeyBindingDef ParentName="MultiplayerKeyBinding">
<defName>MpTogglePingMenu</defName>
<label>toggle ping menu</label>
</KeyBindingDef>

</Defs>
89 changes: 88 additions & 1 deletion Source/Client/Comp/Game/MultiplayerGameComp.cs
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.Linq;
using HarmonyLib;
using Multiplayer.API;
using Multiplayer.Client.Saving;
using Multiplayer.Common;
using Multiplayer.Common.Networking.Packet;
using UnityEngine;
using Verse;

namespace Multiplayer.Client.Comp
Expand All @@ -21,6 +24,45 @@ public class MultiplayerGameComp : IExposable, IHasSessionData

public string idBlockBase64;

// Bucketed by placer faction loadID; SortedDictionary guarantees identical enumeration on every client.
public SortedDictionary<int, List<PingInfo>> markersByFaction = new();
public int nextMarkerId;

// Bumped on every markersByFaction mutation; read by PingMenuWindow's row cache. Runtime-only.
public int markersVersion;

// Host-authoritative, copied from ServerSettings at game start - every client must agree (drives FIFO eviction).
public int markerCapPerPlayer = PingMarkerCap.Default;

// Materialized merge of markersByFaction.Values; rebuilt on markersVersion change.
private readonly List<PingInfo> cachedAllMarkers = new();
private int cachedAllMarkersVersion = -1;

public IReadOnlyList<PingInfo> AllMarkers
{
get
{
if (cachedAllMarkersVersion != markersVersion)
{
cachedAllMarkers.Clear();
foreach (var bucket in markersByFaction.Values)
cachedAllMarkers.AddRange(bucket);
cachedAllMarkersVersion = markersVersion;
}
return cachedAllMarkers;
}
}

public List<PingInfo> GetOrCreateFactionMarkers(int factionLoadId)
{
if (!markersByFaction.TryGetValue(factionLoadId, out var bucket))
{
bucket = new List<PingInfo>();
markersByFaction[factionLoadId] = bucket;
}
return bucket;
}

public bool IsLowestWins => timeControl == TimeControl.LowestWins;

public PlayerData LocalPlayerDataOrNull => playerData.GetValueOrDefault(Multiplayer.session.playerId);
Expand All @@ -35,6 +77,27 @@ public void ExposeData()
Scribe_Values.Look(ref timeControl, "timeControl");
Scribe_Values.Look(ref nextSessionId, "nextSessionId");

// Re-bucket must run in LoadingVars - Scribe_Collections.Look only populates the ref then.
List<PingInfo> markersFlat = Scribe.mode == LoadSaveMode.Saving ? AllMarkers.ToList() : null;
Scribe_Collections.Look(ref markersFlat, "mpMarkers", LookMode.Deep);
if (Scribe.mode == LoadSaveMode.LoadingVars)
{
// Always drop stale select-times; loaded ids may overlap last-session marker ids.
LocationPings.DropStaleSelectTimes();
if (markersFlat != null)
{
markersByFaction = new SortedDictionary<int, List<PingInfo>>();
foreach (var m in markersFlat)
GetOrCreateFactionMarkers(m.placedByFactionLoadId).Add(m);
markersVersion++;
}
}
Scribe_Values.Look(ref nextMarkerId, "mpNextMarkerId");
Scribe_Values.Look(ref markerCapPerPlayer, "mpMarkerCapPerPlayer", PingMarkerCap.Default);
if (Scribe.mode == LoadSaveMode.LoadingVars)
// Hand-edited saves can land out-of-range values.
markerCapPerPlayer = PingMarkerCap.Clamp(markerCapPerPlayer);

// Store for back-compat conversion in GameExposeComponentsPatch
if (Scribe.mode == LoadSaveMode.LoadingVars)
Scribe_Values.Look(ref idBlockBase64, "globalIdBlock");
Expand All @@ -43,12 +106,30 @@ public void ExposeData()
public void WriteSessionData(ByteWriter writer)
{
SyncSerialization.WriteSync(writer, playerData);

// Joiner needs host's live marker list, not the autosave - markers placed between
// save and join would otherwise be missing, and nextMarkerId would diverge.
SyncSerialization.WriteSync(writer, AllMarkers.ToList());
SyncSerialization.WriteSync(writer, Math.Max(0, nextMarkerId));
SyncSerialization.WriteSync(writer, PingMarkerCap.Clamp(markerCapPerPlayer));
}

public void ReadSessionData(ByteReader reader)
{
playerData = SyncSerialization.ReadSync<Dictionary<int, PlayerData>>(reader);
DebugSettings.godMode = LocalPlayerDataOrNull?.godMode ?? false;

var markersFlat = SyncSerialization.ReadSync<List<PingInfo>>(reader);
// Negative ids would alias with legacy markerId == 0 rows.
nextMarkerId = Math.Max(0, SyncSerialization.ReadSync<int>(reader));
markerCapPerPlayer = PingMarkerCap.Clamp(SyncSerialization.ReadSync<int>(reader));

// Session data is fresher than the autosave the joiner just loaded - overwrite.
markersByFaction = new SortedDictionary<int, List<PingInfo>>();
if (markersFlat != null)
foreach (var m in markersFlat)
GetOrCreateFactionMarkers(m.placedByFactionLoadId).Add(m);
markersVersion++;
}

[SyncMethod(debugOnly = true)]
Expand All @@ -57,6 +138,12 @@ public void SetGodMode(int playerId, bool godMode)
playerData[playerId].godMode = godMode;
}

[SyncMethod]
public void SetMarkerCapPerPlayer(int newCap)
{
markerCapPerPlayer = PingMarkerCap.Clamp(newCap);
}

public TimeSpeed GetLowestTimeVote(int tickableId, bool excludePaused = false)
{
return (TimeSpeed)playerData.Values
Expand Down
3 changes: 2 additions & 1 deletion Source/Client/Debug/DebugActions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,8 @@ public static void ShowDesync()
{
Find.WindowStack.Add(new DesyncedWindow(
"Debug action",
new SaveableDesyncInfo(Multiplayer.game.sync, new ClientSyncOpinion(0), new ClientSyncOpinion(0), 0, true)
new SaveableDesyncInfo(Multiplayer.game.sync, new ClientSyncOpinion(0), new ClientSyncOpinion(0), 0, true,
new SaveableDesyncInfo.SnapshotFreshness(IsFresh: false, ElapsedMs: 0, SnapshotTick: -1, DesyncTick: -1, FallbackReason: "debug action - no live snapshot"))
));
}

Expand Down
37 changes: 33 additions & 4 deletions Source/Client/Desyncs/SaveableDesyncInfo.cs
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,13 @@ public class SaveableDesyncInfo(
ClientSyncOpinion local,
ClientSyncOpinion remote,
int diffAt,
bool diffAtFound)
bool diffAtFound,
SaveableDesyncInfo.SnapshotFreshness snapshotFreshness)
{
public readonly ClientSyncOpinion local = local;
public readonly ClientSyncOpinion remote = remote;
public readonly int diffAt = diffAt;
public readonly SnapshotFreshness snapshotFreshness = snapshotFreshness;
private readonly Task<string> metadata = Task.Run(MetadataGenerator.Generate);
private readonly Task<FileInfo> replay = Task.Run(SaveReplayIfApplicable);

Expand Down Expand Up @@ -60,9 +62,10 @@ public void Save([CanBeNull] HostInfo hostInfo)
}
catch (AggregateException e)
{
if (e.InnerExceptions.SingleOrDefault(inner => inner is TaskCanceledException) == null) throw;
if (!e.InnerExceptions.Any(inner => inner is TaskCanceledException)) throw;
}
if (replay.IsCompletedSuccessfully) {
if (replay.IsCompletedSuccessfully)
{
var replayFile = replay.Result;
zip.CreateEntryFromFile(replayFile.FullName, "replay.rwmts", CompressionLevel.NoCompression);
DeleteFileSilent(replayFile);
Expand All @@ -71,9 +74,11 @@ public void Save([CanBeNull] HostInfo hostInfo)
catch (Exception e)
{
Log.Error($"Exception writing desync info: {e}");
if (replay.IsCompletedSuccessfully)
DeleteFileSilent(replay.Result);
}

Log.Message($"Desync info writing took {watch.ElapsedMilliseconds}");
Log.Message($"Desync info writing took {watch.ElapsedMilliseconds} ms");
}

private string GetLocalTraces()
Expand Down Expand Up @@ -106,6 +111,17 @@ private string GetDesyncDetails()
{
var desyncInfo = new StringBuilder();

// gameComp can be null if the session was torn down between desync and Save click.
var comp = Multiplayer.game?.gameComp;
var markerCount = comp?.AllMarkers.Count.ToStringSafe() ?? "n/a";
var nextMarkerId = comp?.nextMarkerId.ToStringSafe() ?? "n/a";
var markerCap = comp?.markerCapPerPlayer.ToStringSafe() ?? "n/a";

// SnapshotTick == -1 means we have no snapshot to compare against (e.g. DebugActions.ShowDesync).
var lag = snapshotFreshness.SnapshotTick >= 0
? (snapshotFreshness.DesyncTick - snapshotFreshness.SnapshotTick).ToStringSafe()
: "n/a";

desyncInfo
.AppendLine("###Tick Data###")
.AppendLine($"Arbiter Connected And Playing|||{Multiplayer.session.ArbiterPlaying}")
Expand All @@ -123,6 +139,16 @@ private string GetDesyncDetails()
.AppendLine($"Async time active|||{Multiplayer.GameComp.asyncTime}")
.AppendLine($"Multifaction active|||{Multiplayer.GameComp.multifaction}")
.AppendLine($"Map Count|||{Find.Maps?.Count.ToStringSafe()}")
.AppendLine($"Marker Count|||{markerCount}")
.AppendLine($"Next Marker Id|||{nextMarkerId}")
.AppendLine($"Marker Cap Per Player|||{markerCap}")
.AppendLine("\n###Replay Snapshot Freshness###")
.AppendLine($"Snapshot Tick|||{snapshotFreshness.SnapshotTick}")
.AppendLine($"Desync Tick|||{snapshotFreshness.DesyncTick}")
.AppendLine($"Snapshot Lag Ticks|||{lag}")
.AppendLine($"Refresh Succeeded|||{snapshotFreshness.IsFresh}")
.AppendLine($"Refresh Elapsed (ms)|||{snapshotFreshness.ElapsedMs}")
.AppendLine($"Fallback Reason|||{snapshotFreshness.FallbackReason ?? "n/a"}")
.AppendLine("\n###CPU Info###")
.AppendLine($"Processor Name|||{SystemInfo.processorType}")
.AppendLine($"Processor Speed (MHz)|||{SystemInfo.processorFrequency}")
Expand Down Expand Up @@ -196,4 +222,7 @@ private static void DeleteFileSilent(FileInfo file)
}

public record HostInfo([CanBeNull] string Traces, [CanBeNull] string JittedMethods);

// SnapshotTick == DesyncTick when refresh succeeded; otherwise stale (autosave-aligned).
public record SnapshotFreshness(bool IsFresh, long ElapsedMs, int SnapshotTick, int DesyncTick, [CanBeNull] string FallbackReason);
}
50 changes: 49 additions & 1 deletion Source/Client/Desyncs/SyncCoordinator.cs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using Multiplayer.Client.Desyncs;
using Multiplayer.Client.Util;
Expand Down Expand Up @@ -128,13 +129,60 @@ private void HandleDesync(ClientSyncOpinion oldOpinion, ClientSyncOpinion newOpi
var diffAt = FindTraceHashesDiffTick(local, remote, out var found);
Multiplayer.Client.Send(new ClientDesyncedPacket(local.startTick, diffAt));

var snapshotInfo = TryRefreshSnapshotForDesync();
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Any more details about this feature?

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure no problem. I primarily used this on my side to get quicker debug info on desyncs I was hitting during dev. Essentially it gives you a quick snap point to when the desync got detected which is usually a few ticks after the actual divergence, but there's a Snapshot Lag Ticks field in the desync_info that tells you relatively how far behind it ended up. So you can load up the replay and look at what the world state was around the bad tick, instead of from the last cached snapshot.

Regarding what info it actually gives you, you can extract the replay.rwmts from the desync zip and use dev tools to inspect what was happening at that moment. For the marker work that was mostly visually checking which markers each client actually had on the map at a bad tick, combined with the stack traces in local_traces.txt that gave me a quick way to spot something like "client 1 has marker X but client 2 doesn't" without having to manually step to get there.

Implementation wise it's basically wrapping SaveLoad.SaveGameData + CreateGameDataSnapshot to grab the current state locally, without sending anything to the server or triggering a join point for other players. Runs on the main thread since I believe Scribe and Find.Maps aren't thread-safe.

Probably should've taken this out of the PR though, it was actually just a quick thing I added to get more debug info on my side and isn't really related to the category-pings PR. The other thing is, the way it's done currently means the embedded replay in the desync zip starts at the desync tick instead of the earlier cached snapshot, so you lose the ability to forward-sim from that earlier point and see the points leading up to the bad ticks, which is most probably more useful than the snapshot of information I was using. I was actually thinking of doing this as a separate PR where we maybe expand the desync zip to include both the snap-to-the-point snapshot for quick debugging of the divergence itself and then the full replay version for seeing what led up to it. Would you like me to remove this from the current PR and possibly create a new PR for it or just scrap it fully?

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's a nice idea, but let's break this out into a separate PR, it's out of scope here and could use some refinement.

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Noted, will be pushing a commit soon with updates for all of your comments 😁


MpUI.ClearWindowStack();
Find.WindowStack.Add(new DesyncedWindow(
desyncMessage,
new SaveableDesyncInfo(this, local, remote, diffAt, found)
new SaveableDesyncInfo(this, local, remote, diffAt, found, snapshotInfo)
));
}

// Refresh dataSnapshot synchronously on the main thread so the embedded replay.rwmts
// captures the divergent tick instead of the (default 5 min stale) autosave-aligned join
// point. SaveGameData isn't thread-safe (Scribe, Find.Maps). No SendGameData - peers must
// not see a join-point broadcast spawned by a desync. If the capture overruns the budget
// below we keep the stale snapshot, since we can't predict the cost ahead of time.
private const int SnapshotRefreshBudgetMs = 2000;

private static SaveableDesyncInfo.SnapshotFreshness TryRefreshSnapshotForDesync()
{
var staleAt = Multiplayer.session.dataSnapshot?.CachedAtTime ?? -1;
var nowTick = TickPatch.Timer;
var watch = Stopwatch.StartNew();

try
{
var fresh = Replay.CaptureLocalSnapshot();
watch.Stop();

if (watch.ElapsedMilliseconds > SnapshotRefreshBudgetMs)
{
Log.Warning(
$"[MP] Desync snapshot capture took {watch.ElapsedMilliseconds} ms " +
$"(budget {SnapshotRefreshBudgetMs} ms); keeping stale snapshot from tick {staleAt}.");
return new SaveableDesyncInfo.SnapshotFreshness(
IsFresh: false, ElapsedMs: watch.ElapsedMilliseconds, SnapshotTick: staleAt,
DesyncTick: nowTick, FallbackReason: $"capture exceeded {SnapshotRefreshBudgetMs} ms budget");
}

Multiplayer.session.dataSnapshot = fresh;
return new SaveableDesyncInfo.SnapshotFreshness(
IsFresh: true, ElapsedMs: watch.ElapsedMilliseconds, SnapshotTick: fresh.CachedAtTime,
DesyncTick: nowTick, FallbackReason: null);
}
catch (Exception e)
{
watch.Stop();
Log.Warning(
$"[MP] Desync snapshot capture threw after {watch.ElapsedMilliseconds} ms; " +
$"keeping stale snapshot from tick {staleAt}. Exception: {e.Message}");
return new SaveableDesyncInfo.SnapshotFreshness(
IsFresh: false, ElapsedMs: watch.ElapsedMilliseconds, SnapshotTick: staleAt,
DesyncTick: nowTick, FallbackReason: $"exception: {e.GetType().Name}: {e.Message}");
}
}

private static int FindTraceHashesDiffTick(ClientSyncOpinion local, ClientSyncOpinion remote, out bool found)
{
found = true;
Expand Down
4 changes: 4 additions & 0 deletions Source/Client/EarlyInit.cs
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,10 @@ internal static void InitSync()

internal static void LatePatches()
{
// Inject runtime translation keys (e.g. MarkerInspectTab labelKey) into the active language.
// Needed for keys that travel through vanilla code paths which hardcode .Translate().
PingRuntimeTranslations.Register();

if (MpVersion.IsDebug)
Log.Message("== Structure == \n" + SyncDict.syncWorkers.PrintStructure());
}
Expand Down
20 changes: 20 additions & 0 deletions Source/Client/Multiplayer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,26 @@ public static void StopMultiplayer()
OnMainThread.ClearScheduled();
LongEventHandler.ClearQueuedEvents();

// Faction loadIDs are per-world; mutes by loadID would silently apply to unrelated
// factions in the next session. Username mutes survive - usernames are stable.
if (settings is { } s && s.hiddenFactionLoadIds is { Count: > 0 })
{
s.hiddenFactionLoadIds.Clear();
MultiplayerLoader.Multiplayer.instance?.WriteSettings();
}
// PingInfo instances die with the session; sweep vanilla's static selectTimes dict.
LocationPings.DropStaleSelectTimes();
// Close ping windows individually so each runs PostClose (rect persistence).
// ClearWindowStack bypasses PostClose and their OnGUI dereferences soon-dead state.
var ws = Find.WindowStack;
if (ws != null)
{
ws.WindowOfType<PingLabelWindow>()?.Close(false);
ws.WindowOfType<PingMenuWindow>()?.Close(false);
ws.WindowOfType<PingFiltersDialog>()?.Close(false);
ws.WindowOfType<PingHostSettingsDialog>()?.Close(false);
}

if (session != null)
{
session.Stop();
Expand Down
10 changes: 10 additions & 0 deletions Source/Client/MultiplayerGame.cs
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,16 @@ public void ChangeRealPlayerFaction(Faction newFaction, bool regenMapDrawers = t

Find.MainTabsRoot?.EscapeCurrentTab();
Find.ColonistBar?.MarkColonistsDirty();

// regenMapDrawers:false is the internal SaveAndReload faction-switch (transient).
// Only close the rename modal and drop the gizmo cache on a real switch.
if (regenMapDrawers)
{
var loc = Multiplayer.session?.locationPings;
if (loc != null)
loc.cachedGizmos = null; // Next BuildGizmos rebuilds the key alongside.
Find.WindowStack?.WindowOfType<PingLabelWindow>()?.Close(false);
}
}
}
}
Loading
Loading