Skip to content

feature: allow partial stage/unstage/discard for non-UTF8 text #2260

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
love-linger merged 1 commit into from
Apr 14, 2026
+41 −36
Merged

feature: allow partial stage/unstage/discard for non-UTF8 text #2260

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 22 additions & 24 deletions src/Commands/Diff.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,24 +45,23 @@ public Diff(string repo, Models.DiffOption opt, int unified, bool ignoreWhitespa
using var proc = new Process();
proc.StartInfo = CreateGitStartInfo(true);
proc.Start();

var text = await proc.StandardOutput.ReadToEndAsync().ConfigureAwait(false);

using var ms = new System.IO.MemoryStream();
await proc.StandardOutput.BaseStream.CopyToAsync(ms, CancellationToken).ConfigureAwait(false);
var bytes = ms.ToArray();
var start = 0;
var end = text.IndexOf('\n', start);
while (end > 0)
while (start < bytes.Length)
{
var line = text[start..end];
ParseLine(line);

start = end + 1;
end = text.IndexOf('\n', start);
var end = Array.IndexOf(bytes, (byte)'\n', start);
Copy link
Copy Markdown
Collaborator

@love-linger love-linger Apr 14, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can not use (byte)'\n' which actually 0x0A to split lines without given encoding. For example:

LF in different encodings:

  • UTF-8: 0x0A
  • UTF-16BE: 0x000A
  • UTF-16LE: 0x0A00
  • UTF-32BE: 0x0000000A
  • UTF-32LE: 0x0A000000

When using 0x0A to split lines, there maybe some extra 0x00 byte remains

Copy link
Copy Markdown
Contributor Author

@gotoh gotoh Apr 14, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

UTF16/32 is not considered because git diff will treat it as a binary file unless we add --text.
So sourcegit cannot handle them at all.
In the case that utf16/32 is specified as working-tree-encoding in .gitattributes, git will output in utf-8, so this shouldn't be a problem, I think.

if (end < 0)
end = bytes.Length;
var next = end + 1;
if (start <= end - 1 && bytes[end - 1] == '\r')
end--;
if (!_result.IsBinary)
ParseLine(bytes[start..end]);
start = next;
}

if (start < text.Length)
ParseLine(text[start..]);

await proc.WaitForExitAsync().ConfigureAwait(false);
await proc.WaitForExitAsync(CancellationToken).ConfigureAwait(false);
}
catch
{
Expand All @@ -82,10 +81,9 @@ public Diff(string repo, Models.DiffOption opt, int unified, bool ignoreWhitespa
return _result;
}

private void ParseLine(string line)
private void ParseLine(byte[] lineBytes)
{
if (_result.IsBinary)
return;
var line = Encoding.UTF8.GetString(lineBytes);

if (line.StartsWith("old mode ", StringComparison.Ordinal))
{
Expand Down Expand Up @@ -168,7 +166,7 @@ private void ParseLine(string line)

_oldLine = int.Parse(match.Groups[1].Value);
_newLine = int.Parse(match.Groups[2].Value);
_last = new Models.TextDiffLine(Models.TextDiffLineType.Indicator, line, 0, 0);
_last = new Models.TextDiffLine(Models.TextDiffLineType.Indicator, lineBytes, 0, 0);
_result.TextDiff.Lines.Add(_last);
}
}
Expand All @@ -177,7 +175,7 @@ private void ParseLine(string line)
if (line.Length == 0)
{
ProcessInlineHighlights();
_last = new Models.TextDiffLine(Models.TextDiffLineType.Normal, "", _oldLine, _newLine);
_last = new Models.TextDiffLine(Models.TextDiffLineType.Normal, Array.Empty<byte>(), _oldLine, _newLine);
_result.TextDiff.Lines.Add(_last);
_oldLine++;
_newLine++;
Expand All @@ -195,7 +193,7 @@ private void ParseLine(string line)
}

_result.TextDiff.DeletedLines++;
_last = new Models.TextDiffLine(Models.TextDiffLineType.Deleted, line.Substring(1), _oldLine, 0);
_last = new Models.TextDiffLine(Models.TextDiffLineType.Deleted, lineBytes[1..], _oldLine, 0);
_deleted.Add(_last);
_oldLine++;
}
Expand All @@ -209,7 +207,7 @@ private void ParseLine(string line)
}

_result.TextDiff.AddedLines++;
_last = new Models.TextDiffLine(Models.TextDiffLineType.Added, line.Substring(1), 0, _newLine);
_last = new Models.TextDiffLine(Models.TextDiffLineType.Added, lineBytes[1..], 0, _newLine);
_added.Add(_last);
_newLine++;
}
Expand All @@ -221,7 +219,7 @@ private void ParseLine(string line)
{
_oldLine = int.Parse(match.Groups[1].Value);
_newLine = int.Parse(match.Groups[2].Value);
_last = new Models.TextDiffLine(Models.TextDiffLineType.Indicator, line, 0, 0);
_last = new Models.TextDiffLine(Models.TextDiffLineType.Indicator, lineBytes, 0, 0);
_result.TextDiff.Lines.Add(_last);
}
else
Expand All @@ -233,7 +231,7 @@ private void ParseLine(string line)
return;
}

_last = new Models.TextDiffLine(Models.TextDiffLineType.Normal, line.Substring(1), _oldLine, _newLine);
_last = new Models.TextDiffLine(Models.TextDiffLineType.Normal, lineBytes[1..], _oldLine, _newLine);
_result.TextDiff.Lines.Add(_last);
_oldLine++;
_newLine++;
Expand Down
31 changes: 19 additions & 12 deletions src/Models/DiffResult.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ public class TextRange(int p, int n)
public class TextDiffLine
{
public TextDiffLineType Type { get; set; } = TextDiffLineType.None;
public byte[] RawContent { get; set; } = [];
public string Content { get; set; } = "";
public int OldLineNumber { get; set; } = 0;
public int NewLineNumber { get; set; } = 0;
Expand All @@ -33,10 +34,13 @@ public class TextDiffLine
public string NewLine => NewLineNumber == 0 ? string.Empty : NewLineNumber.ToString();

public TextDiffLine() { }
public TextDiffLine(TextDiffLineType type, string content, int oldLine, int newLine)
public TextDiffLine(TextDiffLineType type, byte[] rawContent, int oldLine, int newLine)
{
if (rawContent == null)
throw new System.ArgumentNullException(nameof(rawContent));
Type = type;
Content = content;
Content = System.Text.Encoding.UTF8.GetString(rawContent);
RawContent = rawContent;
OldLineNumber = oldLine;
NewLineNumber = newLine;
}
Expand Down Expand Up @@ -158,7 +162,7 @@ public void GeneratePatchFromSelection(Change change, string fileTreeGuid, TextD
writer.WriteLine($"+++ b/{change.Path}");

// If last line of selection is a change. Find one more line.
string tail = null;
TextDiffLine tail = null;
if (selection.EndLine < Lines.Count)
{
var lastLine = Lines[selection.EndLine - 1];
Expand All @@ -173,7 +177,7 @@ public void GeneratePatchFromSelection(Change change, string fileTreeGuid, TextD
(revert && line.Type == TextDiffLineType.Added) ||
(!revert && line.Type == TextDiffLineType.Deleted))
{
tail = line.Content;
tail = line;
break;
}
}
Expand Down Expand Up @@ -256,8 +260,8 @@ public void GeneratePatchFromSelection(Change change, string fileTreeGuid, TextD
}
}

if (!string.IsNullOrEmpty(tail))
writer.WriteLine($" {tail}");
if (tail != null)
WriteLine(writer, ' ', tail);
writer.Flush();
}

Expand All @@ -273,7 +277,7 @@ public void GeneratePatchFromSelectionSingleSide(Change change, string fileTreeG
writer.WriteLine($"+++ b/{change.Path}");

// If last line of selection is a change. Find one more line.
string tail = null;
TextDiffLine tail = null;
if (selection.EndLine < Lines.Count)
{
var lastLine = Lines[selection.EndLine - 1];
Expand All @@ -288,15 +292,15 @@ public void GeneratePatchFromSelectionSingleSide(Change change, string fileTreeG
{
if (line.Type == TextDiffLineType.Normal || line.Type == TextDiffLineType.Added)
{
tail = line.Content;
tail = line;
break;
}
}
else
{
if (line.Type == TextDiffLineType.Normal || line.Type == TextDiffLineType.Deleted)
{
tail = line.Content;
tail = line;
break;
}
}
Expand Down Expand Up @@ -408,8 +412,8 @@ public void GeneratePatchFromSelectionSingleSide(Change change, string fileTreeG
}
}

if (!string.IsNullOrEmpty(tail))
writer.WriteLine($" {tail}");
if (tail != null)
WriteLine(writer, ' ', tail);
writer.Flush();
}

Expand Down Expand Up @@ -564,7 +568,10 @@ private bool ProcessIndicatorForPatchSingleSide(StreamWriter writer, TextDiffLin

private static void WriteLine(StreamWriter writer, char prefix, TextDiffLine line)
{
writer.WriteLine($"{prefix}{line.Content}");
writer.Write($"{prefix}");
writer.Flush();
writer.BaseStream.Write(line.RawContent); // write original bytes
writer.WriteLine();

if (line.NoNewLineEndOfFile)
writer.WriteLine("\\ No newline at end of file");
Expand Down