Support custom decoding error handler (#956)

* Support custom decoding error handler

* Eliminate data copy when decoding IPythonBuffer with PythonEncoding

* Eliminate data copy when decoding IPythonBuffer with PythonEncoding on net46

* Disable failing test on Mono

* Eliminate second encoding pass for common cases

* Update after review

* Disable failing test on Mono

Author: BCSharp

Src/IronPython/Runtime/Operations/StringOps.cs

@@ -1780,6 +1780,9 @@ internal static string RawDecode(CodeContext/*!*/ context, IBufferProtocol data,
         private static DecoderFallback ReplacementFallback = new DecoderReplacementFallback("\ufffd");
 
         internal static string DoDecode(CodeContext context, IPythonBuffer buffer, string? errors, string encoding, Encoding e, int numBytes = -1) {
+            // Precondition: only bytes-like buffers accepted
+            Debug.Assert(buffer.IsCContiguous());
+
             var span = buffer.AsReadOnlySpan();
             int start = GetStartingOffset(span, e);
             int length = (numBytes >= 0 ? numBytes : span.Length) - start;
@@ -1791,40 +1794,35 @@ Encoding setFallback(Encoding enc, DecoderFallback fb) {
                 enc.DecoderFallback = fb;
                 return enc;
             }
+            PythonEncoding? pe = null; // to avoid downcasting later
             switch (errors) {
                 case null:
                 case "backslashreplace":
                 case "xmlcharrefreplace":
                 case "strict": e = setFallback(e, new ExceptionFallback(e is UTF8Encoding)); break;
                 case "replace": e = setFallback(e, ReplacementFallback); break;
-                case "ignore": e = setFallback(e, new PythonDecoderFallback(encoding, buffer, start)); break;
-                case "surrogateescape": e =  new PythonSurrogateEscapeEncoding(e, encoding); break;
-                case "surrogatepass": e =  new PythonSurrogatePassEncoding(e, encoding); break;
-                default:
-                    e = setFallback(e, new PythonDecoderFallback(encoding,
-                        buffer, start,
-                        () => LightExceptions.CheckAndThrow(PythonOps.LookupEncodingError(context, errors))));
-                    break;
+                case "ignore": e = setFallback(e, new DecoderReplacementFallback(string.Empty)); break;
+                case "surrogateescape": e = pe = new PythonSurrogateEscapeEncoding(e, encoding); break;
+                case "surrogatepass": e = pe = new PythonSurrogatePassEncoding(e, encoding); break;
+                default: e = pe = new PythonErrorHandlerEncoding(context, e, encoding, errors); break;
             }
 
             string decoded = string.Empty;
             try {
-                unsafe {
-                    fixed (byte* ptr = span.Slice(start)) {
-                        if (ptr != null) {
-                            if (e is UnicodeEscapeEncoding ue) {
-                                // This overload is not virtual, but the base implementation is inefficient for this encoding
-                                decoded = ue.GetString(ptr, length);
-                            } else {
-                                decoded = e.GetString(ptr, length);
-                            }
-                        }
+                if (pe != null) {
+                    decoded = pe.GetString(buffer, start, length);
+                } else {
+                    if (e is UnicodeEscapeEncoding ue) {
+                        // This overload is not virtual, but the base implementation is inefficient for this encoding
+                        decoded = ue.GetString(span.Slice(start, length));
+                    } else {
+                        decoded = e.GetString(span.Slice(start, length));
                     }
                 }
             } catch (DecoderFallbackException ex) {
                 // augmenting the caught exception instead of creating UnicodeDecodeError to preserve the stack trace
-                ex.Data["encoding"] = encoding;
-                ex.Data["object"] = Bytes.Make(span.Slice(start, length).ToArray());
+                if (!ex.Data.Contains("encoding")) ex.Data["encoding"] = encoding;
+                if (!ex.Data.Contains("object")) ex.Data["object"] = Bytes.Make(span.Slice(start, length).ToArray()); ;
                 throw;
             }
 
@@ -2219,6 +2217,8 @@ private string EscapeEncode(string s, int index, int count) {
                     ReprEncode(s, index, count, isUniEscape: true);
             }
 
+            public override string EncodingName => _raw ? "rawunicodeescape" : "unicodeescape";
+
             public override int GetByteCount(string s)
                 => EscapeEncode(s, 0, s.Length).Length;
 
@@ -2240,10 +2240,12 @@ public override int GetBytes(char[] chars, int charIndex, int charCount, byte[]
             public override string GetString(byte[] bytes, int index, int count)
                 => LiteralParser.ParseString(bytes, index, count, _raw, GetErrorHandler());
 
-            public new unsafe string GetString(byte* bytes, int byteCount) {
-                var data = new ReadOnlySpan<byte>(bytes, byteCount);
-                return LiteralParser.ParseString(data, _raw, GetErrorHandler());
-            }
+#if NETCOREAPP
+            public new string GetString(ReadOnlySpan<byte> bytes)
+#else
+            public string GetString(ReadOnlySpan<byte> bytes)
+#endif
+                => LiteralParser.ParseString(bytes, _raw, GetErrorHandler());
 
             public override unsafe int GetCharCount(byte* bytes, int count)
                 => LiteralParser.ParseString(new ReadOnlySpan<byte>(bytes, count), _raw, GetErrorHandler()).Length;
@@ -2303,127 +2305,6 @@ public override int GetChars(byte[] bytes, int byteIndex, int byteCount, char[]
 
         #region  Unicode Encode/Decode Fallback Support
 
-        /// When encoding or decoding strings if an error occurs CPython supports several different
-        /// behaviors, in addition it supports user-extensible behaviors as well.  For the default
-        /// behavior we're ok - both of us support throwing and replacing.  For custom behaviors
-        /// we define a single fallback for decoding and encoding that calls the python function to do
-        /// the replacement.
-        ///
-        /// When we do the replacement we call the provided handler w/ a UnicodeEncodeError or UnicodeDecodeError
-        /// object which contains:
-        ///         encoding    (string, the encoding the user requested)
-        ///         object      (the original string or bytes being encoded/decoded)
-        ///         start       (the start of the invalid sequence)
-        ///         end         (the exclusive end of the invalid sequence)
-        ///         reason      (the error message, e.g. 'unexpected byte code', not sure of others)
-        ///
-        /// The decoder returns a tuple of (str, int) where str is the replacement string
-        /// and int is an index where encoding/decoding should continue.
-        /// TODO: returned int is currently ignored, assumed to be equal to end (i.e. the index is not adjusted).
-
-        private class PythonDecoderFallbackBuffer : DecoderFallbackBuffer {
-            private readonly object? _function;
-            private readonly string _encoding;
-            private readonly IPythonBuffer _data;
-            private readonly int _offset;
-            private Bytes? _byteData;
-            private string? _buffer;
-            private int _bufferIndex;
-
-            public PythonDecoderFallbackBuffer(string encoding, IPythonBuffer data, int offset, object? callable) {
-                _encoding = encoding;
-                _data = data;
-                _offset = offset;
-                _function = callable;
-            }
-
-            public override int Remaining {
-                get {
-                    if (_buffer == null) return 0;
-                    return _buffer.Length - _bufferIndex;
-                }
-            }
-
-            public override char GetNextChar() {
-                if (_buffer == null || _bufferIndex >= _buffer.Length) return Char.MinValue;
-
-                return _buffer[_bufferIndex++];
-            }
-
-            public override bool MovePrevious() {
-                if (_bufferIndex > 0) {
-                    _bufferIndex--;
-                    return true;
-                }
-                return false;
-            }
-
-            public override void Reset() {
-                _buffer = null;
-                _bufferIndex = 0;
-                base.Reset();
-            }
-
-            public override bool Fallback(byte[] bytesUnknown, int index) {
-                if (_function != null) {
-                    // create the exception object to hand to the user-function...
-                    _byteData ??= Bytes.Make(_data.AsReadOnlySpan().Slice(_offset).ToArray());
-                    var exObj = PythonExceptions.CreatePythonThrowable(PythonExceptions.UnicodeDecodeError, _encoding, _byteData, index, index + bytesUnknown.Length, "unexpected code byte");
-
-                    // call the user function...
-                    object? res = PythonCalls.Call(_function, exObj);
-
-                    string replacement = CheckReplacementTuple(res, "decoding", index + bytesUnknown.Length);
-
-                    // finally process the user's request.
-                    _buffer = replacement;
-                    _bufferIndex = 0;
-                    return true;
-                }
-
-                return false;
-            }
-
-        }
-
-        private class PythonDecoderFallback : DecoderFallback {
-            private readonly string encoding;
-            private readonly IPythonBuffer data;
-            private readonly int offset;
-            private readonly Func<object>? lookup;
-            private object? function;
-
-            public PythonDecoderFallback(string encoding, IPythonBuffer data, int offset, Func<object>? lookup = null) {
-                this.encoding = encoding;
-                this.data = data;
-                this.offset = offset;
-                this.lookup = lookup;
-            }
-
-            public override DecoderFallbackBuffer CreateFallbackBuffer() {
-                if (function == null && lookup != null) {
-                    function = lookup.Invoke();
-                }
-                return new PythonDecoderFallbackBuffer(encoding, data, offset, function);
-            }
-
-            public override int MaxCharCount {
-                get { throw new NotImplementedException(); }
-            }
-        }
-
-        private static string CheckReplacementTuple(object? res, string encodeOrDecode, int cursorPos) {
-            // verify the result is sane...
-            if (res is PythonTuple tres && tres.__len__() == 2
-                    && Converter.TryConvertToString(tres[0], out string? replacement)
-                    && Converter.TryConvertToInt32(tres[1], out int newPos)) {
-                if (newPos != cursorPos) throw new NotImplementedException($"Moving {encodeOrDecode} cursor not implemented yet");
-                return replacement;
-            }
-
-            throw PythonOps.TypeError("{1} error handler must return tuple containing (str, int), got {0}", PythonOps.GetPythonTypeName(res), encodeOrDecode);
-        }
-
         private class BackslashEncoderReplaceFallback : EncoderFallback {
             private class BackslashReplaceFallbackBuffer : EncoderFallbackBuffer {
                 private List<char> _buffer = new List<char>();
@@ -2931,6 +2812,6 @@ internal static void IdentifyUtfEncoding(string encodingName, out int charWidth,
             }
         }
 
-        #endregion
+#endregion
     }
 }