Merge pull request #102 from MikhailArkhipov/enc

Remove exceptions on ASCII read

Author: Mikhail Arkhipov

src/Analysis/Engine/Impl/Parsing/Parser.cs

@@ -133,11 +133,9 @@ public static Parser CreateParser(Stream stream, PythonLanguageVersion version)
         /// </summary>
         public static Parser CreateParser(Stream stream, PythonLanguageVersion version, ParserOptions parserOptions = null) {
             var options = parserOptions ?? ParserOptions.Default;
-
-            var defaultEncoding = version.Is2x() ? PythonAsciiEncoding.Instance : Encoding.UTF8;
+            var defaultEncoding = version.Is2x() ? Encoding.ASCII : Encoding.UTF8;
 
             var reader = GetStreamReaderWithEncoding(stream, defaultEncoding, options.ErrorSink);
-
             return CreateParser(reader, version, options);
         }
 
@@ -5325,9 +5323,7 @@ public AsciiEncodingInfoWrapper()
                 : base((EncodingInfo)null) {
             }
 
-            public override Encoding GetEncoding() {
-                return PythonAsciiEncoding.Instance;
-            }
+            public override Encoding GetEncoding() => Encoding.ASCII;
         }
 
         class EncodingWrapper : Encoding {
@@ -5478,23 +5474,7 @@ private static string ReadOneLine(List<byte> previewedBytes, ref int curIndex, S
             return noNewlineRes;
         }
 
-        /// <summary>
-        /// Returns an Encoding object which raises a BadSourceException when invalid characters are encountered.
-        /// </summary>
-        public static Encoding DefaultEncoding {
-            get {
-                return PythonAsciiEncoding.SourceEncoding;
-            }
-        }
-
-        /// <summary>
-        /// Returns an Encoding object which will not provide any fallback for invalid characters.
-        /// </summary>
-        public static Encoding DefaultEncodingNoFallback {
-            get {
-                return PythonAsciiEncoding.SourceEncodingNoFallback;
-            }
-        }
+        public static Encoding DefaultEncoding => Encoding.ASCII;
 
         #endregion
 

src/Analysis/Engine/Impl/Parsing/PythonAsciiEncoding.cs

@@ -22,144 +22,6 @@
 using Microsoft.PythonTools.Analysis.Infrastructure;
 
 namespace Microsoft.PythonTools.Parsing {
-    /// <summary>
-    /// Simple implementation of ASCII encoding/decoding.  The default instance (PythonAsciiEncoding.Instance) is
-    /// setup to always convert even values outside of the ASCII range.  The EncoderFallback/DecoderFallbacks can
-    /// be replaced with versions that will throw exceptions instead though.
-    /// </summary>
-    [Serializable]
-    sealed class PythonAsciiEncoding : Encoding {
-        internal static readonly Encoding Instance = MakeNonThrowing();
-        internal static readonly Encoding SourceEncoding = MakeSourceEncoding();
-        internal static readonly Encoding SourceEncodingNoFallback = MakeSourceEncodingNoFallback();
-
-        internal PythonAsciiEncoding()
-            : base() {
-        }
-
-        internal static Encoding MakeNonThrowing() {
-            // we need to Clone the new instance here so that the base class marks us as non-readonly
-            Encoding enc = (Encoding)new PythonAsciiEncoding().Clone();
-            enc.DecoderFallback = new NonStrictDecoderFallback();
-            enc.EncoderFallback = new NonStrictEncoderFallback();
-            return enc;
-        }
-
-        private static Encoding MakeSourceEncoding() {
-            // we need to Clone the new instance here so that the base class marks us as non-readonly
-            Encoding enc = (Encoding)new PythonAsciiEncoding().Clone();
-            enc.DecoderFallback = new SourceNonStrictDecoderFallback();
-            return enc;
-        }
-
-        private static Encoding MakeSourceEncodingNoFallback() {
-            // we need to Clone the new instance here so that the base class marks us as non-readonly
-            Encoding enc = (Encoding)new PythonAsciiEncoding().Clone();
-            enc.DecoderFallback = new SourceNonStrictDecoderFallbackNoFallback();
-            return enc;
-        }
-
-        public override int GetByteCount(char[] chars, int index, int count) {
-            int byteCount = 0;
-            int charEnd = index + count;
-            while (index < charEnd) {
-                char c = chars[index];
-                if (c > 0x7f) {
-                    EncoderFallbackBuffer efb = EncoderFallback.CreateFallbackBuffer();
-                    if (efb.Fallback(c, index)) {
-                        byteCount += efb.Remaining;
-                    }
-                } else {
-                    byteCount++;
-                }
-                index++;
-            }
-            return byteCount;
-        }
-
-        public override int GetBytes(char[] chars, int charIndex, int charCount, byte[] bytes, int byteIndex) {
-            int charEnd = charIndex + charCount;
-            int outputBytes = 0;
-            while (charIndex < charEnd) {
-                char c = chars[charIndex];
-                if (c > 0x7f) {
-                    EncoderFallbackBuffer efb = EncoderFallback.CreateFallbackBuffer();
-                    if (efb.Fallback(c, charIndex)) {
-                        while (efb.Remaining != 0) {
-                            bytes[byteIndex++] = (byte)efb.GetNextChar();
-                            outputBytes++;
-                        }
-                    }
-                } else {
-                    bytes[byteIndex++] = (byte)c;
-                    outputBytes++;
-                }
-                charIndex++;
-            }
-            return outputBytes;
-        }
-
-        public override int GetCharCount(byte[] bytes, int index, int count) {
-            int byteEnd = index + count;
-            int outputChars = 0;
-            while (index < byteEnd) {
-                byte b = bytes[index];
-                if (b > 0x7f) {
-                    DecoderFallbackBuffer dfb = DecoderFallback.CreateFallbackBuffer();
-                    if (dfb.Fallback(new byte[] { b }, index)) {
-                        outputChars += dfb.Remaining;
-                    }
-                } else {
-                    outputChars++;
-                }
-                index++;
-            }
-            return outputChars;
-        }
-
-        public override int GetChars(byte[] bytes, int byteIndex, int byteCount, char[] chars, int charIndex) {
-            int byteEnd = byteIndex + byteCount;
-            int outputChars = 0;
-            while (byteIndex < byteEnd) {
-                byte b = bytes[byteIndex];
-                if (b > 0x7f) {
-                    DecoderFallbackBuffer dfb = DecoderFallback.CreateFallbackBuffer();
-                    if (dfb.Fallback(new byte[] { b }, byteIndex)) {
-                        while (dfb.Remaining != 0) {
-                            chars[charIndex++] = dfb.GetNextChar();
-                            outputChars++;
-                        }
-                    }
-                } else {
-                    chars[charIndex++] = (char)b;
-                    outputChars++;
-                }
-                byteIndex++;
-            }
-            return outputChars;
-        }
-
-        public override int GetMaxByteCount(int charCount) {
-            return charCount * 4;
-        }
-
-        public override int GetMaxCharCount(int byteCount) {
-            return byteCount;
-        }
-
-        public override string WebName {
-            get {
-                return "ascii";
-            }
-        }
-
-        public override string EncodingName {
-            get {
-                return "ascii";
-            }
-        }
-    }
-
     class NonStrictEncoderFallback : EncoderFallback {
         public override EncoderFallbackBuffer CreateFallbackBuffer() {
             return new NonStrictEncoderFallbackBuffer();

src/Analysis/Engine/Impl/Parsing/Tokenizer.cs

@@ -2631,39 +2631,8 @@ private void RefillBuffer() {
             }
 
             // make the buffer full:
-            try {
-                int count = _reader.Read(_buffer, _end, _buffer.Length - _end);
-                _end += count;
-            } catch (BadSourceException bse) {
-                StreamReader streamReader = _reader as StreamReader;
-                if (streamReader != null && streamReader.CurrentEncoding != PythonAsciiEncoding.SourceEncoding) {
-                    _errors.Add(
-                        "(unicode error) '{0}' codec can't decode byte 0x{1:x} in position {2}".FormatUI(
-                            Parser.NormalizeEncodingName(streamReader.CurrentEncoding.WebName),
-                            bse.BadByte,
-                            bse.Index + CurrentIndex
-                        ),
-                        null,
-                        CurrentIndex + bse.Index,
-                        CurrentIndex + bse.Index + 1,
-                        ErrorCodes.SyntaxError,
-                        Severity.FatalError
-                    );
-                } else {
-                    _errors.Add(
-                        "Non-ASCII character '\\x{0:x}' at position {1}, but no encoding declared; see http://www.python.org/peps/pep-0263.html for details".FormatUI(
-                            bse.BadByte,
-                            bse.Index + CurrentIndex
-                        ),
-                        null,
-                        CurrentIndex + bse.Index,
-                        CurrentIndex + bse.Index + 1,
-                        ErrorCodes.SyntaxError,
-                        Severity.FatalError
-                    );
-                }
-                throw;
-            }
+            int count = _reader.Read(_buffer, _end, _buffer.Length - _end);
+            _end += count;
 
             ClearInvalidChars();
         }