revise normalize newlines to match php-textile and fix #47 and related issues.

Author: ikirudennis

tests/test_block.py

@@ -69,3 +69,25 @@ def test_blockcode_comment():
     t = textile.Textile()
     result = t.parse(input)
     assert result == expect
+
+def test_extended_pre_block_with_many_newlines():
+    """Extra newlines in an extended pre block should not get cut down to only
+    two."""
+    text = '''pre.. word
+
+another
+
+word
+
+
+yet anothe word'''
+    expect = '''<pre>word
+
+another
+
+word
+
+
+yet anothe word</pre>'''
+    result = textile.textile(text)
+    assert result == expect

tests/test_github_issues.py

@@ -137,3 +137,22 @@ def test_github_issue_45():
     result = textile.textile(text)
     expect = '\t<p><a href="https://myabstractwiki.ru/index.php/%D0%97%D0%B0%D0%B3%D0%BB%D0%B0%D0%B2%D0%BD%D0%B0%D1%8F_%D1%81%D1%82%D1%80%D0%B0%D0%BD%D0%B8%D1%86%D0%B0">test</a></p>'
     assert result == expect
+
+def test_github_issue_47():
+    """Incorrect wrap pre-formatted value"""
+    text = '''pre.. word
+
+another
+
+word
+
+yet anothe word'''
+    result = textile.textile(text)
+    expect = '''<pre>word
+
+another
+
+word
+
+yet anothe word</pre>'''
+    assert result == expect

textile/core.py

@@ -433,12 +433,12 @@ def block(self, text):
                     r'(?::(?P<cite>\S+))? (?P<content>.*)$'.format(tre,
                         align_re_s, cls_re_s))
             match = re.search(pattern, line, flags=re.S | re.U)
+            if out:
+                last_item_is_a_shelf = out[-1] in self.shelf
             # tag specified on this line.
             if match:
                 # if we had a previous extended tag but not this time, close up
                 # the tag
-                if out:
-                    last_item_is_a_shelf = out[-1] in self.shelf
                 if ext and match.group('tag') and last_item_is_a_shelf:
                     content = out.pop()
                     content = generate_tag(block.inner_tag, content,
@@ -467,9 +467,14 @@ def block(self, text):
                 if ext and out:
                     line = '{0}\n\n{1}'.format(out.pop(), line)
                 whitespace = ' \t\n\r\f\v'
-                if ext or not line[0] in whitespace:
+                try:
+                    line_first_char_blank = line[0] in whitespace
+                except IndexError:
+                    line_first_char_blank = True
+                if ext or not line_first_char_blank:
                     block = Block(self, tag, atts, ext, cite, line)
-                    if block.tag == 'p' and not has_raw_text(block.content):
+                    if (block.tag == 'p' and not has_raw_text(block.content)
+                        or last_item_is_a_shelf):
                         line = block.content
                     else:
                         line = generate_tag(block.outer_tag, block.content,

textile/utils.py

@@ -110,9 +110,8 @@ def list_type(list_string):
 
 def normalize_newlines(string):
     out = string.strip()
-    out = re.sub(r'\r\n', '\n', out)
-    out = re.sub(r'\n{3,}', '\n\n', out)
-    out = re.sub(r'\n\s*\n', '\n\n', out)
+    out = re.sub(r'\r\n?', '\n', out)
+    out = re.sub(r'^[ \t]*\n', '\n', out, flags=re.M)
     out = re.sub(r'"$', '" ', out)
     return out