move default encoding to constants file, use it as fallback for add(), fix #67

also add ability to specify encoding of values added to the constants using add_with_encoding()

Author: derek73

nameparser/config/__init__.py

@@ -43,6 +43,8 @@
 from nameparser.config.titles import FIRST_NAME_TITLES
 from nameparser.config.regexes import REGEXES
 
+DEFAULT_ENCODING = 'UTF-8'
+
 class SetManager(collections.Set):
     '''
     Easily add and remove config variables per module or instance. Subclass of
@@ -84,15 +86,23 @@ def __next__(self):
             self.count = c + 1
             return getattr(self, self.elements[c]) or next(self)
 
+    def add_with_encoding(self, s, encoding=None):
+        """
+        Add the lower case and no-period version of the string to the set. Pass an
+        explicit `encoding` parameter to specify the encoding of binary strings that
+        are not DEFAULT_ENCODING (UTF-8).
+        """
+        encoding = encoding or sys.stdin.encoding or DEFAULT_ENCODING
+        if type(s) == binary_type:
+            s = s.decode(encoding)
+        self.elements.add(lc(s))
+
     def add(self, *strings):
         """
         Add the lower case and no-period version of the string arguments to the set.
-        Returns ``self`` for chaining.
+        Can pass a list of strings. Returns ``self`` for chaining.
         """
-        for s in strings:
-            if type(s) == binary_type:
-                s = s.decode(sys.stdin.encoding)
-            self.elements.add(lc(s))
+        [self.add_with_encoding(s) for s in strings]
         return self
 
     def remove(self, *strings):
@@ -193,7 +203,7 @@ def suffixes_prefixes_titles(self):
         if not self._pst:
             self._pst = self.prefixes | self.suffix_acronyms | self.suffix_not_acronyms | self.titles
         return self._pst
-    
+
     def __repr__(self):
         return "<Constants() instance>"
 

nameparser/parser.py

@@ -11,6 +11,7 @@
 from nameparser.util import log
 from nameparser.config import CONSTANTS
 from nameparser.config import Constants
+from nameparser.config import DEFAULT_ENCODING
 
 ENCODING = 'utf-8'
 
@@ -69,13 +70,13 @@ class HumanName(object):
     unparsable = True
     _full_name = ''
 
-    def __init__(self, full_name="", constants=CONSTANTS, encoding=ENCODING, 
+    def __init__(self, full_name="", constants=CONSTANTS, encoding=DEFAULT_ENCODING,
                 string_format=None):
         self.C = constants
         if type(self.C) is not type(CONSTANTS):
             self.C = Constants()
 
-        self.ENCODING = encoding
+        self.encoding = encoding
         self.string_format = string_format or self.C.string_format
         # full_name setter triggers the parse
         self.full_name = full_name
@@ -127,15 +128,15 @@ def __unicode__(self):
         if self.string_format:
             # string_format = "{title} {first} {middle} {last} {suffix} ({nickname})"
             _s = self.string_format.format(**self.as_dict())
-            # remove trailing punctation from missing nicknames
+            # remove trailing punctuation from missing nicknames
             _s = _s.replace(str(self.C.empty_attribute_default),'').replace(" ()","").replace(" ''","").replace(' ""',"")
             return self.collapse_whitespace(_s).strip(', ')
         return " ".join(self)
 
     def __str__(self):
         if sys.version >= '3':
             return self.__unicode__()
-        return self.__unicode__().encode(self.ENCODING)
+        return self.__unicode__().encode(self.encoding)
 
     def __repr__(self):
         if self.unparsable:
@@ -152,7 +153,7 @@ def __repr__(self):
             }
         if sys.version >= '3':
             return _string
-        return _string.encode(self.ENCODING)
+        return _string.encode(self.encoding)
 
     def as_dict(self, include_empty=True):
         """
@@ -355,7 +356,7 @@ def full_name(self, value):
         self.original = value
         self._full_name = value
         if isinstance(value, binary_type):
-            self._full_name = value.decode(self.ENCODING)
+            self._full_name = value.decode(self.encoding)
         self.parse_full_name()
 
     def collapse_whitespace(self, string):

tests.py

@@ -189,6 +189,7 @@ def test_blank_name(self):
         self.m(hn.first, "", hn)
         self.m(hn.last, "", hn)
 
+
 class FirstNameHandlingTests(HumanNameTestBase):
     def test_first_name(self):
         hn = HumanName("Andrew")
@@ -1058,7 +1059,6 @@ def test119(self):
         self.m(hn.last, "Almighty", hn)
 
 
-
 class HumanNameConjunctionTestCase(HumanNameTestBase):
     # Last name with conjunction
     def test_last_name_with_conjunction(self):
@@ -1244,6 +1244,7 @@ def test_conjunction_in_an_address_with_a_first_name_title(self):
         # if you want to be technical, Queen is in FIRST_NAME_TITLES
         self.m(hn.first, "Elizabeth", hn)
 
+
 class ConstantsCustomization(HumanNameTestBase):
 
     def test_add_title(self):
@@ -1335,6 +1336,12 @@ def test_none_empty_attribute_string_formatting(self):
         hn.C.empty_attribute_default = None
         self.assertEqual('', str(hn), hn)
 
+    def test_add_constant_with_explicit_encoding(self):
+        c = Constants()
+        c.titles.add_with_encoding(b'b\351ck', encoding='latin_1')
+        self.assertIn('béck', c.titles)
+
+
 class HumanNameNicknameTestCase(HumanNameTestBase):
     # https://code.google.com/p/python-nameparser/issues/detail?id=33
     def test_nickname_in_parenthesis(self):