Submodules structure and init file refactoring. In particular preconstructed symbols are now each in a dedicated package, and there is not anymore a dynamic init file - that was too much against python best practices.

Author: Sylvain MARIE

code_generation/mini_lambda_methods_generation.py

@@ -1,3 +1,4 @@
+from __future__ import print_function
 from inspect import isclass
 from sys import getsizeof
 try:
@@ -139,13 +140,15 @@ def generate_code():
         print(o)
 
     # generate
-    generate_from_template('mini_lambda_template.mako', 'generated.py',
-                           dict(to_override=to_override, to_override_with_exception=to_override_with_exception))
-    generate_from_template('mini_lambda_template_2.mako', 'generated2.py',
+    generate_from_template('tpl_magic_methods.mako', 'generated_magic.py',
                            dict(to_override=to_override, to_override_with_exception=to_override_with_exception))
+    generate_from_template('tpl_magic_methods_replacements.mako', 'generated_magic_replacements.py',
+                           dict(to_override=to_override, to_override_with_exception=to_override_with_exception,
+                                generate_all=True))
 
     # (2) to-do list for the second template
-    import_lines, to_create = define_goodies()
+    # --(a) builtins
+    import_lines, to_create = define_goodies_builtins()
 
     # check outside of the template that it works:
     for o in import_lines:
@@ -154,8 +157,23 @@ def generate_code():
         print(o)
 
     # generate
-    generate_from_template('goodies_template.mako', 'goodies_generated.py',
-                           dict(import_lines=import_lines, to_create=to_create))
+    generate_from_template('tpl_preconverted_symbols.mako', 'symbols/builtins.py',
+                           dict(import_lines=import_lines, to_create=to_create, generate_all=True))
+
+    # --(b) math
+    for pkg in [math, decimal]:
+        import_lines, to_create = define_goodies(packages=[pkg])
+
+        # check outside of the template that it works:
+        for o in import_lines:
+            print(o)
+        for o in to_create:
+            print(o)
+
+        # generate but without the '__all__' symbol since packages may not contain the same symbols across versions of
+        # python
+        generate_from_template('tpl_preconverted_symbols.mako', 'symbols/%s_.py' % pkg.__name__,
+                               dict(import_lines=import_lines, to_create=to_create, generate_all=False))
 
 
 def generate_from_template(src_file, dst_file, template_vars):
@@ -399,25 +417,36 @@ def define_what_needs_to_be_written():
     return to_override_2, to_override_with_exception_2
 
 
-def define_goodies():
-    # type: (...) -> Tuple[List[str], List[Goodie]]
-    """
-
-    :return:
-    """
-    packages = [math, decimal]
+def define_goodies_builtins():
+    import_list = list()
+    goodies_list = list()
     built_in_functions = ['abs', 'all', 'any', 'bin', 'bool', 'bytearray', 'bytes', 'callable', 'chr',
                           'classmethod', 'compile', 'complex', 'delattr', 'dict', 'dir', 'divmod', 'enumerate',
                           'eval', 'filter', 'float', 'format', 'frozenset', 'getattr', 'globals', 'hasattr',
                           'hash', 'hex', 'id', 'input', 'int', 'isinstance', 'issubclass', 'iter', 'len',
                           'list', 'locals', 'map', 'max', 'memoryview', 'min', 'next', 'object', 'oct', 'open', 'ord',
                           'pow', 'print', 'property', 'range', 'repr', 'reversed', 'round', 'set', 'setattr', 'slice',
                           'sorted', 'staticmethod', 'str', 'sum', 'super', 'tuple', 'type', 'vars', 'zip']
-    # these do not work in python 2
-    protected_functions = ['ascii', 'help', 'exec']
+    # these do not work in python 2: the statement is reserved.
+    # protected_functions = ['ascii', 'help', 'exec']
+
+    for function_name in built_in_functions:
+        if isclass(eval(function_name)):
+            goodies_list.append(Goodie(item_name=function_name.capitalize(), class_name=function_name))
+        else:
+            goodies_list.append(Goodie(item_name=function_name.capitalize(), function_name=function_name))
+
+    return import_list, goodies_list
+
+
+def define_goodies(packages):
+    # type: (...) -> Tuple[List[str], List[Goodie]]
+    """
+
+    :return:
+    """
     import_list = list()
     goodies_list = list()
-
     for package in packages:
         # import pprint
         # pprint(getmembers(math, callable))
@@ -442,9 +471,6 @@ def define_goodies():
         # we do not import anymore at the top but in each one so as to put additional try/catch around them
         # import_list.append(import_string[0:-1])
 
-    for function_name in built_in_functions:
-        goodies_list.append(Goodie(item_name=function_name.capitalize(), function_name=function_name))
-
     return import_list, goodies_list
 
 

…e_generation/mini_lambda_template_2.mako …tion/tpl_magic_methods_replacements.makocode_generation/mini_lambda_template_2.mako renamed to code_generation/tpl_magic_methods_replacements.mako code_generation/mini_lambda_template_2.mako renamed to code_generation/tpl_magic_methods_replacements.mako

@@ -4,6 +4,16 @@
 from mini_lambda.main import _LambdaExpression
 from sys import getsizeof
 
+% if generate_all:
+__all__ = [
+    % for o in to_override_with_exception:
+        % if o.module_method_name != 'Format':
+    '${o.module_method_name}',
+        % endif
+    % endfor
+]
+% endif
+
 
 # ******* All replacement methods for the magic methods throwing exceptions ********
 % for o in to_override_with_exception:

code_generation/goodies_template.mako …generation/tpl_preconverted_symbols.makocode_generation/goodies_template.mako renamed to code_generation/tpl_preconverted_symbols.mako code_generation/goodies_template.mako renamed to code_generation/tpl_preconverted_symbols.mako

@@ -9,6 +9,14 @@ from mini_lambda.main import C, make_lambda_friendly_class, make_lambda_friendly
 ${o}
 % endfor
 
+% if generate_all:
+__all__ = [
+    % for o in to_create:
+    '${o.item_name}',
+    % endfor
+]
+% endif
+
 % for o in to_create:
     % if o.import_line is None or len(o.import_line) == 0:
         % if o.constant_name:

mini_lambda/__init__.py

@@ -1,34 +1,29 @@
-from mini_lambda.utils_init import __remove_all_external_symbols, __get_all_submodules_symbols
+from mini_lambda.base import FunctionDefinitionError, evaluate, get_repr
 
-__PACKAGE_NAME = 'mini_lambda'
-__SUBMODULES_TO_EXPORT = ['base', 'generated', 'generated2', 'goodies', 'goodies_generated', 'main']
-# TODO we could rather rely on a regexp mechanism
+# this one only exports one private class, no need
+# from mini_lambda.generated import *
 
-# (1) allow users to do
-#     import <package> as p and then p.<symbol>
-__all__ = __get_all_submodules_symbols(__PACKAGE_NAME, __SUBMODULES_TO_EXPORT)
-__all__ += ['numpy_', 'pandas_']
-# Note: this is one way to do it, but it would be simpler to check the names in globals() at the end of this file.
+from mini_lambda.main import _, L, F, C, Not, And, Or, Format, Get, In, Slice, InputVar, Constant, \
+    make_lambda_friendly, make_lambda_friendly_method, make_lambda_friendly_class
 
-# (2) allow users to do
-#     from <package> import <symbol>
-#
-# The following works, but unfortunately IDE like pycharm do not understand
-from mini_lambda.base import *
-from mini_lambda.generated import *
-from mini_lambda.goodies_generated import *
-from mini_lambda.main import *
-from mini_lambda.main import _
-from mini_lambda.generated2 import *
-from mini_lambda.goodies import *
+__all__ = [
+    # submodules
+    'base', 'generated_magic', 'generated_magic_replacements', 'main', 'symbols', 'vars',
+    # symbols
+    'FunctionDefinitionError', 'evaluate', 'get_repr',
+    '_', 'L', 'F', 'C', 'InputVar', 'Constant', 'make_lambda_friendly', 'make_lambda_friendly_method',
+    'make_lambda_friendly_class',
+    'Not', 'And', 'Or', 'Format', 'Get', 'In', 'Slice',
+]
 
-# remove all symbols that were imported above but do not belong in this package
-__remove_all_external_symbols(__PACKAGE_NAME, globals())
+# for these two ones we can, there is a `__all__` inside
+from mini_lambda.vars import *
+from mini_lambda.vars import __all__ as vall
 
-# Otherwise exhaustive list would be required, which is sad
-# ...
+from mini_lambda.generated_magic_replacements import *
+from mini_lambda.generated_magic_replacements import __all__ as gall
 
-# print(__all__)
-# print(globals().keys())
-# print('Done')
+from mini_lambda.symbols import *
+from mini_lambda.symbols import __all__ as sall
 
+__all__ = __all__ + vall + gall + sall

mini_lambda/main.py

@@ -9,8 +9,8 @@
 
 from mini_lambda.base import get_repr, _PRECEDENCE_BITWISE_AND, _PRECEDENCE_BITWISE_OR, _PRECEDENCE_BITWISE_XOR, \
     _PRECEDENCE_SUBSCRIPTION_SLICING_CALL_ATTRREF, evaluate, _PRECEDENCE_EXPONENTIATION, \
-    _PRECEDENCE_POS_NEG_BITWISE_NOT, _get_root_var
-from mini_lambda.generated import _LambdaExpressionGenerated, FunctionDefinitionError
+    _PRECEDENCE_POS_NEG_BITWISE_NOT, _get_root_var, FunctionDefinitionError
+from mini_lambda.generated_magic import _LambdaExpressionGenerated
 
 
 this_module = sys.modules[__name__]

mini_lambda/symbols/__init__.py

@@ -0,0 +1,12 @@
+from .builtins import *
+
+# no need to import them, they'll be imported on demand
+# from mini_lambda import preconverted_math
+# from mini_lambda import preconverted_decimal
+
+__all__ = [
+    'builtins', 'math_', 'decimal_'
+]
+
+from .builtins import __all__ as ball
+__all__ += ball

mini_lambda/tests/test_mini_lambda.py

@@ -613,7 +613,7 @@ def test_evaluator_int_convertible():
 
 def test_evaluator_maths():
     """ """
-    from mini_lambda import Floor, Ceil
+    from mini_lambda.symbols.math_ import Floor, Ceil
     from math import floor, ceil, trunc
 
     x = InputVar('x', float)
@@ -772,7 +772,8 @@ def test_constants_named():
 def test_generated_methods():
     """ Tests that equivalent methods generated by the package from various packages (currently, only math) work"""
 
-    from mini_lambda import x, _, Sin
+    from mini_lambda import x, _
+    from mini_lambda.symbols.math_ import Sin
     from math import sin
 
     sine = _(Sin(x))

mini_lambda/tests/test_readme.py

@@ -29,7 +29,8 @@ def test_doc_index_1():
 def test_doc_index_2():
     """ Tests that the second example in the documentation main page works """
 
-    from mini_lambda import s, x, _, Log  # this is a dynamic creation hence pycharm does not see it
+    from mini_lambda import s, x, _  # this is a dynamic creation hence pycharm does not see it
+    from mini_lambda.symbols.math_ import Log
 
     # various lambda functions
     is_lowercase = _(s.islower())
@@ -145,7 +146,9 @@ def test_doc_usage_syntax_2():
     """ Tests that the second example in doc/usage in the syntax section works """
     from mini_lambda import b, i, s, l, x
     from mini_lambda import Slice, Get, Not, In
-    from mini_lambda import Iter, Repr, Format, Len, Int, Any, Log, DDecimal
+    from mini_lambda import Iter, Repr, Format, Len, Int, Any
+    from mini_lambda.symbols.math_ import Log
+    from mini_lambda.symbols.decimal_ import DDecimal
     from math import log
     from decimal import Decimal
 
@@ -162,7 +165,7 @@ def test_doc_usage_syntax_2():
         expr = log(x)                         # fails
     expr = Log(x)                         # OK
     # constructing with the variable as arg
-    with pytest.raises(TypeError if version_info >= (3, 0) else FunctionDefinitionError):
+    with pytest.raises(TypeError):
         expr = Decimal(x)                     # fails
     expr = DDecimal(x)                    # OK
     # getting with the variable as the key
@@ -174,6 +177,8 @@ def test_doc_usage_syntax_2():
         expr = 'hello'[0:i]                   # fails
     expr = Get('hello', Slice(0, i))      # OK
     # representing: Repr/Str/Bytes/Sizeof/Hash
+    assert repr(l) == '<_LambdaExpression: l>'
+    l.repr_on = False
     with pytest.raises(FunctionDefinitionError):
         expr = repr(l)                        # fails
     expr = Repr(l)                        # OK
@@ -219,7 +224,8 @@ def test_doc_usage_syntax_2():
 
 def test_doc_usage_other_constants():
     """ Tests that the example in doc/usage in the others/constants section works  """
-    from mini_lambda import x, _, E, C
+    from mini_lambda import x, _, C
+    from mini_lambda.symbols.math_ import E
     from math import e
 
     assert str(_(x + e)) == 'x + 2.718281828459045'
@@ -309,7 +315,7 @@ def bar2(cls, times, num, den):
 def test_doc_usage_other_classes():
     """ Tests that the example in doc/usage in the others/classes section works """
     from mini_lambda import _, make_lambda_friendly_class
-    from mini_lambda.numpy_ import X
+    from mini_lambda.vars.numpy_ import X
     import numpy as np
     import pandas as pd
 
@@ -323,7 +329,7 @@ def test_doc_usage_other_classes():
 def test_doc_usage_all_at_once():
     """ Tests that the example in doc/usage in the others/anything section works """
     from mini_lambda import _, C
-    from mini_lambda.numpy_ import X
+    from mini_lambda.vars.numpy_ import X
     import numpy as np
     import pandas as pd
 
@@ -336,9 +342,11 @@ def test_doc_usage_all_at_once():
 def test_doc_usage_already_imported():
     """ Tests that the example in doc/usage in the others/preconverted section works """
 
-    from mini_lambda import DDecimal  # Decimal class
     from mini_lambda import Print  # print() function
-    from mini_lambda import Pi  # math.pi constant
+
+    from mini_lambda.symbols.decimal_ import DDecimal  # Decimal class
+    from mini_lambda.symbols.builtins import Print  # print() function
+    from mini_lambda.symbols.math_ import Pi  # math.pi constant
 
 
 def test_doc_optional():
@@ -347,9 +355,9 @@ def test_doc_optional():
     with pytest.raises(ImportError):
         from mini_lambda import X
 
-    from mini_lambda.numpy_ import X
+    from mini_lambda.vars.numpy_ import X
 
     with pytest.raises(ImportError):
         from mini_lambda import df
 
-    from mini_lambda.pandas_ import df
+    from mini_lambda.vars.pandas_ import df