Skip to content

CodeSmith modules

__init__.py

Entry point for the codesmith package, defines the IPython magic functions.

codesmith(line='', cell=None, local_ns=None)

Defines line and cell magic functions %codesmith and %%codesmith (respectively). local_ns should be set by IPython

Source code in codesmith/__init__.py 
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
@register_line_cell_magic
@needs_local_scope
def codesmith(line='', cell=None, local_ns=None):
  """ Defines line and cell magic functions %codesmith and %%codesmith 
      (respectively). local_ns *should* be set by IPython """

  global_ns = get_ipython().user_ns
  if local_ns is None: local_ns = global_ns
  verbose = debug = False
  p = py.cell  # Default language is python (py), and entry point is py.cell
  line += '  ' # Need spaces at end for cell magic argument parsing

  # Parse the arguments to (%)%codesmith, each starts with '-'
  while line[0] == '-':
    first_space = line.index(' ')
    arg = line[1:first_space]
    line = line[first_space+1:]
    if arg == 'v': verbose = True
    elif arg == 'vd': verbose = debug = True
    else:
      if verbose: print(f"parser:>>{arg}<<")
      import builtins
      p = builtins.eval(arg, local_ns)
      if verbose: print(p)

  if verbose: print(f"line:>>{line}<<")
  if not cell: cell = line

  readout = p.read(cell, verbose=verbose)
  if verbose: print("output of read_", readout, type(readout))
  try:
    evalout = eval_(readout, global_ns, local_ns, debug)
    if verbose: print("output of eval_", evalout, type(evalout))
    if evalout: display(evalout)
  except Exception as e:
    display(e)
  return None

reload()

For active development: reloads this module and all submodules

Source code in codesmith/__init__.py 
11
12
13
14
15
16
def reload():
  """For active development: reloads this module and all submodules"""
  package_name = 'codesmith'
  import importlib
  for k,v in list(sys.modules.items()):
    if k.startswith(package_name): importlib.reload(v)

read.py

Provides the parsing functions for codesmith grammars

Grammar

Bases: dict

Stores an entire TSP Language grammar in a dict of Rule's

Source code in codesmith/read.py 
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
class Grammar(dict):
  """Stores an entire TSP Language grammar in a dict of Rule's"""

  def __getitem__(self, name):
    """Generate a new Rule if it doesn't exist"""
    if name not in self:
      self[name] = Rule(name)
    return super().__getitem__(name)
  __getattr__ = __getitem__ # dot values work like subscripts

  def __setitem__(self, name, value):
    super().__setitem__(name, value)
  __setattr__ = __setitem__

  def __call__(self, f):
    """ deprecated: for use as a decorator"""
    name = f.__name__
    syntax = [f.__annotations__.get(var, Empty()) 
                for var in f.__code__.co_varnames]

    if not just_pass(f): syntax.append(f)
    self[name] <<= tuple(syntax)
    return self.name

__call__(f)

deprecated: for use as a decorator

Source code in codesmith/read.py 
125
126
127
128
129
130
131
132
133
def __call__(self, f):
  """ deprecated: for use as a decorator"""
  name = f.__name__
  syntax = [f.__annotations__.get(var, Empty()) 
              for var in f.__code__.co_varnames]

  if not just_pass(f): syntax.append(f)
  self[name] <<= tuple(syntax)
  return self.name

__getitem__(name)

Generate a new Rule if it doesn't exist

Source code in codesmith/read.py 
114
115
116
117
118
def __getitem__(self, name):
  """Generate a new Rule if it doesn't exist"""
  if name not in self:
    self[name] = Rule(name)
  return super().__getitem__(name)

Out

Class to allow special outputs in grammars. Instatiated once as OUT below, and used like: OUT>>"{}{}" with a formatted string after a right-shift operator.

Source code in codesmith/read.py 
26
27
28
29
30
31
32
class Out:
  """Class to allow special outputs in grammars.
  Instatiated once as `OUT` below, and used like: `OUT>>"{}{}"` with a 
  formatted string after a right-shift operator.
  """
  def __rshift__(self, other):
    return lambda *toks: other.format(*toks)

Rule

Bases: Forward

A single rule in a (codesmith-style) pyparsing grammar

Source code in codesmith/read.py 
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
class Rule(Forward):
  """A single rule in a (codesmith-style) pyparsing grammar"""

  def __init__(self, name):
    self.rule_name = name
    self.clauses = []
    Forward.__init__(self)
    self.set_name(self.rule_name)

  def __ilshift__(self, syntax):
    """The main way to define a rule uses the augmented left-shift operator <<="""
    semantics = lambda *toks: ' '.join(str(t) for t in toks)
    if isinstance(syntax, tuple):
      if isinstance(syntax[-1], (FunctionType, BuiltinFunctionType)):
        *syntax, semantics = syntax
    else: syntax = (syntax,)
    syntax = list(syntax)

    for i,s in enumerate(syntax):
      #if isinstance(s, Literal): s = s.match
      if isinstance(s,str):
        if is_reference_id(s): syntax[i] = Keyword(s)
        else:                  syntax[i] = Literal(s)

    #print("LR check", self, self == syntax[0], syntax)
    if syntax[0] == self: #left recursive
      if not self.expr: 
        raise ValueError("left resursion requires a base case (prior clauses)")
      syntax = self.expr + OneOrMore(Group(And(syntax[1:])))
      f = semantics
      semantics = lambda *toks: reduce(lambda first, second: f(first, *second), toks)
    else: syntax = And(syntax)

    syntax.set_parse_action(lambda ts: semantics(*ts))
    self.clauses.append(syntax)
    self << (Or(self.clauses) if len(self.clauses) > 1 else self.clauses[0])
    return self

  def parse(self, s):
    """ deprecated """
    result = self.parse_string(s, parse_all=True)[0]
    print("Original parse", result)
    toast = parse_reference(result)
    print("AST", dump_tree(toast, maxline=75))
    back = unparse_reference(toast)
    print(back)
    try: return exec(back)
    except Exception as e: return e

  def read(self, s, verbose=False):
    """Applies this rule to string s, returning an AST in the reference language"""
    result = self.parse_string(s, parse_all=True)[0]
    if verbose: print("Original parse", result)
    toast = parse_reference(result)
    if verbose: print("AST", dump_tree(toast, maxline=75))
    return toast

__ilshift__(syntax)

The main way to define a rule uses the augmented left-shift operator <<=

Source code in codesmith/read.py 
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
def __ilshift__(self, syntax):
  """The main way to define a rule uses the augmented left-shift operator <<="""
  semantics = lambda *toks: ' '.join(str(t) for t in toks)
  if isinstance(syntax, tuple):
    if isinstance(syntax[-1], (FunctionType, BuiltinFunctionType)):
      *syntax, semantics = syntax
  else: syntax = (syntax,)
  syntax = list(syntax)

  for i,s in enumerate(syntax):
    #if isinstance(s, Literal): s = s.match
    if isinstance(s,str):
      if is_reference_id(s): syntax[i] = Keyword(s)
      else:                  syntax[i] = Literal(s)

  #print("LR check", self, self == syntax[0], syntax)
  if syntax[0] == self: #left recursive
    if not self.expr: 
      raise ValueError("left resursion requires a base case (prior clauses)")
    syntax = self.expr + OneOrMore(Group(And(syntax[1:])))
    f = semantics
    semantics = lambda *toks: reduce(lambda first, second: f(first, *second), toks)
  else: syntax = And(syntax)

  syntax.set_parse_action(lambda ts: semantics(*ts))
  self.clauses.append(syntax)
  self << (Or(self.clauses) if len(self.clauses) > 1 else self.clauses[0])
  return self

parse(s)

deprecated

Source code in codesmith/read.py 
89
90
91
92
93
94
95
96
97
98
def parse(self, s):
  """ deprecated """
  result = self.parse_string(s, parse_all=True)[0]
  print("Original parse", result)
  toast = parse_reference(result)
  print("AST", dump_tree(toast, maxline=75))
  back = unparse_reference(toast)
  print(back)
  try: return exec(back)
  except Exception as e: return e

read(s, verbose=False)

Applies this rule to string s, returning an AST in the reference language

Source code in codesmith/read.py 
100
101
102
103
104
105
106
def read(self, s, verbose=False):
  """Applies this rule to string s, returning an AST in the reference language"""
  result = self.parse_string(s, parse_all=True)[0]
  if verbose: print("Original parse", result)
  toast = parse_reference(result)
  if verbose: print("AST", dump_tree(toast, maxline=75))
  return toast

BlockOf(p)

The codesmith version of pyparsings IndentedBlock

Source code in codesmith/read.py 
40
41
42
43
44
45
46
47
48
49
def BlockOf(p):
  """The codesmith version of pyparsings `IndentedBlock`"""
  blockp = IndentedBlock(p, grouped=False)
  @blockp.set_parse_action
  def a(s,loc, toks):
    while s[loc] in blockp.whiteChars: loc +=1
    white = '\n' + ' '*col(loc,s)
    #print("COLS", white, "d", toks)
    return white + white.join(str(t) for t in toks)
  return blockp

ListOf(p, delim=',', trailer=True, min=None)

The codesmith version of pyparsing's delimited_list

Source code in codesmith/read.py 
35
36
37
38
def ListOf(p, delim=',', trailer=True, min=None):
  """The codesmith version of pyparsing's `delimited_list`"""
  return delimited_list(p, delim=delim, combine=True, min=min,
                        allow_trailing_delim=trailer)

eval.py

Provides a specialized evaluation procedure for TSP languages

RemoveOmega

Bases: NodeTransformer

NodeTransformer to remove ω wrappers in an AST

Source code in codesmith/eval.py 
50
51
52
53
54
55
56
class RemoveOmega(NodeTransformer):
  """NodeTransformer to remove ω wrappers in an AST"""
  def visit_Call(self, node):
    self.generic_visit(node) #visit children
    if isinstance(node.func, Name) and node.func.id == 'ω': out = node.args[0]
    else: out = node
    return out

argvars(node)

Returns the argument list for a lambda expression

Source code in codesmith/eval.py 
74
75
76
77
78
def argvars(node):
  """ Returns the argument list for a lambda expression """
  node = ensurenode(node)
  if not isinstance(node, Lambda): return []
  return [arg.arg for arg in node.args.args]

do_eval(node, globals, locals, debug=False, shadowed=[])

Internal function for eval_

Source code in codesmith/eval.py 
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
def do_eval(node, globals, locals, debug=False, shadowed=[]):
  """Internal function for eval_"""
  #node = AddAst().visit(node)
  node = fix_missing_locations(node)
  if shadowed:
    globals = {k:globals[k] for k in globals if k not in shadowed}
    locals = {k:locals[k] for k in locals if k not in shadowed}
  source = unparse(node) #For debugging only, remove later
  if debug: print("do_eval on", source)
  if (isinstance(node, Module) and len(node.body) == 1 
        and isinstance(node.body[0], Expr)):
    node = Expression(body=node.body[0].value)
  if isinstance(node, expressions):
    if not isinstance(node, Expression): node = Expression(body=node)
    code = compile(node, source, "eval")
    evaled = eval(code, globals, locals)
    if debug: print("eval of", dump(node), "returned",  evaled, type(evaled))
    return evaled

  if not isinstance(node, statements): return
  if not isinstance(node, Module): node = Module(body=[node])
  code = compile(node, source, "exec")
  exec(code, globals, locals)

ensurenode(node)

Parses node into AST if necessary

Source code in codesmith/eval.py 
67
68
69
70
71
72
def ensurenode(node):
  """ Parses node into AST if necessary """
  if isinstance(node, AST): return node
  if hasattr(node, "__ast__"): return node.__ast__
  node = str(node)
  return parse(node, node, "eval").body

eval_(node, globals=None, locals=None, debug=False)

Returns the value for node if evaluable, or a string with as many parts of node eval'ed as possible otherwise

Source code in codesmith/eval.py 
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
def eval_(node, globals=None, locals=None, debug=False):
  """ Returns the value for node if evaluable, or 
      a string with as many parts of node eval'ed as possible otherwise """

  # set parsed, globals, and locals correctly
  node = ensurenode(node)
  from sys import _getframe
  caller = _getframe(1)
  globals = globals or caller.f_globals
  locals = locals or caller.f_locals

  # define _simplify based on these values for globals and locals
  def _simplify(node, shadowed=set()):
    """ Returns the value for parsed if evaluable, or 
        an AST with as many parts of parsed eval'ed as possible otherwise """

    if debug: print("Simplifying", dump(node))

    if isinstance(node, Lambda):
      shadowed |= set(argvars(node))
      node.body = ensurenode(_simplify(node.body, shadowed))
      return node

    # Do lambda applications manually, in case they return a lambda
    if isinstance(node, Call):
      if debug: print("In Call", unparse(node))
      func = _simplify(node.func)
      func = get_lambda(func)
      if func:
        args = argvars(func)
        body = func.body
        subs = dict(zip(args, node.args))
        if debug: print("Lambda sub", unparse(body), {arg : dump(subs[arg]) for arg in subs})
        node = sub(body, subs)
        if debug: print("Body with subs:", unparse(node))
        node = _simplify(node)
        if debug: print("After simplification:", unparse(node))
        return node

    try:
      evaled = do_eval(node, globals, locals, debug, shadowed)
      if unprintable(evaled): return node
      return evaled
    except Exception as e: #TODO: be more selective
      if debug: print("Error in do_eval of", unparse(node), e) 

    for field, value in iter_fields(node):
      if debug: print("Working on", field)#, ":", dump(value))
      if isinstance(value, list):
        repl = [ensurenode(_simplify(x)) #need ensurenode?
                  if isinstance(x, AST) else x 
                for x in value]
      elif not isinstance(value, AST): continue #skip constants
      else:
        repl = ensurenode(_simplify(value))

      setattr(node,field,repl)

    return node

  # call the internal function and return the value or a string
  out = _simplify(node)
  ret = unparse(out)
  if isinstance(node, Lambda):
    print(ret)
    lm = eval_(node, globals, locals, debug)
    lm.__ast__ = node
    return lm
  return ω(ret)

freevars(node)

Returns the free variables in node

Source code in codesmith/eval.py 
87
88
89
90
91
92
93
94
95
96
97
def freevars(node):
  """ Returns the free variables in node """
  node = ensurenode(node)
  if isinstance(node, Name): return {node.id}

  free = set()
  for child in iter_child_nodes(node):
    free |= freevars(child)
  # remove bound variables
  if isinstance(node, Lambda): free -= set(argvars(node))
  return free

get_lambda(node)

Abstraction to allow either Lambda ast nodes themselves or, say, functions that have an __ast__ attribute to act as lambdas

Source code in codesmith/eval.py 
 99
100
101
102
103
104
def get_lambda(node):
  """Abstraction to allow either Lambda ast nodes themselves or, say, 
  functions that have an `__ast__` attribute to act as lambdas"""
  if isinstance(node, Lambda): return node
  if hasattr(node, "__ast__"): return get_lambda(node.__ast__)
  return False

newvar(oldvar, usedvars)

Generates a name for oldvar that is not in usedvars

Source code in codesmith/eval.py 
80
81
82
83
84
85
def newvar(oldvar, usedvars):
  """ Generates a name for oldvar that is not in usedvars """
  var = oldvar
  while var in usedvars:
    var = var + oldvar #repeat the name to form a new name
  return var

sub(node, subs)

Makes the substitutions in subs to node without variable capture subs maps variables (as strings) to their AST substitutions returns the new AST node

Source code in codesmith/eval.py 
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
def sub(node, subs):
  """ Makes the substitutions in subs to node without variable capture 
        subs maps variables (as strings) to their AST substitutions
        returns the new AST node
  """ 
  node = ensurenode(node) #For debugging, remove? -no, in case of __ast__ node

  if isinstance(node, Name) and node.id in subs:
    return ensurenode(subs[node.id])  #ensurenode for debugging, remove?

  # Capture avoidance code for lambdas
  if isinstance(node, Lambda):
    args = argvars(node)
    # Shadow any variables bound inside the lambda
    for arg in args: subs.pop(arg, "")
    # Collect potential variable capture cases
    frees = {var for val in subs.values() for var in freevars(val)}
    # captures will hold the actual name clashes
    captures = {arg for arg in args if arg in frees}
    if captures:
      #print("CLASH! with", captures)
      varsubs = dict()
      frees |= set(args) | freevars(node) # more vars to avoid in renaming
      for oldvar in captures:
        varsubs[oldvar] = newvar(oldvar, frees)
        frees.add(varsubs[oldvar]) # now avoid the new name too!
      # Replace in the lambda signature:
      for arg in node.args.args: #One level up from argvars, to make changes
        if arg.arg in varsubs: arg.arg = varsubs[arg.arg]
      # Replace in the body:
      node.body = sub(node.body, varsubs)      

  # Recurse on children of node
  for field, value in iter_fields(node):
    if isinstance(value, list):
      setattr(node,field,[ensurenode(sub(val, subs)) for val in value])
    elif isinstance(value, AST):
      setattr(node,field,ensurenode(sub(value, subs)))

  return node

unparse(node, remove_omega=False)

Attempts to convert node into python source code, or at least dump the AST; otherwise, returns node

Source code in codesmith/eval.py 
58
59
60
61
62
63
64
65
def unparse(node, remove_omega=False):
  """ Attempts to convert node into python source code, 
      or at least dump the AST; otherwise, returns node """
  if isinstance(node, AST):
    if remove_omega: node = fix_missing_locations(RemoveOmega().visit(node))
    try:    return astor.to_source(node).rstrip()
    except: return dump(node)
  return node

unprintable(x)

True for items not easily printed out in their full form

Source code in codesmith/eval.py 
106
107
108
109
110
def unprintable(x):
  """True for items not easily printed out in their full form"""
  #print("unprintable?", x, type(x), repr(x))
  if isinstance(x,str): return False
  return (isinstance(x,(type, FunctionType, BuiltinFunctionType)))

wrapper.py

The Wrapper class and ω (omega) function for custom objects in a TSP language

Wrapper

Bases: object

Base Class for a wrapped object (the "candy" object in the wrapper)

Source code in codesmith/wrapper.py 
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
class Wrapper(object):
  """Base Class for a wrapped object (the "candy" object in the wrapper)"""

  @classmethod
  def candy_class(cls):
    """Get the class object (type) for the candy"""
    # candy's class is always #2 in mro
    return cls.__mro__[2]


  def candy_repr(self):
    """Get the repr for the candy class"""
    # By default, just use the candy class's __repr__
    return self.candy_class().__repr__(self)

  def candy_type(self):
    """Get the type of the candy class"""
    return self.candy_class().__name__

  #def __repr__(self):
  #  return f"{self.candy_repr()}: {self.candy_type()}"

  def _repr_html_(self):
    """Generate suitable output html for the wrapped class"""
    if hasattr(self,"_repr_svg_"):
      out = self.candy_class()._repr_svg_(self)
    else:
      from html import escape
      out = escape(f"{self.candy_repr()}")

    out += ("\n<span style='float:right; font-family:monospace; "
            "font-weight:bold; background-color:#e5e5ff; color:black;'>"
            f"{self.candy_type()}</span>")
    return out

candy_class() classmethod

Get the class object (type) for the candy

Source code in codesmith/wrapper.py 
11
12
13
14
15
@classmethod
def candy_class(cls):
  """Get the class object (type) for the candy"""
  # candy's class is always #2 in mro
  return cls.__mro__[2]

candy_repr()

Get the repr for the candy class

Source code in codesmith/wrapper.py 
18
19
20
21
def candy_repr(self):
  """Get the repr for the candy class"""
  # By default, just use the candy class's __repr__
  return self.candy_class().__repr__(self)

candy_type()

Get the type of the candy class

Source code in codesmith/wrapper.py 
23
24
25
def candy_type(self):
  """Get the type of the candy class"""
  return self.candy_class().__name__

register_candy(*candy)

Generate a new, default Wrapper class for one or more candy class

Source code in codesmith/wrapper.py 
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
def register_candy(*candy):
  """Generate a new, default Wrapper class for one or more candy class"""
  default = candy[0]
  wrapper_name = capwords(default.__name__) + "Wrapper"
  wrapper = type(wrapper_name, (Wrapper, default), {})
  for cls in candy:
    wrappers[cls] = wrapper

  def make_new(method):
    def newmethod(self, *args, **kwargs):
      out = getattr(default, method)(self, *args, **kwargs)
      # if isinstance(out, candy):
      #   print(wrapper_name, method, "->", out, type(out))
      #   return wrapper(out)
      return ω(out)
    return newmethod

  from inspect import getmembers, ismethoddescriptor
  for method in [m  for c in candy 
                    for m, _ in getmembers(c, ismethoddescriptor) ]:
    if method == '__init__': continue
    setattr(wrapper, method, make_new(method))
  return default

register_wrapper(*args)

Decorator to register a defined wrapper for one or more candy classes

Source code in codesmith/wrapper.py 
58
59
60
61
62
63
64
65
66
67
68
69
def register_wrapper(*args):
  """Decorator to register a defined wrapper for one or more candy classes"""
  if len(args) == 1 and issubclass(args[0], Wrapper):
    wrapper = args[0]
    wrappers[wrapper.candy_class()] = wrapper
    return wrapper

  def _wrap(wrapper):
    for candy in args:
      wrappers[candy] = wrapper
    return wrapper
  return _wrap

unregister_candy(candy)

Unregister a wrapper for this candy class

Source code in codesmith/wrapper.py 
95
96
97
def unregister_candy(candy):
  """Unregister a wrapper for this candy class"""
  return wrappers.pop(candy,None)

ω(x)

Wrapper factory that checks for defined subclass, or creates one if necessary

Source code in codesmith/wrapper.py 
49
50
51
52
53
54
55
def ω(x):
  """Wrapper factory that checks for defined subclass, or creates one if necessary"""
  candy_class = x.__class__
  try:
    return wrappers[candy_class](x)
  except (TypeError, KeyError):
    return x

python.py

A codesmith grammar for the python language itself

Follows the offical python grammar very closely: https://docs.python.org/3/reference/grammar.html