seminars.fb

Programming Fundamentals → “Metaprogramming in Python: Writing code that writes code”

Seminar (Fri, May 7, 2021; 12:30 PM PST)

Theme: Programming Fundamentals

Topic: Metaprogramming

Presenter	James Powell james@dutc.io
Date	Friday, May 7, 2021
Time	12:30 PM PST

Computers make our lives easier by allowing us to automate tasks we would otherwise do by hand. But the practice of writing code itself can be tedious. So why can’t I write a computer programme that automates my task of writing computer programmes that automates my task of writing computer programmes that automates my task of writing computer programmes…?

This seminar will present a view of metaprogramming approaches in Python, focusing on questions like:

• what is metaprogramming, and when can it be a productive approach for writing more robust code (and automating the process of writing boilerplate)?
• what does metaprogramming mean in Python, and why does it appear to be so closely tied with the object construction protocols?
• what metaprogramming techniques are you already using without even knowing?
• what is collections.namedtuple, dataclasses.dataclass, and enum.Enum and how do they represent three core metaprogramming approaches: eval/exec, def- and class-decorators, and metaclasses/init_subclass/build_class?
• what are update anomalies, and how can metaprogramming approaches help us to add programmatic structure (to our code!) to help avoid them?

Contents

• Notes
  • Part I: Premise
  • Part II: Mechanisms & Overview
    • Decorators and Class Decorators
    • Class Construction
    • eval and exec
  • Part III: Summary

Notes

print("Let's go!")

Part I: Premise

Avoid repetition!

Avoid “update anomalies!”

from pandas import DataFrame, date_range, IndexSlice
from string import ascii_lowercase
from numpy import tile, repeat, hstack

from numpy.random import default_rng
from pandas import Timestamp
rng = default_rng(Timestamp('2021-05-07').asm8.astype('uint32'))

devices1 = rng.choice([*ascii_lowercase], size=(num_devices := 5, 8))
devices1[:, -4:] = [*'.net']
devices1 = devices1.view('<U8').ravel()

devices2 = rng.choice([*ascii_lowercase], size=(len(devices1) - 2, 8))
devices2[:, -4:] = [*'.net']
devices2 = devices2.view('<U8').ravel()
devices2 = hstack([devices1[:2], devices2])

time = date_range('2021-05-07 9:00', periods=(num_periods := 8), freq='1H')

df1 = DataFrame({
    'device':  repeat(devices1, len(time)),
    'time':    tile(time, len(devices1)),
    'signal':  rng.normal(size=(len(time) * len(devices1))),
})

df2 = DataFrame({
    'device':  repeat(devices2, len(time)),
    'time':    tile(time, len(devices2)),
    'signal':  rng.normal(size=(len(time) * len(devices2))),
})

print(
    #  df1.head(3),
    #  df2.head(3),
    #  df1['device'].unique(),
    #  df2['device'].unique(),
    #  df1.set_index(['device', 'time']),
    #  df2.set_index(['device', 'time']),
    #  (
    #        df1.set_index(['device', 'time'])
    #      + df2.set_index(['device', 'time'])
    #  ).dropna(),
)

print(
    df1.set_index(['device', 'time']).loc[IndexSlice[:, '2021-05-07 12:00':]] * .5,
    #  df2.set_index(['device', 'time']).loc[IndexSlice[:, '2021-05-07 12:00':]] * .5,
)

def clean_data(df):
    df = df.set_index(['device', 'time'])
    ...
    return df

def f(data):
    data = clean_data(data, remove_outliers=True) 
    ...

def g(data):
    data = clean_data(data, remove_outliers=True) 
    ...

def h(data):
    data = clean_data(data, remove_outliers=False) 
    ...

def remove_outliers(data):
    ...
def normalize_units(data):
    ...
def scale_data(data):
    ...

def f(data):
    data = remove_outliers(data)
    data = normalize_units(data)

def g(data):
    data = scale_data(data)
    data = normalize_units(data)

def h(data):
    data = scale_data(data)
    data = normalize_units(data)

def clean_data_400g(data):
    pass

def clean_data_100g(data):
    pass

Part II: Mechanisms & Overview

1. decorators & class decorators
2. class construction (__build_class__, metaclasses, and __init_subclass__)
3. eval/exec

Decorators and Class Decorators

class T:
    def __init__(self, x, y):
        self.x, self.y = x, y
    def __repr__(self):
        return f'T({self.x!r}, {self.y!r})'
    def __eq__(self, other):
        return self.x == other.x and self.y == other.y
    def __hash__(self):
        return hash((self.x, self.y))
            
a = T(1, 2)
b = T(1, 2)
print(
    f'{a      = }',
    f'{a == b = }',
    sep='\n'
)

from dataclasses import dataclass
from functools import total_ordering

@total_ordering
@dataclass
class T:
    x : int
    y : int
    def __lt__(self, other):
        return (self.x, self.y) < (other.x, other.y)
    #  def __gt__(self, other):
    #      return (self.x, self.y) > (other.x, other.y)
    #  def __lte__(self, other):
    #      return (self.x, self.y) <= (other.x, other.y)
    #  def __gte__(self, other):
    #      return (self.x, self.y) >= (other.x, other.y)

x = T(1, 2)
y = T(1, 2)
z = T(3, 4)
print(
    x,
    x == y,
    x < z,
    sep='\n'
)

from inspect import signature

def f(name='Aditya'):
    ''' prints a message to the screen '''
    print(f'Hello, {name}!')

f('Raj')

f.__name__ = f.__qualname__ = 'eff'

print(
    f'{f              = }',
    f'{f.__name__     = }',
    f'{f.__doc__      = }',
    f'{f.__defaults__ = }',
    f'{signature(f)   = }',
    sep='\n',
)

def do_twice(f):
    for _ in range(2):
        f()

def f(name='Raj'):
    ''' prints a message to the screen '''
    print(f'Hello, {name}!')

def g(msg='Goobye', name='Raj'):
    ''' prints a message to the screen '''
    print(f'{msg}, {name}!')

do_twice(f)
do_twice(g)

def do_twice(f, *args, **kwargs):
    for _ in range(2):
        f(*args, **kwargs)

def f(name='Raj'):
    ''' prints a message to the screen '''
    print(f'Hello, {name}!')

do_twice(f, name='Radhika')

def do_n_times(f, *args, times=2, **kwargs):
    for _ in range(times):
        f(*args, **kwargs)

def f(name='Raj', times='good'):
    ''' prints a message to the screen '''
    print(f"Hello, {name}! We're having a {times} time")

do_n_times(f, times=4, name='Radhika')

for _ in range(10):
    def f():
        pass

f()

def do_n_times(f, times=2):
    def inner(*args, **kwargs):
        for _ in range(times):
            f(*args, **kwargs)
    return inner

def f(name='Raj'):
    ''' prints a message to the screen '''
    print(f'Hello, {name}!')

new_f = do_n_times(f, times=4)
new_f(name='Radhika')

do_n_times(f, times=4)(name='Radhika')

def do_n_times(f, times=2):
    def inner(*args, **kwargs):
        for _ in range(times):
            f(*args, **kwargs)
    return inner

def f(name='Raj'):
    ''' prints a message to the screen '''
    print(f'Hello, {name}!')
f = do_n_times(f)

f(name='Radhika')

def do_twice(f):
    def inner(*args, **kwargs):
        for _ in range(2):
            f(*args, **kwargs)
    return inner

@do_twice
def f(name='Raj'):
    ''' prints a message to the screen '''
    print(f'Hello, {name}!')
#  f = do_twice(f)

f(name='Radhika')

def do_n_times(times=2):
    def dec(f):
        def inner(*args, **kwargs):
            for _ in range(times):
                f(*args, **kwargs)
        return inner
    return dec

@do_n_times(times=4)
def f(name='Raj'):
    ''' prints a message to the screen '''
    print(f'Hello, {name}!')

f(name='Radhika')

from functools import wraps

def do_n_times(times=2):
    def dec(f):
        @wraps(f)
        def inner(*args, **kwargs):
            for _ in range(times):
                f(*args, **kwargs)
        return inner
    return dec

@do_n_times(times=4)
def f(name='Raj'):
    ''' prints a message to the screen '''
    print(f'Hello, {name}!')

f(name='Radhika')

from functools import wraps

def dec(f):
    @wraps(f)
    def inner(*args, **kwargs):
        return f(*args, **kwargs)
    return inner

# “metaphor”
@dec
def f():
    pass
#  f = dec(f)

from functools import wraps

registry = {}
def metadata(**kwargs):
    def dec(f):
        registry[f] = kwargs
        return f
    return dec

@metadata(
    author = 'Radhika R',
    email  = 'radr@example.com',
)
def f():
    pass

@metadata(
    author = 'Aditya A',
    email  = 'aa@example.com',
    platform = '...',
)
def g():
    pass

print(registry)

class T:
    pass

def foo(self):
   pass
T.foo = foo

x = T()
x.foo()

print(
    T,
    f'{T.__name__ = }',
    f'{T.__mro__  = }',
    f'{dir(T)     = }',
    sep='\n',
)

def dec(cls):
    return cls

@dec
class T:
    pass

print(
    T,
    #  f'{dir(T) = }',
)

from dataclasses import dataclass

class Interface:
    pass

@dataclass
class Derived(Interface):
    x : int
    y : int

Class Construction

from enum import Enum, auto
from random import choice

class Colors(Enum):
    Red   = auto()
    Green = auto()
    Blue  = auto()

print(
    Colors,
    Colors.Red,
    Colors['Red'],
    Colors['Red'] is Colors['Red'],
    #  Colors['Orange'],
    #  f'{Colors["Red"].value = }',
    #  f'{[*Colors]           = }',
    f'{choice([*Colors])   = }',
    sep='\n',
)

class T:
    pass

print(T)

for _ in range(10):
    class T:
        pass

print(T)

for _ in range(10):
    def f():
        pass
        
for _ in range(10):
    class T:
        for _ in range(10):
            def f():
                pass

print(T)

def f():
    class T:
        pass
    return T

print(f())
print(f())
print(f())
T = f()
x = T()
T = f()
print(f'{isinstance(x, T) = }')

def f():
    class T:
        pass
    return T

from dis import dis
dis(f.__code__)

import builtins

@lambda f: setattr(builtins, '__build_class__', f(builtins.__build_class__))
def bind(orig):
    def __build_class__(func, name, *args, **kwargs):
        print(f'{name = }, {func = }')
        cls = orig(func, name, *args, **kwargs)
        return cls
    return __build_class__

class A:
    pass
class B:
    pass

print(
    #  A,
)

import pandas
import matplotlib.pyplot
import networkx

class A:
    pass

class B(A):
    def __new__(cls):
        print(f'C.__new__({cls!r})')
        return super().__new__(cls)
    def __init__(self):
        print(f'C.__init__({self!r})')
        super().__init__()
    def __call__(self):
        print(f'C.__call__({self!r})')

x = B()
print(
    f'{x   = }',
    f'{x() = }',
    sep='\n'
)

class metaclass(type):
    def __call__(self):
        print(f'metaclass.__call__({self!r})')
        return super().__call__()

    def __new__(cls, name, bases, body, **kwds):
        print(f'metaclass.__new__({cls!r}, {name!r}, {bases!r}, {body!r}, {kwds!r})')
        return super().__new__(cls, name, bases, body)

    def __init__(self, name, bases, body, **kwds):
        print(f'metaclass.__init__({self!r}, {name!r}, {bases!r}, {body!r}, {kwds!r})')
        return super().__init__(name, bases, body)

    @classmethod
    def __prepare__(cls, name, bases, **kwds):
        print(f'metaclass.__prepare__({cls!r}, {name!r}, {bases!r}, {kwds!r})')
        return super().__prepare__(cls, name, bases, **kwds)
        
class A(metaclass=metaclass): pass
#  x = A()
class B(A, x=10, y=200, z=3_000): pass
class C(B):
    def __new__(cls):
        print(f'C.__new__({cls!r})')
        return super().__new__(cls)
    def __init__(self):
        print(f'C.__init__({self!r})')
        super().__init__()
    def __call__(self):
        print(f'C.__call__({self!r})')
    #  def __init_subclass__(cls, **kwds):
    #      print(f'C.__init_subclass__({cls!r}, {kwds!r})')
#  class D(C): pass
#  class E(D, x='ecks', y='why', z='zee'): pass

class Base:
    pass

# ---

class Derived(Base):
    pass

class Base:
    def foo(self):
        pass
    pass

# ---

print(
    Base,
)
if not hasattr(Base, 'foo'):
    raise TypeError(f'{Base} must have attribute "foo"')
class Derived(Base):
    def bar(self):
        return self.foo()
    pass

class Base:
    def foo(self):
        return self.bar() + 1

# ---

class Derived(Base):
    pass

class BaseMeta(type):
    def __init__(self, name, bases, body):
        if name != 'Base' and 'bar' not in body:
            raise TypeError(f'{name} must have attribute "foo"')
            
class Base(metaclass=BaseMeta):
    def foo(self):
        return self.bar()

# ---

class Derived(Base):
    def bar(self):
        pass
    pass

def check(cls):
    if not hasattr(cls, 'bar'):
        raise TypeError(f'{cls} must have attribute "bar"')
    return cls

#  @check
class Derived:
    #  def bar(self):
    #      pass
    pass

class Base:
    def foo(self):
        return self.bar()
    def __init_subclass__(cls):
        if not hasattr(cls, 'bar'):
            raise TypeError(f'{cls} must have attribute "bar"')

# ---

class Derived(Base):
    def bar(self):
        pass
    pass

from dataclasses import dataclass

class Component:
    TYPES = {}

    @classmethod
    def from_name(cls, name, **fields):
        return cls.TYPES[name](**fields)

    def __init_subclass__(cls, names):
        for n in names:
            cls.TYPES[n] = cls

@dataclass
class Resistor(Component, names={'resistor'}):
    resistance : float

@dataclass
class Capacitor(Component, names={'capacitor', 'cap'}):
    capacitance : float

x = Component.from_name('resistor',  resistance=1.5)
y = Component.from_name('capacitor', capacitance=3.5)
print(
    #  f'{x = }',
    #  f'{y = }',
    sep='\n',
)

from pandas import Index, date_range
idx = Index([1, 2, 3])
idx = Index([1., 2., 3.])
idx = Index(range(3))
idx = Index([*'abc'])
idx = Index(date_range('2021-05-07', periods=3))
print(
    idx,
)

from dataclasses import dataclass
from collections import namedtuple

class Component:
    TYPES = {}

    @classmethod
    def from_name(cls, name, **fields):
        return cls.TYPES[name](**fields)

    def __init_subclass__(cls, names):
        for n in names:
            cls.TYPES[n] = cls

ComponentType = namedtuple('ComponentType', 'cls_name names fields')
component_types = [
    ComponentType('Resistor',  frozenset({'resistor'}),         {'resistance':  float}),
    ComponentType('Capacitor', frozenset({'capacitor', 'cap'}), {'capacitance': float}),
    ComponentType('Inductor',  frozenset({'inductor'}),         {'inductance':  float}),
]

for ct in component_types:
    locals()[ct.cls_name] = type(ct.cls_name,
                                 (Component,),
                                 {'__annotations__': ct.fields},
                                 names=ct.names)
    locals()[ct.cls_name] = dataclass(locals()[ct.cls_name])

x = Component.from_name('resistor', resistance=1.5)
y = Component.from_name('capacitor', capacitance=3.5)
print(
    f'{x = }',
    f'{y = }',
    sep='\n',
)

eval and exec

print(
    eval('1 + 1')
)

exec('x = 1 + 1', globals())
print(
    x,
)

class A:
    pass

class B:
    pass

class C:
    pass

from string import ascii_uppercase
from textwrap import dedent

for cls_name in ascii_uppercase[:3]:
    code = dedent(f'''
        class {cls_name}:
            pass
    ''')
    print(code)
    exec(code, globals())

x, y, z = A(), B(), C()
print(
    #  x,
    #  y,
    #  z,
    sep='\n'
)

from sys import version_info
assert version_info.major == 2

from collections import namedtuple

T = namedtuple('T', 'a b c', verbose=True)

Part III: Summary

1. function decorators (e.g., functools.total_ordering)
2. class decorators (e.g., dataclasses.dataclass)
3. instance construction
4. class construction
5. type
6. __build_class__
7. metaclass (e.g., enum.Enum)
8. __init_subclass__
9. exec (e.g., Python 2’s collections.namedtuple)