prim_audit.py

#!/usr/bin/env python3

import os
import pprint
import re
import sys
import textwrap

from collections import defaultdict

def k(s):
    if s is True:
        s = ''
    return s

def srepr(s):
    r = repr(s)
    if len(r) > 100:
        r = r[:100] + '...'
    return r


IGNORE_EXT = [
    'magic.lef',
    'netlist.tsv',
    'svg',
]

def files_by_extension(dname):
    files = set()
    ext2base = defaultdict(set)
    ext2file = defaultdict(set)

    for fname in os.listdir(dname):
        fpath = os.path.join(dname, fname)
        if not os.path.isfile(fpath):
            continue
        files.add(fname)
        assert '.' in fname, (fname, dname)
        base, ext = fname.split('.', 1)

        lib, base = base.split('__', 1)
        assert lib.startswith('sky130_'), lib
        if '__' in base:
            cellname, base = base.split('__', 1)
        else:
            base = ''

        if ext in IGNORE_EXT:
            continue

        ext2base[ext].add(base)
        ext2file[ext].add(fname)

    oext2base = {}
    for k in ext2base:
        oext2base[k] = list(sorted(ext2base[k]))

    oext2file = {}
    for k in ext2file:
        oext2file[k] = list(sorted(ext2file[k]))

    ofiles = [os.path.abspath(os.path.join(dname, f)) for f in sorted(files)]
    return oext2base, oext2file, ofiles


RE_SUBCKT = re.compile('^.subckt +(?P<subckt>[^ ]+) (?P<ports>.*)$', flags=re.M)
RE_INCLUDE = re.compile('^.inc((lude +["\'](?P<f>[^"\']+)["\'] *)|(?P<broken>.*))$', flags=re.M)

EXTRA_SUBCKTS = [
    'base',
    'subcell',
]


ERROR_HEADERS = {
    'gds-missing'           : 'No GDS file found for:',
    'gds-example-only'      : 'Only example GDS files found for:',
    'spice-model-missing'   : 'No .model.spice or .pm3.spice file found for:',
    'spice-model-multiple'  : 'Found both a .model.spice and a .pm3.spice file for:',
    'spice-bins-missing'    : 'Found a .pm3.spice file but no bins.csv file for:',
    'spice-include-missing' : 'Missing files included in spice file for:',
    'spice-include-error'   : 'Invalid include statements in file for:',
    'spice-subckt-missing'  : 'No subckt found in files for:',
    'spice-subckt-multiple' : 'Multiple subckts found in files for:',
    'spice-subckt-incorrect': 'Incorrect subckt found in files for:',
}
ERROR_STRINGS = {
    'gds-missing'           : '',
    'gds-example-only'      : '',
    'spice-model-missing'   : '',
    'spice-model-multiple'  : '',
    'spice-bins-missing'    : '',
    'spice-include-missing' : '{e}',
    'spice-include-error'   : '{e}',
    'spice-subckt-missing'  : '{file}: No subckt',
    'spice-subckt-incorrect': '{file}: Incorrect subckt value for {t} - got: {g!r} (from {s!r}), wanted: {w!r} (from {f!r})',
}
ERROR_TYPES = set(ERROR_HEADERS.keys())


def emsg(etype, ekw, inctype=True, indent=''):
    m = ERROR_STRINGS[etype].format(**ekw)
    if inctype:
        if m:
            m = etype+': '+m
            return ('\n'+indent+' '*(len(etype)+2)).join(m.split('\n'))
        else:
            return etype
    return ('\n'+indent).join(m.split('\n'))


def flatten(f):
    fdir = os.path.dirname(f)
    data = open(f).read()
    odata = []

    oend = 0
    for m in RE_INCLUDE.finditer(data):
        odata.append(data[oend:m.start(0)])
        incf = m.group('f')
        if not incf:
            raise IOError('invalid-include: {!r} in {!r}'.format(m.group(0), f))
        incpath = os.path.abspath(os.path.join(fdir, incf))
        if not os.path.isfile(incpath):
            raise FileNotFoundError('invalid-include: {!r} in {!r}'.format(incpath, f))
        try:
            odata.append(flatten(incpath))
        except IOError as e:
            raise e.__class__(str(e) + '\n(included in {!r} byte:{})'.format(f, m.start(0)))
        oend = m.end(0)

    odata.append(data[oend:])
    odata = ''.join(odata)
    m = RE_INCLUDE.search(odata)
    assert not m, m
    return odata



def audit(ext2base, ext2file, files):
    errors = []
    warnings = []

    def add_error(etype, **kw):
        assert etype in ERROR_TYPES, (etype, kw)
        ERROR_STRINGS[etype].format(**kw)
        errors.append((etype, kw))
    def add_warning(etype, **kw):
        assert etype in ERROR_TYPES, (etype, kw)
        ERROR_STRINGS[etype].format(**kw)
        warnings.append((etype, kw))

    if 'gds' not in ext2base:
        add_error('gds-missing')
    elif '' not in ext2base['gds']:
        add_error('gds-example-only')

    if 'model.spice' not in ext2base and 'pm3.spice' not in ext2base:
        add_error('spice-model-missing')

    if 'model.spice' in ext2base and 'pm3.spice' in ext2base:
        if '' in ext2base['model.spice'] and '' in ext2base['pm3.spice']:
            add_error('spice-model-multiple')

    if 'pm3.spice' in ext2base and 'bins.csv' not in ext2base:
        add_error('spice-bins-missing')

    for fpath in files:
        if not fpath.endswith('.spice'):
            continue

        fname = fpath.rsplit('/', 1)[-1]
        fbase, ext = fname.split('.', 1)

        flib, fcell = fbase.split('__', 1)
        fextra = None
        if '__' in fcell:
            fcell, fextra = fcell.split('__', 1)

        try:
            data = flatten(fpath)
        except FileNotFoundError as e:
            add_error('spice-include-missing', e=str(e))
            continue
        except IOError as e:
            add_error('spice-include-error', e=str(e))
            continue

        found_subckts = [m.group('subckt') for m in RE_SUBCKT.finditer(data)]
        if not found_subckts:
            if 'mismatch' in fpath:
                continue
            add_error('spice-subckt-missing', file=fname)
            continue

        for subckt in found_subckts:
            if '__' not in subckt and fbase != subckt:
                add_error(
                    'spice-subckt-incorrect',
                    file=fpath,
                    t='full', w=fbase, g=subckt,
                    f='', s='')
                continue

            subckt_lib, subckt_cell = subckt.split('__', 1)
            subckt_extra = None
            if '__' in subckt_cell:
                subckt_cell, subckt_extra = subckt_cell.split('__', 1)

            if flib != subckt_lib:
                add_error(
                    'spice-subckt-incorrect',
                    file=fpath,
                    t='lib', w=flib, g=subckt_lib,
                    f=fbase, s=subckt)

            if subckt_cell != fcell:
                if subckt_cell.startswith(fcell):
                    subckt_variant = subckt_cell[len(fcell)+1:]
                    if subckt_variant.startswith('_'):
                        subckt_variant = subckt_variant[1:]
                    if '_' in subckt_variant:
                        add_error(
                            'spice-subckt-incorrect',
                            file=fpath,
                            t='variant', w='no _', g=subckt_variant,
                            f=fbase, s=subckt)
                    else:
                        add_warning(
                            'spice-subckt-incorrect',
                            file=fpath,
                            t='variant', w='seperate file', g=subckt_variant,
                            f=fbase, s=subckt)
                else:
                    add_error(
                        'spice-subckt-incorrect',
                        file=fpath,
                        t='cell', w=fcell, g=subckt_cell,
                        f=fbase, s=subckt)

            if subckt_extra in EXTRA_SUBCKTS:
                continue
            elif fextra is not None and subckt_extra is not None:
                add_error(
                    'spice-subckt-incorrect',
                    file=fpath,
                    t='extra', w=fextra, g=subckt_extra,
                    f=fbase, s=subckt)

    return errors, warnings


GOOD = 0
ERROR = 1


def main(argv):
    data = defaultdict(dict)

    libs = []

    for a in argv:
        assert os.path.exists(a), a
        assert os.path.isdir(a), a

        apath = os.path.abspath(a)

        print()
        print(a)
        print('='*75)

        allgood = []
        witherrors = {}
        libs.append((apath, (allgood, witherrors)))

        for dname in sorted(os.listdir(apath)):
            dpath = os.path.join(a, dname)
            if not os.path.isdir(dpath):
                print('Not directory', dpath)
                continue

            ext2base, ext2file, files = files_by_extension(dpath)
            data[dname] = ext2base

            errors, warnings = audit(ext2base, ext2file, files)
            if not errors:
                print()
                print(dname, 'is all good')
                allgood.append(dname)
                continue
            else:
                witherrors[dname] = errors

            print()
            print(dname)
            print(textwrap.indent(pprint.pformat(files), '   '))
            print()
            if errors:
                print('Errors:')
                for etype, ekw in errors:
                    print(' *', emsg(etype, ekw, indent='   '))
            print()
            if warnings:
                print('Warnings:')
                for etype, ekw in warnings:
                    print(' *', emsg(etype, ekw, indent='   '))

            continue
            print('-'*10)
            for k, v in sorted(ext2base.items()):
                base_found = '' in v
                examples = [i[len('example_'):] for i in v if i.startswith('example_')]
                nonexamples = [i for i in v if i and not i.startswith('example_')]

                print(' *', '%15s,' % k, ['Not Found', '    Found'][base_found], end='')
                if examples:
                    print(', Examples:', srepr(examples), end='')
                if nonexamples:
                    print(', Other:', srepr(nonexamples), end='')
                print()
            print('-'*10)


            #pprint.pprint(ext2file)
            #print('-'*10)
            #pprint.pprint(ext2base)

        numfiles = len(allgood)+len(witherrors)
        def g(f):
            if len(libs) < 2:
                return ''
            if f in libs[-2][1][GOOD]:
                return '(still good)'
            elif f in libs[-2][1][ERROR]:
                return '(previously broken)'
            else:
                return '(new)'
        def e(f):
            if len(libs) < 2:
                return ''
            if f in libs[-2][1][ERROR]:
                return '(still broken)'
            elif f in libs[-2][1][GOOD]:
                return '(previously good)'
            else:
                return '(new)'
        def p(l):
            return '(%02.f%%)' % (len(l)/numfiles*100)

        if len(libs) < 2:
            all_previous = []
        else:
            all_previous = set(list(libs[-2][1][ERROR])+list(libs[-2][1][GOOD]))
        print()
        print()
        print()
        print("Summary for:", a)
        print('='*75)
        print('All good', len(allgood), 'cells', p(allgood))
        print('-'*20)
        for f in allgood:
            print('  ', '%20s' % g(f), f)
            if f in all_previous:
                all_previous.remove(f)
        print('-'*20)
        print()
        print('With errors', len(witherrors), 'cells', p(witherrors))

        byerrors = defaultdict(list)
        for f in witherrors:
            for etype, ekw in witherrors[f]:
                byerrors[etype].append((f,ekw))
            if f in all_previous:
                all_previous.remove(f)

        def k(v):
            f, ekw = v
            return f, list(ekw.items())

        print('-'*20)
        for etype, v in byerrors.items():
            print()
            print(ERROR_HEADERS[etype], len(v), p(v))
            for f, ekw in sorted(v, key=k):
                m = emsg(etype, ekw, inctype=False, indent=' '*(24+len(f)+4))
                if m:
                    m = ' - '+m
                    m = m.replace(apath, '')
                print('  ', '%20s' % e(f), f, m)
        print('-'*20)

        if all_previous:
            print()
            print('Removed', len(all_previous), 'cells')
            print('-'*20)
            for f in sorted(all_previous):
                print('  ', '%20s' % '(removed)', f)
            print('-'*20)
        else:
            print('None removed')

        print('='*75)

    print()
    print()
    return 0



if __name__ == "__main__":
    try:
        sys.exit(main(sys.argv[1:]))
    except SystemExit as e:
        raise
    except:
        import traceback
        traceback.print_exc()
        sys.exit(1)