all-mods.py

#!/usr/bin/env python3

import os
import json
from collections import defaultdict

import common

def module():
    return defaultdict(set)

def library():
    return defaultdict(module)


modules = defaultdict(library)

for p in os.listdir('.'):
    if not p.startswith('.files.mod2files.'):
        continue
    if not p.endswith('.json'):
        continue

    ftype = p[17:-5]
    data = json.load(open(p))

    for mod, files in data.items():
        libname, modname = common.lib_extract_from_name(mod)
        assert libname is None, (libname, modname, mod)
        assert modname is not None, (libname, modname, mod)

        for f in files:
            if 'rechar' in f:
                continue
            flibname, fmodname = common.mod_extract_from_path(f)
            if fmodname in ('pads', 'power_pads_lib', 'pads_routing', 'primdev', 's8_osu130'):
                fmodname = None
            if fmodname is None:
                fmodname = modname

            llibname = libname
            if llibname is None:
                llibname = flibname

            #if fmodname is not None and modname is not None:
            #    assert fmodname == modname, (
            #        ftype, ("file:", flibname, fmodname), f, ("origin:", libname, modname), mod)

            modules[modname][llibname][ftype].add(f)
            modules[modname]['all'][ftype].add(f)


for mod in sorted(modules):
    libs = list(modules[modname].keys())
    libs.remove('all')
    if None in libs:
        libs.remove(None)
    libs.sort()
    print("%-40s %-30s" % (modname, modules[modname].keys()))


all_files = set()
for mod in modules:

    # Write out a summary file for the module
    f = open(os.path.join('modules', mod+'.rst'), 'w')
    f.write('='*len(mod))
    f.write('\n')
    f.write(mod)
    f.write('\n')
    f.write('='*len(mod))
    f.write('\n')
    f.write('\n')
    libraries = list(modules[mod].keys())
    f.write('Found in:\n')
    for lib in libraries:
        if lib == 'all':
            continue
        lib = str(lib)
        f.write(' * ')
        f.write(lib)
        f.write('\n')

    f.write('\n')
    for lib in libraries:
        if lib == 'all':
            continue
        lib = str(lib)
        f.write(len(lib)*'-')
        f.write('\n')
        f.write(lib)
        f.write('\n')
        f.write(len(lib)*'-')
        f.write('\n')
        f.write('\n')

        for t in modules[mod][lib]:
            f.write(t)
            f.write('\n')
            f.write('+'*len(t))
            f.write('\n')
            f.write('\n')

            for fn in sorted(modules[mod][lib][t]):
                f.write(' - ')
                f.write(fn)
                f.write('\n')
            f.write('\n')
    f.close()

    # Write out a file list for the module
    for lib in libraries:
        files = set()
        for t in modules[mod][lib]:
            for fn in modules[mod][lib][t]:
                files.add(fn)

        libdir = os.path.join('modules', str(lib))
        if not os.path.exists(libdir):
            os.makedirs(libdir)

        with open(os.path.join(libdir, mod+'.files'), 'w') as f:
            for fn in sorted(files):
                f.write(fn)
                f.write('\n')

    print()
    print(mod, end='')
    libraries = list(modules[mod].keys()) #list(x for x in modules[mod].keys() if x is not None)
    if len(libraries) > 0:
        print(" in", libraries)
    else:
        print()
    print("="*75)
    files = defaultdict(set)

    for l in modules[mod]:
        lib = modules[mod][l]
        if l is not None:
            print("In lib", l, "has", end=" ")
        else:
            print("Has", end=" ")
        print(list(lib.keys()), "files")
        for t in lib:
            for f in lib[t]:
                files[t].add(f)

    for t in sorted(files):
        if t == 'files':
            continue
        print()
        print(t)
        print('-'*5)
        for f in sorted(files[t]):
            all_files.add(f)
            print(f)

    def in_other(f):
        for t in files:
            if t == 'files':
                continue
            if f in files[t]:
                return True
        return False

    if 'files' in files:
        print()
        print("Other files")
        print('-'*5)
        for f in sorted(files['files']):
            if in_other(f):
                continue
            else:
                print(f)

print()
print()
print("Remaining unexamined files")
print("="*75)
with open('.files.unused', 'w') as f:
    for fn in common.files([]):
        if fn in all_files:
            continue
        print(fn)
        f.write(fn)
        f.write('\n')