#!/usr/bin/env python3
import re, glob, json, csv
import datetime
import yaml
import subprocess
import argparse, logging, shutil, sys, os, collections
from git_utils import install_artifacts
import git
from project_urls import filler_project_url, test_project_urls, project_urls


# pipe handling
from signal import signal, SIGPIPE, SIG_DFL
signal(SIGPIPE, SIG_DFL)

tmp_dir = '/tmp/tt'
DEFAULT_NUM_PROJECTS = 250


def unique(duplist):
    unique_list = []
    # traverse for all elements
    for x in duplist:
        # check if exists in unique_list or not
        if x not in unique_list:
            unique_list.append(x)
    return unique_list


class Projects():

    def __init__(self, args):
        self.args = args
        # project urls are defined in project_urls module
        # by default, this method will fill empty slots with the 'filler project'
        if self.args.test:
            self.project_urls = test_project_urls
            project_dir = 'test_projects'
        else:
            self.project_urls = project_urls
            project_dir = 'projects'

        logging.info(f"loaded {len(self.project_urls)} projects")

        if not os.path.exists(project_dir):
            os.makedirs(project_dir)

        # useful for contacting people when I find problems in the repo
        try:
            with open('git_url_to_email.json') as fh:
                self.git_url_to_email_map = json.load(fh)
        except FileNotFoundError:
            self.git_url_to_email_map = {}

        self.projects = []
        filler = False
        filler_id = 0
        for index in range(args.limit_num_projects):
            if args.single >= 0 and args.single != index:
                continue
            if index < args.start_from:
                continue
            if args.end_at != 0 and index > args.end_at:
                continue

            try:
                git_url = self.project_urls[index]
            except IndexError:
                if filler is False:
                    git_url = filler_project_url
                    filler_id = 0  # first project is always fill
                filler = True

            try:
                email = self.git_url_to_email_map[git_url]
            except KeyError:
                email = 'none'

            project = Project(index, git_url, email, args, project_dir, fill=filler, fill_id=filler_id)

            # clone git repos locally & gds artifacts from action build
            if args.clone_all:
                if filler is False:
                    logging.info(f"cloning {project}")
                    project.clone()

            if args.fetch_gds:
                if filler is False:
                    logging.info(f"fetching gds for {project}")
                    project.fetch_gds()

            if args.update_all:
                if filler is False:
                    # only updates code, not gds artifacts
                    logging.info(f"git pull for {project}")
                    project.pull()

            # projects should now be installed, so load all the data from the yaml files
            logging.debug("loading project yaml")
            # fill projects will load from the fill project's directory
            project.load_yaml()

            logging.info(project)

            if args.harden:
                if filler is False:
                    project.harden()

            if args.update_caravel:
                logging.debug("copying files to caravel")
                if filler is False:
                    project.copy_files_to_caravel()

                # check all top level module ports are correct
                if not project.art_only:
                    project.check_ports()
                project.check_num_cells()

            self.projects.append(project)

        # now do some sanity checks
        all_macro_instances = [project.get_macro_instance() for project in self.projects]
        self.assert_unique(all_macro_instances)

        all_top_files = [project.get_top_verilog_filename() for project in self.projects if not project.is_fill()]
        self.assert_unique(all_top_files)

        all_gds_files = [project.get_macro_gds_filename() for project in self.projects if not project.is_fill()]
        self.assert_unique(all_gds_files)

        if args.metrics:
            self.build_metrics()

    def assert_unique(self, check):
        duplicates = [item for item, count in collections.Counter(check).items() if count > 1]
        if duplicates:
            logging.error("duplicate projects: {}".format(duplicates))
            exit(1)

    def build_metrics(self):
        total_seconds = 0
        for project in self.projects:
            dt = datetime.datetime.strptime(project.metrics['total_runtime'][:-3], '%Hh%Mm%Ss')
            delt = datetime.timedelta(hours=dt.hour, minutes=dt.minute, seconds=dt.second)
            total_seconds += delt.total_seconds()
        logging.info(f"build time for all projects {total_seconds / 3600} hrs")


class Project():

    def __init__(self, index, git_url, email, args, project_dir, fill, fill_id=0):
        self.git_url = git_url
        self.args = args
        self.email = email
        self.index = index
        self.fill = fill
        self.project_dir = project_dir
        self.local_dir = self.gen_local_dir(fill_id)
        self.load_metrics()

    def load_metrics(self):
        try:
            with open(self.get_metrics_path()) as fh:
                self.metrics = next(csv.DictReader(fh))
        except FileNotFoundError:
            self.metrics = {}

    # if the project is a filler, then use the config from the first fill project
    def gen_local_dir(self, fill_id):
        if self.fill:
            return os.path.join(os.path.join(self.project_dir, f'{fill_id :03}'))
        else:
            return os.path.join(os.path.join(self.project_dir, f'{self.index :03}'))

    def check_ports(self):
        top = self.get_macro_name()
        sources = [os.path.join(self.local_dir, 'src', src) for src in self.src_files]
        source_list = " ".join(sources)

        json_file = 'ports.json'
        yosys_cmd = f"yosys -qp 'read_verilog -lib -sv {source_list}; hierarchy -top {top} ; proc; write_json {json_file}'"
        p = subprocess.run(yosys_cmd, shell=True)
        if p.returncode != 0:
            logging.error(f"yosys port read failed for {self}")
            exit(1)

        with open(json_file) as fh:
            ports = json.load(fh)
        os.unlink(json_file)

        module_ports = ports['modules'][top]['ports']
        for port in ['io_in', 'io_out']:
            if port not in module_ports:
                logging.error(f"{self} {port} not found in top")
                exit(1)
            if len(module_ports[port]['bits']) != 8:
                logging.error(f"{self} {port} doesn't have 8 bits")
                exit(1)

    def check_num_cells(self):
        num_cells = 0
        try:
            yosys_report = glob.glob(f'{self.local_dir}/runs/wokwi/reports/synthesis/1-synthesis.*0.stat.rpt')[0]  # can't open a file with \ in the path
            with open(yosys_report) as fh:
                for line in fh.readlines():
                    m = re.search(r'Number of cells:\s+(\d+)', line)
                    if m is not None:
                        num_cells = int(m.group(1))
            if not self.fill and self.index != 0:
                if self.is_hdl():
                    if num_cells < 20:
                        logging.warning(f"{self} only has {num_cells} cells")
                else:
                    if num_cells < 11:
                        logging.warning(f"{self} only has {num_cells} cells")

        except IndexError:
            logging.warning(f"couldn't open yosys cell report for cell checking {self}")

    def is_fill(self):
        return self.fill

    def is_wokwi(self):
        if self.wokwi_id != 0:
            return True

    def is_hdl(self):
        return not self.is_wokwi()

    def load_yaml(self):
        try:
            with open(os.path.join(self.local_dir, 'info.yaml')) as fh:
                self.yaml = yaml.safe_load(fh)
        except FileNotFoundError:
            logging.error(f"yaml file not found for {self} - do you need to --clone the project repos?")
            exit(1)

        self.wokwi_id = self.yaml['project']['wokwi_id']
        self.yaml['project']['git_url'] = self.git_url
        self.art_only = self.yaml['project'].get('art_only', False)

        if self.is_hdl():
            self.top_module             = self.yaml['project']['top_module']
            self.src_files              = self.yaml['project']['source_files']
            self.top_verilog_filename   = self.find_top_verilog()
        else:
            self.top_module             = f"user_module_{self.wokwi_id}"
            self.src_files              = [f"user_module_{self.wokwi_id}.v"]
            self.top_verilog_filename   = self.src_files[0]

        self.macro_instance         = f"{self.top_module}_{self.index :03}"
        self.scanchain_instance     = f"scanchain_{self.index :03}"

    # docs stuff for index on README.md
    def get_index_row(self):
        return f'| {self.index} | {self.yaml["documentation"]["author"]} | {self.yaml["documentation"]["title"]} | {self.get_project_type_string()} | {self.git_url} |\n'

    def get_project_type_string(self):
        if self.is_wokwi():
            return f"[Wokwi]({self.get_wokwi_url()})"
        else:
            return "HDL"

    def get_project_doc_yaml(self):
        # fstring dict support is limited to one level deep, so put the git url and wokwi url in the docs key
        docs = self.yaml['documentation']
        docs['project_type'] = self.get_project_type_string()
        docs['git_url'] = self.git_url
        return docs

    def get_wokwi_url(self):
        return f'https://wokwi.com/projects/{self.wokwi_id}'

    # top module name is defined in one of the source files, which one?
    def find_top_verilog(self):
        rgx_mod  = re.compile(r"(?:^|[\W])module[\s]{1,}([\w]+)")
        top_verilog = []
        for src in self.src_files:
            with open(os.path.join(self.local_dir, 'src', src)) as fh:
                for line in fh.readlines():
                    for match in rgx_mod.finditer(line):
                        if match.group(1) == self.top_module:
                            top_verilog.append(src)
        assert len(top_verilog) == 1
        return top_verilog[0]

    def clone(self):
        if os.path.exists(self.local_dir):
            git_remote = list(git.Repo(self.local_dir).remotes[0].urls)[0]
            if self.git_url == git_remote:
                logging.info("git repo already exists and is correct - skipping")
            else:
                logging.error("git repo exists and remote doesn't match - abort")
                logging.error(f"{self.git_url} != {git_remote}")
                exit(1)
        else:
            logging.info("clone")
            git.Repo.clone_from(self.git_url, self.local_dir, recursive=True)

    def pull(self):
        repo = git.Repo(self.local_dir)
        # reset
        repo.git.reset('--hard')
        o = repo.remotes.origin
        o.pull()

    def harden(self):
        logging.info(f"hardening {self}")

       # copy golden config
        shutil.copyfile('golden_config.tcl', os.path.join(self.local_dir, 'src', 'config.tcl'))

        cwd = os.getcwd()
        os.chdir(self.local_dir)

        # setup user config, not including python fails on github action
        if 'LOCAL' in os.environ:
            configure_cmd = './configure.py --create-user-config'
        else:
            configure_cmd = 'python ./configure.py --create-user-config'
        p = subprocess.run(configure_cmd, shell=True)
        if p.returncode != 0:
            logging.error(f"configure failed for {self}")
            exit(1)

        # requires PDK_ROOT, OPENLANE_ROOT & OPENLANE_IMAGE_NAME to be set in local environment
        harden_cmd = 'docker run --rm -v $OPENLANE_ROOT:/openlane -v $PDK_ROOT:$PDK_ROOT -v $(pwd):/work -e PDK_ROOT=$PDK_ROOT -u $(id -u $USER):$(id -g $USER) $OPENLANE_IMAGE_NAME /bin/bash -c "./flow.tcl -overwrite -design /work/src -run_path /work/runs -tag wokwi"'
        env = os.environ.copy()
        p = subprocess.run(harden_cmd, shell=True, env=env)
        if p.returncode != 0:
            logging.error(f"harden failed for {self}")
            exit(1)

        os.chdir(cwd)

    def __str__(self):
        if self.args.log_email:
            return f"[{self.index:03} : {self.email} : {self.git_url}]"
        else:
            return f"[{self.index:03} : {self.git_url}]"

    def fetch_gds(self):
        install_artifacts(self.git_url, self.local_dir)

    def get_macro_name(self):
        return self.top_module

    # instance name of the project, different for each id
    def get_macro_instance(self):
        return self.macro_instance

    # instance name of the scanchain module, different for each id
    def get_scanchain_instance(self):
        return self.scanchain_instance

    # unique id
    def get_index(self):
        return self.index

    # metrics
    def get_metrics_path(self):
        return os.path.join(self.local_dir, 'runs/wokwi/reports/metrics.csv')

    # name of the gds file
    def get_macro_gds_filename(self):
        return f"{self.top_module}.gds"

    def get_macro_lef_filename(self):
        return f"{self.top_module}.lef"

    # for GL sims & blackboxing
    def get_gl_verilog_filename(self):
        return f"{self.top_module}.v"

    # for simulations
    def get_top_verilog_filename(self):
        if self.is_hdl():
            # make sure it's unique & without leading directories
            # a few people use 'top.v', which is OK as long as the top module is called something more unique
            # but then it needs to be made unique so the source can be found
            filename = os.path.basename(self.top_verilog_filename)
            return f'{self.index :03}_{filename}'
        else:
            return self.top_verilog_filename

    def get_git_url(self):
        return self.git_url

    def copy_files_to_caravel(self):
        files = [
            (f"runs/wokwi/results/final/gds/{self.get_macro_gds_filename()}", f"gds/{self.get_macro_gds_filename()}"),
            (f"runs/wokwi/results/final/lef/{self.get_macro_lef_filename()}", f"lef/{self.get_macro_lef_filename()}"),
            (f"runs/wokwi/results/final/verilog/gl/{self.get_gl_verilog_filename()}", f"verilog/gl/{self.get_gl_verilog_filename()}"),
            (f"src/{self.top_verilog_filename}", f"verilog/rtl/{self.get_top_verilog_filename()}"),
            ]

        logging.debug("copying files into position")
        for from_path, to_path in files:
            from_path = os.path.join(self.local_dir, from_path)
            logging.debug(f"copy {from_path} to {to_path}")
            shutil.copyfile(from_path, to_path)


class CaravelConfig():

    def __init__(self, projects, num_projects):
        self.projects = projects
        self.num_projects = num_projects

    # create macro file & positions, power hooks
    def create_macro_config(self):
        # array size
        rows    = 18
        cols    = 14

        # start point (lower left)
        start_x = 50
        start_y = 95

        # module block sizes
        scanchain_w = 30
        scanchain_spc = 6
        module_w = 150
        module_h = 170

        # how much x & y space to leave between blocks
        space_x = 15
        space_y = 15

        # step sizes
        step_x  = scanchain_w + module_w + scanchain_spc + space_x
        step_y  = module_h + space_y

        logging.info(f"start_x {start_x} start_y {start_y} step_x {step_x} step_y {step_y }")

        num_macros_placed = 0

        # macro.cfg: where macros are placed
        logging.info("creating macro.cfg")
        with open("openlane/user_project_wrapper/macro.cfg", 'w') as fh:
            fh.write("scan_controller 100 100 N\n")
            for row in range(rows):
                if row % 2 == 0:
                    col_order = range(cols)
                    orientation = 'N'
                else:
                    # reverse odd rows to place instances in a zig zag pattern, shortening the scan chain wires
                    col_order = range(cols - 1, -1, -1)
                    orientation = 'S'
                for col in col_order:
                    # skip the space where the scan controller goes on the first row
                    if row == 0 and col <= 1:
                        continue

                    if num_macros_placed < self.num_projects:
                        if orientation == 'N':
                            # scanchain first
                            # co-ords are bottom left corner
                            macro_instance = self.projects[num_macros_placed].get_scanchain_instance()
                            instance = "{} {:<4} {:<4} {}\n".format(
                                macro_instance, start_x + col * step_x, start_y + row * step_y, orientation
                            )
                            fh.write(instance)

                            macro_instance = self.projects[num_macros_placed].get_macro_instance()
                            instance = "{} {:<4} {:<4} {}\n".format(
                                macro_instance, start_x + scanchain_spc + scanchain_w + col * step_x, start_y + row * step_y, orientation
                            )
                            fh.write(instance)
                        else:
                            # macro first
                            macro_instance = self.projects[num_macros_placed].get_macro_instance()
                            instance = "{} {:<4} {:<4} {}\n".format(
                                macro_instance, start_x + col * step_x, start_y + row * step_y, orientation
                            )
                            fh.write(instance)

                            macro_instance = self.projects[num_macros_placed].get_scanchain_instance()
                            instance = "{} {:<4} {:<4} {}\n".format(
                                macro_instance, start_x + module_w + scanchain_spc + col * step_x, start_y + row * step_y, orientation
                            )
                            fh.write(instance)

                    num_macros_placed += 1

        logging.info(f"total user macros placed: {num_macros_placed}")

        # macro_power.tcl: extra file for macro power hooks
        logging.info("creating macro_power.tcl")
        with open("openlane/user_project_wrapper/macro_power.tcl", 'w') as fh:
            fh.write('set ::env(FP_PDN_MACRO_HOOKS) "\\\n')
            fh.write("	")
            fh.write("scan_controller")
            fh.write(" vccd1 vssd1 vccd1 vssd1")
            fh.write(", \\\n")
            for i in range(self.num_projects):
                fh.write("	")
                fh.write(self.projects[i].get_scanchain_instance())
                fh.write(" vccd1 vssd1 vccd1 vssd1, \\\n")
                fh.write("	")
                fh.write(self.projects[i].get_macro_instance())
                fh.write(" vccd1 vssd1 vccd1 vssd1")
                if i != self.num_projects - 1:
                    fh.write(", \\\n")
            fh.write('"\n')

        # extra_lef_gds.tcl
        lefs = []
        gdss = []
        logging.info("creating extra_lef_gds.tcl")
        for project in self.projects:
            if not project.is_fill():
                lefs.append(project.get_macro_lef_filename())
                gdss.append(project.get_macro_gds_filename())

        with open("openlane/user_project_wrapper/extra_lef_gds.tcl", 'w') as fh:
            fh.write('set ::env(EXTRA_LEFS) "\\\n')
            fh.write("$script_dir/../../lef/scan_controller.lef \\\n")
            fh.write("$script_dir/../../lef/scanchain.lef \\\n")
            for i, lef in enumerate(lefs):
                fh.write("$script_dir/../../lef/{}".format(lef))
                if i != len(lefs) - 1:
                    fh.write(" \\\n")
                else:
                    fh.write('"\n')
            fh.write('set ::env(EXTRA_GDS_FILES) "\\\n')
            fh.write("$script_dir/../../gds/scan_controller.gds \\\n")
            fh.write("$script_dir/../../gds/scanchain.gds \\\n")
            for i, gds in enumerate(gdss):
                fh.write("$script_dir/../../gds/{}".format(gds))
                if i != len(gdss) - 1:
                    fh.write(" \\\n")
                else:
                    fh.write('"\n')

    # instantiate inside user_project_wrapper
    def instantiate(self):
        logging.info("instantiating designs in user_project_wrapper.v")

        # NOTE: The user project wrapper initially used vectored signals for clk,
        #       scan, and latch signals. However, this leads to atrocious sim
        #       performance, as any change within the vectored signal is
        #       interpreted as a trigger condition for re-evaluating logic (at
        #       least this is the case under Icarus and Verilator). Therefore
        #       single bit signals are used between stages to limit the impact
        #       of any wire changing.

        # Instance the scan controller
        body = [
            "",
            "wire sc_clk_out, sc_data_out, sc_latch_out, sc_scan_out;",
            "wire sc_clk_in,  sc_data_in;",
            "",
            f"scan_controller #(.NUM_DESIGNS({self.num_projects})) scan_controller (",
            "   .clk                    (wb_clk_i),",
            "   .reset                  (wb_rst_i),",
            "   .active_select          (io_in[20:12]),",
            "   .inputs                 (io_in[28:21]),",
            "   .outputs                (io_out[36:29]),",
            "   .ready                  (io_out[37]),",
            "   .slow_clk               (io_out[10]),",
            "   .set_clk_div            (io_in[11]),",
            "",
            "   .scan_clk_out           (sc_clk_out),",
            "   .scan_clk_in            (sc_clk_in),",
            "   .scan_data_out          (sc_data_out),",
            "   .scan_data_in           (sc_data_in),",
            "   .scan_select            (sc_scan_out),",
            "   .scan_latch_en          (sc_latch_out),",
            "",
            "   .la_scan_clk_in         (la_data_in[0]),",
            "   .la_scan_data_in        (la_data_in[1]),",
            "   .la_scan_data_out       (la_data_out[0]),",
            "   .la_scan_select         (la_data_in[2]),",
            "   .la_scan_latch_en       (la_data_in[3]),",
            "",
            "   .driver_sel             (io_in[9:8]),",
            "",
            "   .oeb                    (io_oeb)",
            ");",
        ]

        # Instance every design on the scan chain
        for idx in range(self.num_projects):
            # First design driven by scan controller, all others are chained
            pfx      = f"sw_{idx:03d}"
            prev_pfx = f"sw_{idx-1:03d}" if idx > 0 else "sc"
            # Pickup the Wokwi design ID and github URL for the project
            giturl = self.projects[idx].get_git_url()

            # Append the instance to the body
            body += [
                "",
                f"// [{idx:03d}] {giturl}",
                f"wire {pfx}_clk_out, {pfx}_data_out, {pfx}_scan_out, {pfx}_latch_out;",
                f"wire [7:0] {pfx}_module_data_in;",
                f"wire [7:0] {pfx}_module_data_out;",
                f"scanchain #(.NUM_IOS(8)) {self.projects[idx].get_scanchain_instance()} (",
                f"    .clk_in          ({prev_pfx}_clk_out),",
                f"    .data_in         ({prev_pfx}_data_out),",
                f"    .scan_select_in  ({prev_pfx}_scan_out),",
                f"    .latch_enable_in ({prev_pfx}_latch_out),",
                f"    .clk_out         ({pfx}_clk_out),",
                f"    .data_out        ({pfx}_data_out),",
                f"    .scan_select_out ({pfx}_scan_out),",
                f"    .latch_enable_out({pfx}_latch_out),",
                f"    .module_data_in  ({pfx}_module_data_in),",
                f"    .module_data_out ({pfx}_module_data_out)",
                ");"
            ]

            # Append the user module to the body
            body += [
                "",
                f"{self.projects[idx].get_macro_name()} {self.projects[idx].get_macro_instance()} (",
            ]
            if not self.projects[idx].art_only:
                body += [
                    f"    .io_in  ({pfx}_module_data_in),",
                    f"    .io_out ({pfx}_module_data_out)",
                ]
            body += [
                ");"

            ]

        # Link the final design back to the scan controller
        body += [
            "",
            "// Connect final signals back to the scan controller",
            f"assign sc_clk_in  = sw_{idx:03d}_clk_out;",
            f"assign sc_data_in = sw_{idx:03d}_data_out;",
            "",
            ""
        ]

        # Write to file
        with open('verilog/rtl/user_project_wrapper.v', 'w') as fh:
            # Insert the Caravel preamble
            with open("upw_pre.v", "r") as fh_pre:
                fh.write(fh_pre.read())
            # Indent, join, and insert the module instances
            fh.write("\n".join([("    " + x).rstrip() for x in body]))
            # Insert the Caravel postamble
            with open("upw_post.v", "r") as fh_post:
                fh.write(fh_post.read())

        # build the blackbox_project_includes.v file - used for blackboxing when building the GDS
        with open('verilog/blackbox_project_includes.v', 'w') as fh:
            fh.write('`include "rtl/scan_controller/scan_controller.v"\n')
            fh.write('`include "rtl/scanchain/scanchain.v"\n')
            for project in self.projects:
                if not project.is_fill():
                    fh.write(f'`include "gl/{project.get_gl_verilog_filename()}"\n')

        # build complete list of filenames for sim
        with open('verilog/includes/includes.rtl.caravel_user_project', 'w') as fh:
            fh.write('-v $(USER_PROJECT_VERILOG)/rtl/user_project_wrapper.v\n')
            fh.write('-v $(USER_PROJECT_VERILOG)/rtl/scan_controller/scan_controller.v\n')
            fh.write('-v $(USER_PROJECT_VERILOG)/rtl/scanchain/scanchain.v\n')
            fh.write('-v $(USER_PROJECT_VERILOG)/rtl/cells.v\n')
            for project in self.projects:
                if not project.is_fill():
                    fh.write(f'-v $(USER_PROJECT_VERILOG)/rtl/{project.get_top_verilog_filename()}\n')

        # build GL includes
        with open('verilog/includes/includes.gl.caravel_user_project', 'w') as fh:
            fh.write('-v $(USER_PROJECT_VERILOG)/gl/user_project_wrapper.v\n')
            fh.write('-v $(USER_PROJECT_VERILOG)/gl/scan_controller.v\n')
            fh.write('-v $(USER_PROJECT_VERILOG)/gl/scanchain.v\n')
            for project in self.projects:
                if not project.is_fill():
                    fh.write(f'-v $(USER_PROJECT_VERILOG)/gl/{project.get_gl_verilog_filename()}\n')

    def list(self):
        for project in self.projects:
            logging.info(project)


class Docs():

    def __init__(self, projects, args):
        self.projects = projects
        self.args = args

    # stuff related to docs
    def build_index(self):
        logging.info("building doc index")
        with open("README_init.md") as fh:
            readme = fh.read()
        with open("README.md", 'w') as fh:
            fh.write(readme)
            fh.write("| Index | Author | Title | Type | Git Repo |\n")
            fh.write("| ----- | ------ | ------| -----| ---------|\n")
            for project in self.projects:
                if not project.is_fill():
                    fh.write(project.get_index_row())

    def update_image(self):
        cmd = "klayout -l caravel.lyp gds/user_project_wrapper.gds -r dump_pic.rb -c klayoutrc"
        logging.info(cmd)
        os.system(cmd)

    # create a json file of all the project info, this is then used by tinytapeout.com to show projects
    def dump_json(self):
        designs = []
        for project in self.projects:
            design = project.get_project_yaml()
            designs.append(design)

        with open(args.dump_json, "w") as fh:
            fh.write(json.dumps(designs, indent=4))
        logging.info(f'wrote json to {args.dump_json}')

    def dump_markdown(self):

        with open("doc_header.md") as fh:
            doc_header = fh.read()

        with open("doc_template.md") as fh:
            doc_template = fh.read()

        with open("INFO.md") as fh:
            doc_info = fh.read()

        with open("VERIFICATION.md") as fh:
            doc_verification = fh.read()

        with open("CREDITS.md") as fh:
            doc_credits = fh.read()

        with open(args.dump_markdown, 'w') as fh:
            fh.write(doc_header)

            for project in self.projects:
                if project.is_fill():
                    continue
                yaml_data = project.get_project_doc_yaml()

                yaml_data['index'] = project.index

                logging.info(f"building datasheet for {project}")
                # handle pictures
                yaml_data['picture_link'] = ''
                if yaml_data['picture']:
                    # skip SVG for now, not supported by pandoc
                    picture_name = yaml_data['picture']
                    if 'svg' not in picture_name:
                        picture_filename = os.path.join(project.local_dir, picture_name)
                        yaml_data['picture_link'] = '![picture]({})'.format(picture_filename)

                # now build the doc & print it
                try:
                    doc = doc_template.format(**yaml_data)
                    fh.write(doc)
                    fh.write("\n\pagebreak\n")
                except IndexError:
                    logging.warning("missing pins in info.yaml, skipping")

            # ending
            fh.write(doc_info)
            fh.write("\n\pagebreak\n")
            fh.write(doc_verification)
            fh.write("\n\pagebreak\n")
            fh.write(doc_credits)

        logging.info(f'wrote markdown to {args.dump_markdown}')

        if args.dump_pdf:
            pdf_cmd = f'pandoc --toc --toc-depth 2 --pdf-engine=xelatex -i {args.dump_markdown} -o {args.dump_pdf}'
            logging.info(pdf_cmd)
            p = subprocess.run(pdf_cmd, shell=True)
            if p.returncode != 0:
                logging.error("pdf command failed")


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="TinyTapeout configuration and docs")

    parser.add_argument('--list', help="list projects", action='store_const', const=True)
    parser.add_argument('--clone-all', help="clone all projects", action="store_const", const=True)
    parser.add_argument('--update-all', help="git pull all projects", action="store_const", const=True)
    parser.add_argument('--fetch-gds', help="git fetch latest gds", action="store_const", const=True)
    parser.add_argument('--single', help="do action on single project", type=int, default=-1)
    parser.add_argument('--start-from', help="do action on projects after this index", type=int, default=0)
    parser.add_argument('--end-at', help="do action on projects before this index", type=int, default=0)
    parser.add_argument('--update-caravel', help='configure caravel for build', action='store_const', const=True)
    parser.add_argument('--harden', help="harden project", action="store_const", const=True)
    parser.add_argument('--limit-num-projects', help='only configure for the first n projects', type=int, default=DEFAULT_NUM_PROJECTS)
    parser.add_argument('--test', help='use test projects', action='store_const', const=True)
    parser.add_argument('--debug', help="debug logging", action="store_const", dest="loglevel", const=logging.DEBUG, default=logging.INFO)
    parser.add_argument('--log-email', help="print persons email in messages", action="store_const", const=True)
    parser.add_argument('--update-image', help="update the image", action="store_const", const=True)
    parser.add_argument('--dump-json', help="dump json of all project data to given file")
    parser.add_argument('--dump-markdown', help="dump markdown of all project data to given file")
    parser.add_argument('--dump-pdf', help="create pdf from the markdown")
    parser.add_argument('--metrics', help="print some project metrics", action="store_const", const=True)

    args = parser.parse_args()

    # setup log
    log_format = logging.Formatter('%(asctime)s - %(levelname)-8s - %(message)s')
    # configure the client logging
    log = logging.getLogger('')
    # has to be set to debug as is the root logger
    log.setLevel(args.loglevel)

    # create console handler and set level to info
    ch = logging.StreamHandler(sys.stdout)
    # create formatter for console
    ch.setFormatter(log_format)
    log.addHandler(ch)

    projects = Projects(args)

    docs = Docs(projects.projects, args=args)
    caravel = CaravelConfig(projects.projects, num_projects=args.limit_num_projects)

    if args.list:
        caravel.list()

    if args.update_caravel:
        caravel.create_macro_config()
        caravel.instantiate()
        if not args.test:
            docs.build_index()

    if args.update_image:
        docs.update_image()

    if args.dump_json:
        docs.dump_json()

    if args.dump_markdown:
        docs.dump_markdown()