#!/usr/bin/env python3
# Copyright 2020 The Skywater PDK Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import common
from common import mod_extract_from_path, convert_libname, version_extract_from_path, strip_strength
import subprocess
import re
import os
import argparse
import sys
import json
import datetime
import time
import glob
import csv
from stat import S_IREAD, S_IRGRP, S_IROTH, S_IWRITE
from collections import defaultdict
from multiprocessing import Pool
from pathlib import Path
try:
    from subprocess import DEVNULL
except ImportError:
    DEVNULL = open(os.devnull, 'wb')

DEBUG = 0

Copyright_header = ('-- Copyright 2020 The Skywater PDK Authors\n'
    '--\n'
    '-- Licensed under the Apache License, Version 2.0 (the "License");\n'
    '-- you may not use this file except in compliance with the License.\n'
    '-- You may obtain a copy of the License at\n'
    '--\n'
    '--     https://www.apache.org/licenses/LICENSE-2.0\n'
    '--\n'
    '-- Unless required by applicable law or agreed to in writing, software\n'
    '-- distributed under the License is distributed on an "AS IS" BASIS,\n'
    '-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n'
    '-- See the License for the specific language governing permissions and\n'
    '-- limitations under the License.\n'
    '\n')

skips = ["s8sram", "s8fmlt", "osu130", "openfpga", "sram",
        "QA_s8_drc" #issue #145
        ]

diff_path = ["s8iom0s8", "scs8hd", "scs8hdll", "scs8hs", "scs8hvl", "scs8lp", "scs8ls", "scs8ms"]

LAYER_NOTFOUND = 121
DESIGN_OR_PLUGIN_NOTFOUND = 120
WIDTH_NOTMATCH = 119
INSTANCE_NOTFOUND = 118
COMPONENT_NOTFOUND = 117
NET_NOTFOUND = 116
TERMINAL_NOTFOUND = 115
TECHNOLOGY_DATABASE_NOTFOUND = 114
SEGMENTATION_FAULT = 113
CANNOT_APPEND_TO_LIBRARY = 112
INVALID_DMSYSTEM = 111
INVALID_CELLVIEW = 110
OUTPUTFILE_EXISTS = 109
CELLVIEW_NOTFOUND = 108
MISSING_LIBRARY = 107
MISSING_FILE = 106
MULTIPLE_CELL_DEF = 105
TRANSLATION_ERROR = 104
SKIPPED = 103
COPY_ERROR = 102
PLUGIN_ERROR = 101
UNKNOWN_ERROR = 100
NO_ERROR = 99

errors_msg = {
          UNKNOWN_ERROR: "unknown error",
          PLUGIN_ERROR: "missing plugin",
          COPY_ERROR: "error while coping",
          SKIPPED: "skipped (due to filters)",
          MULTIPLE_CELL_DEF: "multiple cell definitions",
          MISSING_FILE: "file is missing",
          MISSING_LIBRARY: "missing library",
          CELLVIEW_NOTFOUND: "not found cellview",
          OUTPUTFILE_EXISTS: "output file exists",
          INVALID_CELLVIEW: "invalid cell view",
          TRANSLATION_ERROR: "error while translating",
          INVALID_DMSYSTEM: "invalid DMSystem",
          CANNOT_APPEND_TO_LIBRARY: "cannot append to library",
          SEGMENTATION_FAULT: "segmentation fault",
          TECHNOLOGY_DATABASE_NOTFOUND: "technology database not found",
          TERMINAL_NOTFOUND: "terminal not found in existing design",
          NET_NOTFOUND: "net not found in existing design",
          COMPONENT_NOTFOUND: "component not found in existing design",
          INSTANCE_NOTFOUND: "instance not found in existing design",
          WIDTH_NOTMATCH: "width of the terminal not match",
          DESIGN_OR_PLUGIN_NOTFOUND: "The design or pcell plugIn not found",
          LAYER_NOTFOUND: "layer not found in existing design",
          NO_ERROR: "succesful",
          SKIPPED: "skipped",
          }

TMP_OARESULT_DIR = "oaresults"
unpacked_oa2strm_libs = []

sourcetodests = defaultdict(list)

def remap_path(old_path, input_dir, output_dir):
    """
    >>> remap_path('/home/fullpath/Documents/skywater-src-nda/scs8hdll/V0.1.0/oa/scs8hdll_dv/hrpoly_1p41/symbolic/layout.oa', 'skywater-src-nda', 'tmp')
    'tmp/skywater-pdk/libraries/sky130_fd_sc_hdll/v0.1.0/cells/hrpoly_1p41/dv/oa/symbolic/sky130_fd_sc_hdll__hrpoly_1p41.layout.oa'
    >>> remap_path('/home/fullpath/Documents/skywater-src-nda/s8/V2.0.1/VirtuosoOA/libs/s8rf/Aura_blocking/layout/layout.oa', 'skywater-src-nda', 'tmp')
    'tmp/skywater-pdk/libraries/sky130_fd_pr_rf/v2.0.1/cells/aura_blocking/oa/layout/sky130_fd_pr_rf__aura_blocking.layout.oa'
    >>> remap_path('/home/fullpath/Documents/skywater-src-nda/s8/V2.0.1/VirtuosoOA/libs/s8rf/Aura_blocking/layout/master.tag', 'skywater-src-nda', 'tmp')
    'tmp/skywater-pdk/libraries/sky130_fd_pr_rf/v2.0.1/cells/aura_blocking/oa/layout/sky130_fd_pr_rf__aura_blocking.master.tag'
    >>> remap_path('/home/fullpath/Documents/skywater-src-nda/s8/V2.0.1/VirtuosoOA/libs/s8rf/Aura_blocking/data.dm', 'skywater-src-nda', 'tmp')
    'tmp/skywater-pdk/libraries/sky130_fd_pr_rf/v2.0.1/cells/aura_blocking/oa/Aura_blocking/sky130_fd_pr_rf__aura_blocking.data.dm'
    >>> remap_path('/home/fullpath/Documents/skywater-src-nda/s8/V2.0.0/VirtuosoOA/libs/tech/condiode_grid/ads/data.dm', 'skywater-src-nda', 'tmp')
    'tmp/skywater-pdk/libraries/sky130_fd_pr_base/V2.0.0/tech/condiode_grid/ads/data.dm'
    >>> remap_path('/home/fullpath/Documents/skywater-src-nda/s8/V2.0.1/VirtuosoOA/libs/s8rf/Aura_blocking/data.dm', 'skywater-src-nda', 'tmp')
    'tmp/skywater-pdk/libraries/sky130_fd_pr_rf/v2.0.1/cells/aura_blocking/oa/Aura_blocking/sky130_fd_pr_rf__aura_blocking.data.dm'
    """
    parent_dir = str(input_dir).split("/")[-1]
    output_dir = str(output_dir)
    lib, mod = mod_extract_from_path(old_path)
    new_lib = convert_libname(lib)
    file_name = os.path.basename(old_path)

    try:
        ver = "v" + ".".join([str(v) for v in version_extract_from_path(old_path)])
    except TypeError:
        ver = ""

    if lib != None and ver != "" and lib != "???":
        if mod == None:
            mod = lib
        rest, f_name= os.path.split(old_path)
        if mod not in old_path:
            # Letter case was changed
            temp = mod
            mod = mod.lower()
            f_name = f_name.lower().replace(temp, mod)

        clean_mod = strip_strength(mod)
        _, d_name = os.path.split(rest)
        old_path = f"{output_dir}/skywater-pdk/libraries/{lib}/{ver}/cells/{clean_mod}/{'dv/' if '_dv/' in old_path else''}oa/{d_name}/{lib}__{mod}.{file_name}"
    else:
        index = old_path.find(parent_dir) + len(parent_dir)
        old_path = output_dir +"/skywater-pdk/libraries"  + old_path[index:]
    old_path = old_path.replace("/s8iom0s8/", "/sky130_fd_io/")
    old_path = old_path.replace("/s8/", "/sky130_fd_pr_base/")
    old_path = old_path.replace("/VirtuosoOA/libs", "")

    if  lib != None and new_lib != None:
        new_path = old_path.replace(lib, new_lib)
    else:
        new_path = old_path
    return new_path

def filter(pn):
    for skip in skips:
        if skip in pn:
            return True
    return False

def check_status(output, call=None):
    call_to_key = {
            call_verilog2oa: "OAVLG",
            call_oa2verilog: "OAVLG",
            call_def2oa: "OALEFDEF",
            call_oa2def: "OALEFDEF",
            call_strm2oa: "OASTRM",
            call_oa2strm: "OASTRM",
            call_lef2oa: "OALEFDEF",
            call_oa2lef: "OALEFDEF",
            call_verilogAnnotate: "OAVLG"}
    if call is not None:
        key = call_to_key[call]
    else:
        key = "OA"
    error_code_start = len(key) + 2
    for s in output.split():
        if key in s:
            return s[error_code_start:-2]
    if key != "OA":
        return check_status(output)
    if "Segmentation fault" in output:
        return SEGMENTATION_FAULT
    return -100 # return unknown error code

def call_oa2verilog(libDefFile, lib, cell, view, verilog, pdk_home, sw_pdk_root):
    verilog += ".v"
    convert_cmd = ["bash", "oa_wrappers/oa2verilog_wrapper", f"{libDefFile}", f"{lib}", cell, f"{view}", f"{verilog}",
                   f"{pdk_home}", f"{sw_pdk_root}"]
    return subprocess.Popen(convert_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

def call_verilog2oa(libDefFile, lib, view, viewType, verilog, pdk_home, tmp_pdk_home, sw_pdk_root):
    verilog += ".v"
    convert_cmd = ["bash", "oa_wrappers/verilog2oa_wrapper", f"{libDefFile}", f"{lib}", f"{view}", f"{viewType}", f"{verilog}",
                   f"{pdk_home}", f"{tmp_pdk_home}", f"{sw_pdk_root}"]
    return subprocess.Popen(convert_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

def call_oa2def(libDefFile, lib, cell, view, def_file, pdk_home, sw_pdk_root):
    def_file += ".def"
    convert_cmd = ["bash", "oa_wrappers/oa2def_wrapper", f"{libDefFile}", f"{lib}", f"{cell}", f"{view}", f"{def_file}",
                   f"{pdk_home}", f"{sw_pdk_root}"]
    return subprocess.Popen(convert_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

def call_def2oa(libDefFile, lib, cell, view, def_file, pdk_home, tmp_pdk_home, sw_pdk_root):
    def_file += ".def"
    convert_cmd = ["bash", "oa_wrappers/def2oa_wrapper", f"{libDefFile}", f"{lib}", f"{cell}", f"{def_file}",
                   f"{pdk_home}", f"{tmp_pdk_home}", f"{sw_pdk_root}", f"{args.tmp}/{TMP_OARESULT_DIR}/s8/V1.0.0/VirtuosoOA/libs/technology_library/technology_library.layermap"]
    return subprocess.Popen(convert_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

def call_oa2lef(libDefFile, lib, cell, view, lef, pdk_home, sw_pdk_root):
    lef += ".lef"
    convert_cmd = ["bash", "oa_wrappers/oa2lef_wrapper", f"{libDefFile}", f"{lib}", f"{cell}", f"{lef}",
                   f"{pdk_home}", f"{sw_pdk_root}"]
    return subprocess.Popen(convert_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

def call_oa2strm(libDefFile, lib, cell, view, gds, pdk_home, sw_pdk_root):
    gds += ".gds"
    convert_cmd = ["bash", "oa_wrappers/oa2strm_wrapper", f"{libDefFile}", f"{lib}", f"{gds}",
                   f"{pdk_home}", f"{sw_pdk_root}"]
    return subprocess.Popen(convert_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

def call_lef2oa(libDefFile, lib, cell, view, lef, pdk_home, tmp_pdk_home, sw_pdk_root):
    lef += ".lef"
    convert_cmd = ["bash", "oa_wrappers/lef2oa_wrapper", f"{libDefFile}", f"{lib}", f"{lef}",
                   f"{pdk_home}", f"{tmp_pdk_home}", f"{sw_pdk_root}"]
    return subprocess.Popen(convert_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

def call_strm2oa(libDefFile, lib, cell, view, gds, pdk_home, tmp_pdk_home, sw_pdk_root):
    gds += ".gds"
    convert_cmd = ["bash", "oa_wrappers/strm2oa_wrapper", f"{libDefFile}", f"{lib}", f"{gds}",
                   f"{pdk_home}", f"{tmp_pdk_home}", f"{sw_pdk_root}"]
    return subprocess.Popen(convert_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

def call_verilogAnnotate(libDefFile, reflibs, refviews, cell, verilog, pdk_home, tmp_pdk_home, sw_pdk_root):
    verilog += ".v"
    convert_cmd = ["bash", "oa_wrappers/verilogAnnotate_wrapper", f"{libDefFile}", f"{reflibs}", f"{refviews}", f"{verilog}",
                   f"{pdk_home}", f"{tmp_pdk_home}", f"{sw_pdk_root}"]
    return subprocess.Popen(convert_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

def decode_rc(rc):
    if int(rc) == 50140:
        return DESIGN_OR_PLUGIN_NOTFOUND
    elif int(rc) == 50095:
        return NET_NOTFOUND
    elif int(rc) == 50081 or int(rc) == 50026:
        return INSTANCE_NOTFOUND
    elif int(rc) == 50079:
        return COMPONENT_NOTFOUND
    elif int(rc) == 50072:
        return TERMINAL_NOTFOUND
    elif int(rc) == 50024:
        return LAYER_NOTFOUND
    elif int(rc) == 50008:
        return TECHNOLOGY_DATABASE_NOTFOUND
    elif int(rc) == 40106:
        return MULTIPLE_CELL_DEF
    elif int(rc) == 40058:
        return OUTPUTFILE_EXISTS
    elif int(rc) == 40065:
        return WIDTH_NOTMATCH
    elif int(rc) == 40002:
        return TRANSLATION_ERROR
    elif int(rc) == 40001:
        return MISSING_FILE
    elif int(rc) == 10055 or int(rc) == 50138 or int(rc) == 10032 or int(rc) == 50103 or int(rc) == 50102:
        return PLUGIN_ERROR
    elif int(rc) == 10022:
        return MISSING_LIBRARY
    elif int(rc) == 10016:
        return MISSING_FILE
    elif int(rc) == 10010:
        return CANNOT_APPEND_TO_LIBRARY
    elif int(rc) == 3108:
        return INVALID_DMSYSTEM
    elif int(rc) == 1422:
        return OUTPUTFILE_EXISTS
    elif int(rc) == 1005:
        return CELLVIEW_NOTFOUND
    elif int(rc) == 1014:
        return INVALID_CELLVIEW
    elif int(rc) == SEGMENTATION_FAULT: #113
        return SEGMENTATION_FAULT
    elif int(rc) != 0:
        return UNKNOWN_ERROR
    return None

def copy_file(input_path, output_path):
    if not os.path.exists(input_path):
        return COPY_ERROR
    output_folder = os.path.dirname(output_path)
    os.makedirs(output_folder, exist_ok=True) #create output folder if not exists
    cp_cmd = ["cp", f"{input_path}", f"{output_path}"]
    proc = subprocess.run(cp_cmd)
    return proc.returncode

def move_file(input_path, output_path):
    if not os.path.exists(input_path):
        return COPY_ERROR
    output_folder = os.path.dirname(output_path)
    os.makedirs(output_folder, exist_ok=True) #create output folder if not exists
    mv_cmd = ["mv", f"{input_path}", f"{output_path}"]
    proc = subprocess.run(mv_cmd)
    return proc.returncode


def copy_folder(input_path, output_path):
    if not os.path.exists(input_path):
        return COPY_ERROR
    if os.path.exists(output_path):
        return 0
    output_folder = os.path.dirname(output_path)
    os.makedirs(output_folder, exist_ok=True) #create output folder if not exists
    cp_cmd = ["cp", "-r", f"{input_path}", f"{output_path}"]
    proc = subprocess.run(cp_cmd)
    return proc.returncode

def pack(pn, pack_function):
    if filter(pn):
        return None

    pn_copy = pn
    pn_version = pn_copy.replace(str(args.input), "").split("/")
    pn_version = f"{pn_version[1]}/{pn_version[2]}"
    skip = False
    for diff in diff_path:
        if diff in str(pn):
            skip = True

    if not skip:
        PDK_HOME = ENV_PDK_HOME =  f'{args.tmp}/{TMP_OARESULT_DIR}/{pn_version}/'
        TMP_PDK_HOME = f'.'
    else:
        ENV_PDK_HOME = f'{args.tmp}/{TMP_OARESULT_DIR}/s8/V1.3.0/'
        TMP_PDK_HOME = PDK_HOME = f'{args.tmp}/{TMP_OARESULT_DIR}/{pn_version}/oa'

    input_dir = os.path.dirname(pn)
    view = os.path.basename(input_dir)
    cell = os.path.basename(os.path.split(input_dir)[0])
    lib = os.path.basename(os.path.split(os.path.split(input_dir)[0])[0])
    if pack_function is call_strm2oa:
        lib_path = f"{pn_version}/{lib}/{cell}"
        global unpacked_oa2strm_libs
        if lib_path in unpacked_oa2strm_libs:
            unpacked_oa2strm_libs.remove(lib_path)
        else:
            return None
    output_path = remap_path(pn, args.input, args.tmp)
    output_file, ext = os.path.splitext(output_path)
    os.makedirs(os.path.dirname(output_path), exist_ok=True)
    viewType_dict = { "layout.oa": "maskLayout",
                 "sch.oa": "schematic",
                 "symbol.oa": "schematicSymbol",
                 "netlist.oa": "netlist"}
    filename = os.path.basename(pn)
    viewType = viewType_dict[filename]

    SW_PDK_ROOT = f"{str(args.tmp)}/{TMP_OARESULT_DIR}"

    #Change output dir
    if not skip:
        result_path = pn.replace(f'{args.input}/', f'{args.tmp}/{TMP_OARESULT_DIR}/')
        os.makedirs(os.path.dirname(result_path), exist_ok=True)
        os.makedirs(f"{PDK_HOME}/VirtuosoOA/libs/s8phirs_10r", exist_ok=True)
        os.makedirs(f"{PDK_HOME}/VirtuosoOA/libs/tech", exist_ok=True)
        os.makedirs(f"{PDK_HOME}/VirtuosoOA/libs/technology_library", exist_ok=True)
        os.makedirs(f"{PDK_HOME}/VirtuosoOA/libs/s8rf2", exist_ok=True)
        os.makedirs(f"{PDK_HOME}/VirtuosoOA/libs/s8rf2_dv", exist_ok=True)
    else:
        PDK_HOME = ENV_PDK_HOME =  f'{args.input}/{pn_version}/'
        output_dir = os.path.dirname(pn.replace(str(args.input), f"{str(args.tmp)}/{TMP_OARESULT_DIR}"))
        os.makedirs(f"{output_dir}", exist_ok=True)

    libDefFile_path = f"{args.tmp}/{TMP_OARESULT_DIR}/lib.defs"

    proc = None
    if pack_function is call_verilog2oa:
        proc = pack_function(libDefFile_path, lib, view, viewType, output_file, ENV_PDK_HOME, TMP_PDK_HOME, SW_PDK_ROOT)
    elif pack_function is not None:
        proc = pack_function(libDefFile_path, lib, view, cell, output_file, ENV_PDK_HOME, TMP_PDK_HOME, SW_PDK_ROOT)

    return proc


def unpack(pn, unpack_function):
    if filter(pn):
        return None

    pn_copy = pn
    pn_version = pn_copy.replace(str(args.input), "").split("/")
    pn_version = f"{pn_version[1]}/{pn_version[2]}"
    skip = False
    #For libraries that are not in s8 folder, we need to set explicite PDK_HOME to s8
    #It should not be used anyway, but oa tools needs to find all libraries
    #Even if it is not using them
    for diff in diff_path:
        if diff in str(pn):
            skip = True

    if not skip:
        PDK_HOME = ENV_PDK_HOME =  f'{args.input}/{pn_version}/'
    else:
        ENV_PDK_HOME = f'{args.input}/s8/V1.3.0/'

    SW_PDK_ROOT = args.input

    input_dir = os.path.dirname(pn)
    view = os.path.basename(input_dir)
    cell = os.path.basename(os.path.split(input_dir)[0])
    lib = os.path.basename(os.path.split(os.path.split(input_dir)[0])[0])
    if unpack_function is call_oa2strm:
        lib_path = f"{pn_version}/{lib}/{cell}"
        global unpacked_oa2strm_libs
        if lib_path in unpacked_oa2strm_libs:
            return None
        else:
            unpacked_oa2strm_libs += [lib_path]
    output_path = remap_path(pn, args.input, args.tmp)
    output_file, ext = os.path.splitext(output_path)
    os.makedirs(os.path.dirname(output_path), exist_ok=True)
    viewType_dict = { "layout.oa": "maskLayout",
                 "sch.oa": "schematic",
                 "symbol.oa": "schematicSymbol",
                 "netlist.oa": "netlist"}
    filename = os.path.basename(pn)
    viewType = viewType_dict[filename]

    libDefFile_path = f"{args.input}/lib.defs"

    proc = None
    if unpack_function is not None:
        proc = unpack_function(libDefFile_path, lib, cell, view, output_file, ENV_PDK_HOME, SW_PDK_ROOT)
    return proc


def copy_repacked_to_output():
    global sourcetodests
    oa_files = glob.glob(f"{args.tmp}/{TMP_OARESULT_DIR}/**/*.oa", recursive=True)
    dm_files = glob.glob(f"{args.tmp}/{TMP_OARESULT_DIR}/**/*.dm", recursive=True)
    print(f"Found {len(oa_files)} oa files in {args.tmp}/{TMP_OARESULT_DIR}!")
    print(f"Found {len(dm_files)} dm files in {args.tmp}/{TMP_OARESULT_DIR}!")
    for oa_file in oa_files:
        output_path = remap_path(oa_file, args.input, args.tmp)
        output_dir = os.path.dirname(oa_file)
        filename = os.path.basename(oa_file)
        master_tag_content = [Copyright_header, filename]
        with open(f"{output_dir}/master.tag", "w+") as master_tag:
            master_tag.writelines(master_tag_content)

        sourcetodests[f"{os.path.dirname(oa_file)}/master.tag"].append(f"{output_dir}/master.tag")

        if copy_file(oa_file, output_path):
            return {"return_code": COPY_ERROR, "path": oa_file}
        else:
            sourcetodests[oa_file].append(output_path)

    for dm_file in dm_files:
        output_path = remap_path(dm_file, args.input, args.tmp)
        if copy_file(dm_file, output_path):
            return {"return_code": COPY_ERROR, "path": dm_file}
        else:
            sourcetodests[dm_file].append(output_path)

def absolute_path(path):
    return path.resolve()

def make_readonly_oalib(base_path):
    oa_files = glob.glob(f"{base_path}/**/.oalib", recursive=True)
    for oa_file in oa_files:
        os.chmod(oa_file, S_IREAD|S_IRGRP|S_IROTH)
        if os.geteuid() == 0:
            os.system(f"chattr +i {oa_file}")

def make_readagain_oalib(base_path):
    oa_files = glob.glob(f"{base_path}/**/.oalib", recursive=True)
    for oa_file in oa_files:
        if os.geteuid() == 0:
            os.system(f"chattr -i {oa_file}")
        os.chmod(oa_file, S_IWRITE|S_IREAD|S_IRGRP|S_IROTH)

def copy_files(base_path, copy_path, pattern):
    files = glob.glob(f"{base_path}/**/{pattern}", recursive=True)
    for file_i in files:
        if copy_file(file_i, file_i.replace(str(base_path), str(copy_path))) == COPY_ERROR:
            sys.stderr.write(f"Error copying {file_i} to {file_i.replace(str(base_path), str(copy_path))}\n")
            sys.stderr.flush()


def copy_lib_defs():
    copy_dict = [{"from": f"{args.input}/lib.defs", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/lib.defs"},
            {"from": f"{args.input}/s8iom0s8/V0.2.0/oa/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/s8iom0s8/V0.2.0/oa/cds.lib"},
            {"from": f"{args.input}/s8iom0s8/V0.2.1/oa/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/s8iom0s8/V0.2.1/oa/cds.lib"},
            {"from": f"{args.input}/s8iom0s8/V0.0.0/oa/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/s8iom0s8/V0.0.0/oa/cds.lib"},
            {"from": f"{args.input}/s8iom0s8/V0.1.0/oa/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/s8iom0s8/V0.1.0/oa/cds.lib"},
            {"from": f"{args.input}/scs8hdll/V0.1.0/oa/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/scs8hdll/V0.1.0/oa/cds.lib"},
            {"from": f"{args.input}/scs8lp/V0.0.0/oa/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/scs8lp/V0.0.0/oa/cds.lib"},
            {"from": f"{args.input}/scs8ms/V0.0.0/oa/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/scs8ms/V0.0.0/oa/cds.lib"},
            {"from": f"{args.input}/scs8ls/V0.1.0/oa/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/scs8ls/V0.1.0/oa/cds.lib"},
            {"from": f"{args.input}/scs8hd/V0.0.1/oa/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/scs8hd/V0.0.1/oa/cds.lib"},
            {"from": f"{args.input}/scs8hs/V0.0.0/oa/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/scs8hs/V0.0.0/oa/cds.lib"},
            {"from": f"{args.input}/scs8hvl/V0.0.0/oa/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/scs8hvl/V0.0.0/oa/cds.lib"},
            {"from": f"{args.input}/scs8hvl/V0.0.1/oa/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/scs8hvl/V0.0.1/oa/cds.lib"},
            {"from": f"{args.input}/s8/V1.0.1/VirtuosoOA/examples/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/s8/V1.0.1/VirtuosoOA/examples/cds.lib"},
            {"from": f"{args.input}/s8/V1.1.0/VirtuosoOA/examples/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/s8/V1.1.0/VirtuosoOA/examples/cds.lib"},
            {"from": f"{args.input}/s8/V1.2.1/VirtuosoOA/examples/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/s8/V1.2.1/VirtuosoOA/examples/cds.lib"},
            {"from": f"{args.input}/s8/V1.2.0/VirtuosoOA/examples/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/s8/V1.2.0/VirtuosoOA/examples/cds.lib"},
            {"from": f"{args.input}/s8/V2.0.1/VirtuosoOA/examples/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/s8/V2.0.1/VirtuosoOA/examples/cds.lib"},
            {"from": f"{args.input}/s8/V1.0.0/VirtuosoOA/examples/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/s8/V1.0.0/VirtuosoOA/examples/cds.lib"},
            {"from": f"{args.input}/s8/V1.3.0/VirtuosoOA/examples/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/s8/V1.3.0/VirtuosoOA/examples/cds.lib"},
            {"from": f"{args.input}/s8/V2.0.0/VirtuosoOA/examples/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/s8/V2.0.0/VirtuosoOA/examples/cds.lib"},
            {"from": f"{args.input}/s8/V2.0.1/DRC/dev/libs/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/s8/V2.0.1/DRC/dev/libs/cds.lib"}
                ]

    for copy in copy_dict:
        if not os.path.exists(copy["to"]):
            if copy_file(copy["from"], copy["to"]) == COPY_ERROR:
                print(f"Error copying {copy['from']} to {copy['to']}! Exiting.")
                sys.exit(1)

def make_lib_dirs(base_path):
    folders = [
            f"{base_path}/{TMP_OARESULT_DIR}/s8iom0s8/V0.0.0/oa/s8io_m0s8",
            f"{base_path}/{TMP_OARESULT_DIR}/s8iom0s8/V0.1.0/oa/s8io_m0s8",
            f"{base_path}/{TMP_OARESULT_DIR}/s8iom0s8/V0.2.0/oa/s8io_m0s8",
            f"{base_path}/{TMP_OARESULT_DIR}/s8iom0s8/V0.2.1/oa/s8io_m0s8",
            f"{base_path}/{TMP_OARESULT_DIR}/scs8hdll/V0.1.0/oa/scs8_hdll",
            f"{base_path}/{TMP_OARESULT_DIR}/scs8lp/V0.0.0/oa/scs8_lp",
            f"{base_path}/{TMP_OARESULT_DIR}/scs8ms/V0.0.0/oa/scs8_ms",
            f"{base_path}/{TMP_OARESULT_DIR}/scs8ls/V0.1.0/oa/scs8_ls",
            f"{base_path}/{TMP_OARESULT_DIR}/scs8hd/V0.0.1/oa/scs8_hd",
            f"{base_path}/{TMP_OARESULT_DIR}/scs8hs/V0.0.0/oa/scs8_hs",
            f"{base_path}/{TMP_OARESULT_DIR}/scs8hvl/V0.0.0/oa/scs8_hvl",
            f"{base_path}/{TMP_OARESULT_DIR}/scs8hvl/V0.0.1/oa/scs8_hvl",
            f"{base_path}/{TMP_OARESULT_DIR}/s8/V2.0.1/DRC/dev/libs/QA_s8_drc",
            ]

    for folder in folders:
        os.makedirs(folder, exist_ok=True)

if __name__ == "__main__":
    import doctest
    fails, _ = doctest.testmod()
    if fails>0:
        exit(1)

    parser = argparse.ArgumentParser()
    parser.add_argument(
        "input",
        help="The path to the source directory/file",
        type=Path)
    parser.add_argument(
        "output",
        help="The path to the output directory",
        type=Path)
    parser.add_argument(
        "tmp",
        help="The path to the output directory",
        type=Path)
    parser.add_argument(
        "--sourcetodests",
        help="Mapping from source files to destination files",
        type=Path)
    parser.add_argument(
        "--cpus",
        help="Number od CPUs in the system",
        type=int)
    args = parser.parse_args()

    # Resolve paths to make sure we can find args.input/args.output/args.tmp in file paths
    args.input = args.input.resolve()
    args.output = args.output.resolve()
    args.tmp = args.tmp.resolve()

    # FIXME: set sys.argv as common.files echos back command-line args if sys.argv is > 1
    sys.argv = [sys.argv[0]]


    make_readonly_oalib(args.input) #OA tools is rewriting .oalib every time (even if file exists) so to run it in parallel, we are making them read-only
    copy_files(args.input, f"{args.tmp}/{TMP_OARESULT_DIR}", ".oalib")
    make_readonly_oalib(f"{args.tmp}/{TMP_OARESULT_DIR}")
    copy_lib_defs() #Copy lib.defs and cds.lib to output directory
    make_lib_dirs(args.tmp) #Make empty folders for libs defined in lib.defs

    #Copy technology database to output folder
    copy_files(args.input, f"{args.tmp}/{TMP_OARESULT_DIR}", "tech.db")
    copy_files(args.input, f"{args.tmp}/{TMP_OARESULT_DIR}", "*.layermap")

    os.system(f"find {args.tmp}/{TMP_OARESULT_DIR} -name cds.lib | xargs sed -i 's/\.\//$TMP_PDK_HOME\//g'") #Add $TMP_PDK_HOME to s8iom0s8, scs8hdll, scs8lp, scs8ms, scs8ls, scs8hd, scs8hs, scs8hvl to redirect output files

    #Not needed as QA_s8_drc is now skipped #145
    #copy_folder(str(args.input), f"{args.tmp}/skywater-src-nda")
    #args_input_old = str(args.input)
    #args.input = f"{args.tmp}/skywater-src-nda"

    # Move skywater-src-nda to tmp and rename files with #
    #for file_path in common.files([".oa"]):
    #    if "#" in str(file_path):
    #        copy_file(file_path, str(file_path).replace(args_input_old, f"{str(args.tmp)}/skywater-src-nda").replace("#", "_"))

    begin_time = datetime.datetime.now()

    repack_sequence = [
            {"unpack": call_oa2verilog, "pack": call_verilog2oa,       "name": "oa2verilog"},
            #{"unpack": call_oa2strm,    "pack": call_strm2oa,          "name": "oa2strm"},
            {"unpack": None,            "pack": call_verilogAnnotate,  "name": "verilogAnnotate"},
            {"unpack": call_oa2def,     "pack": call_def2oa,           "name": "oa2def"},
            {"unpack": call_oa2lef,     "pack": call_lef2oa,           "name": "oa2lef"}
            ]

    max_parallel_jobs = 2 * args.cpus
    current_parallel_jobs = max_parallel_jobs

    file_paths = [f for f in common.files([".oa"])]
    # fix paths to point to tmp dir
    #fixed_paths = []
    #for path in file_paths:
    #    fixed_paths.append(path.replace(args_input_old, f"{str(args.tmp)}/skywater-src-nda"))


    csv_status = defaultdict(list)

    def run_unpack_job(file_path, sequence):
        #if "#" in str(file_path):
        #    file_path = str(file_path).replace("#", "_")
        return unpack(file_path, sequence["unpack"])

    def run_pack_job(file_path, sequence):
        #if "#" in str(file_path):
        #    file_path = str(file_path).replace("#", "_")
        global TMP_OARESULT_DIR
        if sequence["pack"] is call_verilog2oa:
            TMP_OARESULT_DIR = f"oaresults{job}"
            copy_lib_defs()
            os.system(f"find {args.tmp}/{TMP_OARESULT_DIR} -name cds.lib | xargs sed -i 's/\.\//$TMP_PDK_HOME\//g'") #Add $TMP_PDK_HOME to s8iom0s8, scs8hdll, scs8lp, scs8ms, scs8ls, scs8hd, scs8hs, scs8hvl to redirect output files
        else:
            TMP_OARESULT_DIR = "oaresults"
        return pack(file_path, sequence["pack"])

    for sequence in repack_sequence:
        errored = defaultdict(list)
        skipped = defaultdict(list)
        unpack_job_success = []
        last_print = 0

        for oa_job in [run_unpack_job, run_pack_job]:
            print(f"Starting {sequence['name']} {'pack' if oa_job is run_pack_job else 'unpack'} job")
            files_left = len(file_paths)
            all_files = files_left
            if sequence['pack'] == call_strm2oa and oa_job is run_pack_job:
                current_parallel_jobs = 1
            else:
                current_parallel_jobs = max_parallel_jobs
            running_jobs = [None for i in range(current_parallel_jobs)]
            current_file = 0

            error_count = 0
            good_count = 0
            skipped_count = 0

            error_map = defaultdict(list)
            job_to_file_path = [None for i in range(current_parallel_jobs)]
            job_status = [None for i in range(current_parallel_jobs)]

            while files_left > 0:
                for job in range(len(running_jobs)):
                    if current_file % 1000 == 0 and last_print != current_file:
                        sys.stderr.write(f"[OA] [{time.strftime('%H:%M:%S')}] Processed {current_file} of {all_files}\n")
                        sys.stderr.flush()
                        last_print = current_file
                    if running_jobs[job] is not None and running_jobs[job].poll() is not None:
                        #Job finished
                        files_left -= 1
                        if running_jobs[job].returncode != 0:
                            error_count += 1
                            stderr = running_jobs[job].stderr.read().decode('utf-8')
                            job_status[job] = errors_msg[decode_rc(check_status(stderr, sequence["unpack"] if oa_job is run_unpack_job else sequence["pack"]))]
                            error_map[job_status[job]].append({file_path: stderr})
                            if decode_rc(check_status(stderr, sequence["unpack"] if oa_job is run_unpack_job else sequence["pack"])) == UNKNOWN_ERROR:
                                sys.stderr.write(f"Job: {job}: {stderr}\n")
                                sys.stderr.flush()
                        else:
                            good_count += 1
                            if oa_job is run_unpack_job:
                                unpack_job_success.append(job_to_file_path[job])

                        # For verilog2oa we creating separate folder for each thread (to make sure tmp folders do not have collision in names)
                        # after generating oa file, we need to move it to origin result folder
                        if oa_job is run_pack_job and sequence["pack"] is call_verilog2oa and running_jobs[job].returncode == 0: # only copy if we produced correct oa file and sequence is verilog2oa
                            file_path_copy = job_to_file_path[job]
                            move_file_path = file_path_copy.replace(str(args.input), f"{args.tmp}/oaresults{job}")
                            move_output_path = file_path_copy.replace(str(args.input), f"{args.tmp}/oaresults")
                            if move_file(move_file_path, move_output_path) == COPY_ERROR:
                                error_map[errors_msg[MISSING_FILE]].append({move_file_path: move_output_path})

                        if job_status[job] is None:
                            job_status[job] = errors_msg[NO_ERROR]
                        csv_status[job_to_file_path[job]].append({f'{sequence["name"]} unpack' if oa_job is run_unpack_job else f'{sequence["name"]} pack': job_status[job]})

                        # When job is done, set it to None
                        running_jobs[job] = None
                        job_status[job] = None

                    if running_jobs[job] is None:
                        #start a new job
                        if current_file < all_files:
                            file_path = file_paths[current_file]
                            job_to_file_path[job] = file_path
                            # For pack job, run only if unpack was successful, for verilogAnnotate, always run, as it does not have unpack job
                            if (oa_job is run_pack_job and file_path in unpack_job_success) or (oa_job is run_unpack_job) or (sequence['pack'] is call_verilogAnnotate):
                                running_jobs[job] = oa_job(file_path, sequence)

                            current_file += 1
                            # check if this job was skipped
                            if running_jobs[job] is None:
                                files_left -= 1
                                skipped_count += 1
                                job_status[job] = errors_msg[SKIPPED]
                                csv_status[job_to_file_path[job]].append({f'{sequence["name"]} unpack' if oa_job is run_unpack_job else f'{sequence["name"]} pack': job_status[job]})
                        else:
                            running_jobs[job] = None

            runname = sequence["name"]
            if oa_job == run_unpack_job:
                runname += "[unpack]"
            else:
                runname += "[pack]"

            log_name = f"errored-{runname.replace('[', '-').replace(']', '')}.txt"
            with open(log_name, mode='wt', encoding='utf-8') as myfile:
                json.dump(error_map, myfile, indent=2)

            for error_type, errors in error_map.items():
                sys.stderr.write(f" >>> OA |{runname}|: [{time.strftime('%H:%M:%S')}] Error type: [{error_type}] count: [{len(errors)}]\n")
                sys.stderr.flush()

            sys.stderr.write(" >>> OA |%s|: [%s] Error count = %d -- %.2f %%\n" % (runname,time.strftime('%H:%M:%S'),error_count,100.0 * error_count / all_files))
            sys.stderr.write(" >>> OA |%s|: [%s] Success count = %d -- %.2f %%\n" % (runname,time.strftime('%H:%M:%S'),good_count, 100.0 * good_count / all_files))
            sys.stderr.write(" >>> OA |%s|: [%s] Skipped count = %d -- %.2f %%\n" % (runname,time.strftime('%H:%M:%S'),skipped_count, 100.0 * skipped_count / all_files))
            sys.stderr.flush()

        with open(f'errored-{sequence["name"]}.txt', mode='wt', encoding='utf-8') as myfile:
            json.dump(errored, myfile, indent=2)

        with open(f'skipped-{sequence["name"]}.txt', mode='wt', encoding='utf-8') as myfile:
            json.dump(skipped, myfile, indent=2)

    with open('file_status.csv', 'w', newline='') as file:
        writer = csv.writer(file)
        writer.writerow(["File", "oa2verilog", "verilog2oa", "verilogAnnotate", "oa2def", "def2oa", "oa2lef", "lef2oa"])
        for file_str in csv_status:
            row = [file_str.replace(str(args.input), ""), csv_status[file_str][0]["oa2verilog unpack"], csv_status[file_str][1]["oa2verilog pack"], csv_status[file_str][3]["verilogAnnotate pack"], csv_status[file_str][4]["oa2def unpack"], csv_status[file_str][5]["oa2def pack"], csv_status[file_str][6]["oa2lef unpack"], csv_status[file_str][7]["oa2lef pack"]]
            writer.writerow(row)

    TMP_OARESULT_DIR = "oaresults"
    copy_repacked_to_output()

    with open(args.sourcetodests, 'w') as srctodst:
        json.dump(sourcetodests, srctodst, indent=2)

    print("Done!")
    print(f"Took: {datetime.datetime.now() - begin_time}")
    make_readagain_oalib(args.input)
    make_readagain_oalib(f"{args.tmp}/{TMP_OARESULT_DIR}")