| #!/usr/bin/env python3 |
| # Copyright 2020 The Skywater PDK Authors |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # https://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| import common |
| from common import mod_extract_from_path, convert_libname, version_extract_from_path, strip_strength |
| import subprocess |
| import re |
| import os |
| import argparse |
| import sys |
| import json |
| import datetime |
| import time |
| import glob |
| import csv |
| from stat import S_IREAD, S_IRGRP, S_IROTH, S_IWRITE |
| from collections import defaultdict |
| from multiprocessing import Pool |
| from pathlib import Path |
| try: |
| from subprocess import DEVNULL |
| except ImportError: |
| DEVNULL = open(os.devnull, 'wb') |
| |
| DEBUG = 0 |
| |
| Copyright_header = ('-- Copyright 2020 The Skywater PDK Authors\n' |
| '--\n' |
| '-- Licensed under the Apache License, Version 2.0 (the "License");\n' |
| '-- you may not use this file except in compliance with the License.\n' |
| '-- You may obtain a copy of the License at\n' |
| '--\n' |
| '-- https://www.apache.org/licenses/LICENSE-2.0\n' |
| '--\n' |
| '-- Unless required by applicable law or agreed to in writing, software\n' |
| '-- distributed under the License is distributed on an "AS IS" BASIS,\n' |
| '-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n' |
| '-- See the License for the specific language governing permissions and\n' |
| '-- limitations under the License.\n' |
| '\n') |
| |
| skips = ["s8sram", "s8fmlt", "osu130", "openfpga", "sram", |
| "QA_s8_drc" #issue #145 |
| ] |
| |
| diff_path = ["s8iom0s8", "scs8hd", "scs8hdll", "scs8hs", "scs8hvl", "scs8lp", "scs8ls", "scs8ms"] |
| |
| LAYER_NOTFOUND = 121 |
| DESIGN_OR_PLUGIN_NOTFOUND = 120 |
| WIDTH_NOTMATCH = 119 |
| INSTANCE_NOTFOUND = 118 |
| COMPONENT_NOTFOUND = 117 |
| NET_NOTFOUND = 116 |
| TERMINAL_NOTFOUND = 115 |
| TECHNOLOGY_DATABASE_NOTFOUND = 114 |
| SEGMENTATION_FAULT = 113 |
| CANNOT_APPEND_TO_LIBRARY = 112 |
| INVALID_DMSYSTEM = 111 |
| INVALID_CELLVIEW = 110 |
| OUTPUTFILE_EXISTS = 109 |
| CELLVIEW_NOTFOUND = 108 |
| MISSING_LIBRARY = 107 |
| MISSING_FILE = 106 |
| MULTIPLE_CELL_DEF = 105 |
| TRANSLATION_ERROR = 104 |
| SKIPPED = 103 |
| COPY_ERROR = 102 |
| PLUGIN_ERROR = 101 |
| UNKNOWN_ERROR = 100 |
| NO_ERROR = 99 |
| |
| errors_msg = { |
| UNKNOWN_ERROR: "unknown error", |
| PLUGIN_ERROR: "missing plugin", |
| COPY_ERROR: "error while coping", |
| SKIPPED: "skipped (due to filters)", |
| MULTIPLE_CELL_DEF: "multiple cell definitions", |
| MISSING_FILE: "file is missing", |
| MISSING_LIBRARY: "missing library", |
| CELLVIEW_NOTFOUND: "not found cellview", |
| OUTPUTFILE_EXISTS: "output file exists", |
| INVALID_CELLVIEW: "invalid cell view", |
| TRANSLATION_ERROR: "error while translating", |
| INVALID_DMSYSTEM: "invalid DMSystem", |
| CANNOT_APPEND_TO_LIBRARY: "cannot append to library", |
| SEGMENTATION_FAULT: "segmentation fault", |
| TECHNOLOGY_DATABASE_NOTFOUND: "technology database not found", |
| TERMINAL_NOTFOUND: "terminal not found in existing design", |
| NET_NOTFOUND: "net not found in existing design", |
| COMPONENT_NOTFOUND: "component not found in existing design", |
| INSTANCE_NOTFOUND: "instance not found in existing design", |
| WIDTH_NOTMATCH: "width of the terminal not match", |
| DESIGN_OR_PLUGIN_NOTFOUND: "The design or pcell plugIn not found", |
| LAYER_NOTFOUND: "layer not found in existing design", |
| NO_ERROR: "succesful", |
| SKIPPED: "skipped", |
| } |
| |
| TMP_OARESULT_DIR = "oaresults" |
| unpacked_oa2strm_libs = [] |
| |
| sourcetodests = defaultdict(list) |
| |
| def remap_path(old_path, input_dir, output_dir): |
| """ |
| >>> remap_path('/home/fullpath/Documents/skywater-src-nda/scs8hdll/V0.1.0/oa/scs8hdll_dv/hrpoly_1p41/symbolic/layout.oa', 'skywater-src-nda', 'tmp') |
| 'tmp/skywater-pdk/libraries/sky130_fd_sc_hdll/v0.1.0/cells/hrpoly_1p41/dv/oa/symbolic/sky130_fd_sc_hdll__hrpoly_1p41.layout.oa' |
| >>> remap_path('/home/fullpath/Documents/skywater-src-nda/s8/V2.0.1/VirtuosoOA/libs/s8rf/Aura_blocking/layout/layout.oa', 'skywater-src-nda', 'tmp') |
| 'tmp/skywater-pdk/libraries/sky130_fd_pr_rf/v2.0.1/cells/aura_blocking/oa/layout/sky130_fd_pr_rf__aura_blocking.layout.oa' |
| >>> remap_path('/home/fullpath/Documents/skywater-src-nda/s8/V2.0.1/VirtuosoOA/libs/s8rf/Aura_blocking/layout/master.tag', 'skywater-src-nda', 'tmp') |
| 'tmp/skywater-pdk/libraries/sky130_fd_pr_rf/v2.0.1/cells/aura_blocking/oa/layout/sky130_fd_pr_rf__aura_blocking.master.tag' |
| >>> remap_path('/home/fullpath/Documents/skywater-src-nda/s8/V2.0.1/VirtuosoOA/libs/s8rf/Aura_blocking/data.dm', 'skywater-src-nda', 'tmp') |
| 'tmp/skywater-pdk/libraries/sky130_fd_pr_rf/v2.0.1/cells/aura_blocking/oa/Aura_blocking/sky130_fd_pr_rf__aura_blocking.data.dm' |
| >>> remap_path('/home/fullpath/Documents/skywater-src-nda/s8/V2.0.0/VirtuosoOA/libs/tech/condiode_grid/ads/data.dm', 'skywater-src-nda', 'tmp') |
| 'tmp/skywater-pdk/libraries/sky130_fd_pr_base/V2.0.0/tech/condiode_grid/ads/data.dm' |
| >>> remap_path('/home/fullpath/Documents/skywater-src-nda/s8/V2.0.1/VirtuosoOA/libs/s8rf/Aura_blocking/data.dm', 'skywater-src-nda', 'tmp') |
| 'tmp/skywater-pdk/libraries/sky130_fd_pr_rf/v2.0.1/cells/aura_blocking/oa/Aura_blocking/sky130_fd_pr_rf__aura_blocking.data.dm' |
| """ |
| parent_dir = str(input_dir).split("/")[-1] |
| output_dir = str(output_dir) |
| lib, mod = mod_extract_from_path(old_path) |
| new_lib = convert_libname(lib) |
| file_name = os.path.basename(old_path) |
| |
| try: |
| ver = "v" + ".".join([str(v) for v in version_extract_from_path(old_path)]) |
| except TypeError: |
| ver = "" |
| |
| if lib != None and ver != "" and lib != "???": |
| if mod == None: |
| mod = lib |
| rest, f_name= os.path.split(old_path) |
| if mod not in old_path: |
| # Letter case was changed |
| temp = mod |
| mod = mod.lower() |
| f_name = f_name.lower().replace(temp, mod) |
| |
| clean_mod = strip_strength(mod) |
| _, d_name = os.path.split(rest) |
| old_path = f"{output_dir}/skywater-pdk/libraries/{lib}/{ver}/cells/{clean_mod}/{'dv/' if '_dv/' in old_path else''}oa/{d_name}/{lib}__{mod}.{file_name}" |
| else: |
| index = old_path.find(parent_dir) + len(parent_dir) |
| old_path = output_dir +"/skywater-pdk/libraries" + old_path[index:] |
| old_path = old_path.replace("/s8iom0s8/", "/sky130_fd_io/") |
| old_path = old_path.replace("/s8/", "/sky130_fd_pr_base/") |
| old_path = old_path.replace("/VirtuosoOA/libs", "") |
| |
| if lib != None and new_lib != None: |
| new_path = old_path.replace(lib, new_lib) |
| else: |
| new_path = old_path |
| return new_path |
| |
| def filter(pn): |
| for skip in skips: |
| if skip in pn: |
| return True |
| return False |
| |
| def check_status(output, call=None): |
| call_to_key = { |
| call_verilog2oa: "OAVLG", |
| call_oa2verilog: "OAVLG", |
| call_def2oa: "OALEFDEF", |
| call_oa2def: "OALEFDEF", |
| call_strm2oa: "OASTRM", |
| call_oa2strm: "OASTRM", |
| call_lef2oa: "OALEFDEF", |
| call_oa2lef: "OALEFDEF", |
| call_verilogAnnotate: "OAVLG"} |
| if call is not None: |
| key = call_to_key[call] |
| else: |
| key = "OA" |
| error_code_start = len(key) + 2 |
| for s in output.split(): |
| if key in s: |
| return s[error_code_start:-2] |
| if key != "OA": |
| return check_status(output) |
| if "Segmentation fault" in output: |
| return SEGMENTATION_FAULT |
| return -100 # return unknown error code |
| |
| def call_oa2verilog(libDefFile, lib, cell, view, verilog, pdk_home, sw_pdk_root): |
| verilog += ".v" |
| convert_cmd = ["bash", "oa_wrappers/oa2verilog_wrapper", f"{libDefFile}", f"{lib}", cell, f"{view}", f"{verilog}", |
| f"{pdk_home}", f"{sw_pdk_root}"] |
| return subprocess.Popen(convert_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
| |
| def call_verilog2oa(libDefFile, lib, view, viewType, verilog, pdk_home, tmp_pdk_home, sw_pdk_root): |
| verilog += ".v" |
| convert_cmd = ["bash", "oa_wrappers/verilog2oa_wrapper", f"{libDefFile}", f"{lib}", f"{view}", f"{viewType}", f"{verilog}", |
| f"{pdk_home}", f"{tmp_pdk_home}", f"{sw_pdk_root}"] |
| return subprocess.Popen(convert_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
| |
| def call_oa2def(libDefFile, lib, cell, view, def_file, pdk_home, sw_pdk_root): |
| def_file += ".def" |
| convert_cmd = ["bash", "oa_wrappers/oa2def_wrapper", f"{libDefFile}", f"{lib}", f"{cell}", f"{view}", f"{def_file}", |
| f"{pdk_home}", f"{sw_pdk_root}"] |
| return subprocess.Popen(convert_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
| |
| def call_def2oa(libDefFile, lib, cell, view, def_file, pdk_home, tmp_pdk_home, sw_pdk_root): |
| def_file += ".def" |
| convert_cmd = ["bash", "oa_wrappers/def2oa_wrapper", f"{libDefFile}", f"{lib}", f"{cell}", f"{def_file}", |
| f"{pdk_home}", f"{tmp_pdk_home}", f"{sw_pdk_root}", f"{args.tmp}/{TMP_OARESULT_DIR}/s8/V1.0.0/VirtuosoOA/libs/technology_library/technology_library.layermap"] |
| return subprocess.Popen(convert_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
| |
| def call_oa2lef(libDefFile, lib, cell, view, lef, pdk_home, sw_pdk_root): |
| lef += ".lef" |
| convert_cmd = ["bash", "oa_wrappers/oa2lef_wrapper", f"{libDefFile}", f"{lib}", f"{cell}", f"{lef}", |
| f"{pdk_home}", f"{sw_pdk_root}"] |
| return subprocess.Popen(convert_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
| |
| def call_oa2strm(libDefFile, lib, cell, view, gds, pdk_home, sw_pdk_root): |
| gds += ".gds" |
| convert_cmd = ["bash", "oa_wrappers/oa2strm_wrapper", f"{libDefFile}", f"{lib}", f"{gds}", |
| f"{pdk_home}", f"{sw_pdk_root}"] |
| return subprocess.Popen(convert_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
| |
| def call_lef2oa(libDefFile, lib, cell, view, lef, pdk_home, tmp_pdk_home, sw_pdk_root): |
| lef += ".lef" |
| convert_cmd = ["bash", "oa_wrappers/lef2oa_wrapper", f"{libDefFile}", f"{lib}", f"{lef}", |
| f"{pdk_home}", f"{tmp_pdk_home}", f"{sw_pdk_root}"] |
| return subprocess.Popen(convert_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
| |
| def call_strm2oa(libDefFile, lib, cell, view, gds, pdk_home, tmp_pdk_home, sw_pdk_root): |
| gds += ".gds" |
| convert_cmd = ["bash", "oa_wrappers/strm2oa_wrapper", f"{libDefFile}", f"{lib}", f"{gds}", |
| f"{pdk_home}", f"{tmp_pdk_home}", f"{sw_pdk_root}"] |
| return subprocess.Popen(convert_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
| |
| def call_verilogAnnotate(libDefFile, reflibs, refviews, cell, verilog, pdk_home, tmp_pdk_home, sw_pdk_root): |
| verilog += ".v" |
| convert_cmd = ["bash", "oa_wrappers/verilogAnnotate_wrapper", f"{libDefFile}", f"{reflibs}", f"{refviews}", f"{verilog}", |
| f"{pdk_home}", f"{tmp_pdk_home}", f"{sw_pdk_root}"] |
| return subprocess.Popen(convert_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
| |
| def decode_rc(rc): |
| if int(rc) == 50140: |
| return DESIGN_OR_PLUGIN_NOTFOUND |
| elif int(rc) == 50095: |
| return NET_NOTFOUND |
| elif int(rc) == 50081 or int(rc) == 50026: |
| return INSTANCE_NOTFOUND |
| elif int(rc) == 50079: |
| return COMPONENT_NOTFOUND |
| elif int(rc) == 50072: |
| return TERMINAL_NOTFOUND |
| elif int(rc) == 50024: |
| return LAYER_NOTFOUND |
| elif int(rc) == 50008: |
| return TECHNOLOGY_DATABASE_NOTFOUND |
| elif int(rc) == 40106: |
| return MULTIPLE_CELL_DEF |
| elif int(rc) == 40058: |
| return OUTPUTFILE_EXISTS |
| elif int(rc) == 40065: |
| return WIDTH_NOTMATCH |
| elif int(rc) == 40002: |
| return TRANSLATION_ERROR |
| elif int(rc) == 40001: |
| return MISSING_FILE |
| elif int(rc) == 10055 or int(rc) == 50138 or int(rc) == 10032 or int(rc) == 50103 or int(rc) == 50102: |
| return PLUGIN_ERROR |
| elif int(rc) == 10022: |
| return MISSING_LIBRARY |
| elif int(rc) == 10016: |
| return MISSING_FILE |
| elif int(rc) == 10010: |
| return CANNOT_APPEND_TO_LIBRARY |
| elif int(rc) == 3108: |
| return INVALID_DMSYSTEM |
| elif int(rc) == 1422: |
| return OUTPUTFILE_EXISTS |
| elif int(rc) == 1005: |
| return CELLVIEW_NOTFOUND |
| elif int(rc) == 1014: |
| return INVALID_CELLVIEW |
| elif int(rc) == SEGMENTATION_FAULT: #113 |
| return SEGMENTATION_FAULT |
| elif int(rc) != 0: |
| return UNKNOWN_ERROR |
| return None |
| |
| def copy_file(input_path, output_path): |
| if not os.path.exists(input_path): |
| return COPY_ERROR |
| output_folder = os.path.dirname(output_path) |
| os.makedirs(output_folder, exist_ok=True) #create output folder if not exists |
| cp_cmd = ["cp", f"{input_path}", f"{output_path}"] |
| proc = subprocess.run(cp_cmd) |
| return proc.returncode |
| |
| def move_file(input_path, output_path): |
| if not os.path.exists(input_path): |
| return COPY_ERROR |
| output_folder = os.path.dirname(output_path) |
| os.makedirs(output_folder, exist_ok=True) #create output folder if not exists |
| mv_cmd = ["mv", f"{input_path}", f"{output_path}"] |
| proc = subprocess.run(mv_cmd) |
| return proc.returncode |
| |
| |
| def copy_folder(input_path, output_path): |
| if not os.path.exists(input_path): |
| return COPY_ERROR |
| if os.path.exists(output_path): |
| return 0 |
| output_folder = os.path.dirname(output_path) |
| os.makedirs(output_folder, exist_ok=True) #create output folder if not exists |
| cp_cmd = ["cp", "-r", f"{input_path}", f"{output_path}"] |
| proc = subprocess.run(cp_cmd) |
| return proc.returncode |
| |
| def pack(pn, pack_function): |
| if filter(pn): |
| return None |
| |
| pn_copy = pn |
| pn_version = pn_copy.replace(str(args.input), "").split("/") |
| pn_version = f"{pn_version[1]}/{pn_version[2]}" |
| skip = False |
| for diff in diff_path: |
| if diff in str(pn): |
| skip = True |
| |
| if not skip: |
| PDK_HOME = ENV_PDK_HOME = f'{args.tmp}/{TMP_OARESULT_DIR}/{pn_version}/' |
| TMP_PDK_HOME = f'.' |
| else: |
| ENV_PDK_HOME = f'{args.tmp}/{TMP_OARESULT_DIR}/s8/V1.3.0/' |
| TMP_PDK_HOME = PDK_HOME = f'{args.tmp}/{TMP_OARESULT_DIR}/{pn_version}/oa' |
| |
| input_dir = os.path.dirname(pn) |
| view = os.path.basename(input_dir) |
| cell = os.path.basename(os.path.split(input_dir)[0]) |
| lib = os.path.basename(os.path.split(os.path.split(input_dir)[0])[0]) |
| if pack_function is call_strm2oa: |
| lib_path = f"{pn_version}/{lib}/{cell}" |
| global unpacked_oa2strm_libs |
| if lib_path in unpacked_oa2strm_libs: |
| unpacked_oa2strm_libs.remove(lib_path) |
| else: |
| return None |
| output_path = remap_path(pn, args.input, args.tmp) |
| output_file, ext = os.path.splitext(output_path) |
| os.makedirs(os.path.dirname(output_path), exist_ok=True) |
| viewType_dict = { "layout.oa": "maskLayout", |
| "sch.oa": "schematic", |
| "symbol.oa": "schematicSymbol", |
| "netlist.oa": "netlist"} |
| filename = os.path.basename(pn) |
| viewType = viewType_dict[filename] |
| |
| SW_PDK_ROOT = f"{str(args.tmp)}/{TMP_OARESULT_DIR}" |
| |
| #Change output dir |
| if not skip: |
| result_path = pn.replace(f'{args.input}/', f'{args.tmp}/{TMP_OARESULT_DIR}/') |
| os.makedirs(os.path.dirname(result_path), exist_ok=True) |
| os.makedirs(f"{PDK_HOME}/VirtuosoOA/libs/s8phirs_10r", exist_ok=True) |
| os.makedirs(f"{PDK_HOME}/VirtuosoOA/libs/tech", exist_ok=True) |
| os.makedirs(f"{PDK_HOME}/VirtuosoOA/libs/technology_library", exist_ok=True) |
| os.makedirs(f"{PDK_HOME}/VirtuosoOA/libs/s8rf2", exist_ok=True) |
| os.makedirs(f"{PDK_HOME}/VirtuosoOA/libs/s8rf2_dv", exist_ok=True) |
| else: |
| PDK_HOME = ENV_PDK_HOME = f'{args.input}/{pn_version}/' |
| output_dir = os.path.dirname(pn.replace(str(args.input), f"{str(args.tmp)}/{TMP_OARESULT_DIR}")) |
| os.makedirs(f"{output_dir}", exist_ok=True) |
| |
| libDefFile_path = f"{args.tmp}/{TMP_OARESULT_DIR}/lib.defs" |
| |
| proc = None |
| if pack_function is call_verilog2oa: |
| proc = pack_function(libDefFile_path, lib, view, viewType, output_file, ENV_PDK_HOME, TMP_PDK_HOME, SW_PDK_ROOT) |
| elif pack_function is not None: |
| proc = pack_function(libDefFile_path, lib, view, cell, output_file, ENV_PDK_HOME, TMP_PDK_HOME, SW_PDK_ROOT) |
| |
| return proc |
| |
| |
| def unpack(pn, unpack_function): |
| if filter(pn): |
| return None |
| |
| pn_copy = pn |
| pn_version = pn_copy.replace(str(args.input), "").split("/") |
| pn_version = f"{pn_version[1]}/{pn_version[2]}" |
| skip = False |
| #For libraries that are not in s8 folder, we need to set explicite PDK_HOME to s8 |
| #It should not be used anyway, but oa tools needs to find all libraries |
| #Even if it is not using them |
| for diff in diff_path: |
| if diff in str(pn): |
| skip = True |
| |
| if not skip: |
| PDK_HOME = ENV_PDK_HOME = f'{args.input}/{pn_version}/' |
| else: |
| ENV_PDK_HOME = f'{args.input}/s8/V1.3.0/' |
| |
| SW_PDK_ROOT = args.input |
| |
| input_dir = os.path.dirname(pn) |
| view = os.path.basename(input_dir) |
| cell = os.path.basename(os.path.split(input_dir)[0]) |
| lib = os.path.basename(os.path.split(os.path.split(input_dir)[0])[0]) |
| if unpack_function is call_oa2strm: |
| lib_path = f"{pn_version}/{lib}/{cell}" |
| global unpacked_oa2strm_libs |
| if lib_path in unpacked_oa2strm_libs: |
| return None |
| else: |
| unpacked_oa2strm_libs += [lib_path] |
| output_path = remap_path(pn, args.input, args.tmp) |
| output_file, ext = os.path.splitext(output_path) |
| os.makedirs(os.path.dirname(output_path), exist_ok=True) |
| viewType_dict = { "layout.oa": "maskLayout", |
| "sch.oa": "schematic", |
| "symbol.oa": "schematicSymbol", |
| "netlist.oa": "netlist"} |
| filename = os.path.basename(pn) |
| viewType = viewType_dict[filename] |
| |
| libDefFile_path = f"{args.input}/lib.defs" |
| |
| proc = None |
| if unpack_function is not None: |
| proc = unpack_function(libDefFile_path, lib, cell, view, output_file, ENV_PDK_HOME, SW_PDK_ROOT) |
| return proc |
| |
| |
| def copy_repacked_to_output(): |
| global sourcetodests |
| oa_files = glob.glob(f"{args.tmp}/{TMP_OARESULT_DIR}/**/*.oa", recursive=True) |
| dm_files = glob.glob(f"{args.tmp}/{TMP_OARESULT_DIR}/**/*.dm", recursive=True) |
| print(f"Found {len(oa_files)} oa files in {args.tmp}/{TMP_OARESULT_DIR}!") |
| print(f"Found {len(dm_files)} dm files in {args.tmp}/{TMP_OARESULT_DIR}!") |
| for oa_file in oa_files: |
| output_path = remap_path(oa_file, args.input, args.tmp) |
| output_dir = os.path.dirname(oa_file) |
| filename = os.path.basename(oa_file) |
| master_tag_content = [Copyright_header, filename] |
| with open(f"{output_dir}/master.tag", "w+") as master_tag: |
| master_tag.writelines(master_tag_content) |
| |
| sourcetodests[f"{os.path.dirname(oa_file)}/master.tag"].append(f"{output_dir}/master.tag") |
| |
| if copy_file(oa_file, output_path): |
| return {"return_code": COPY_ERROR, "path": oa_file} |
| else: |
| sourcetodests[oa_file].append(output_path) |
| |
| for dm_file in dm_files: |
| output_path = remap_path(dm_file, args.input, args.tmp) |
| if copy_file(dm_file, output_path): |
| return {"return_code": COPY_ERROR, "path": dm_file} |
| else: |
| sourcetodests[dm_file].append(output_path) |
| |
| def absolute_path(path): |
| return path.resolve() |
| |
| def make_readonly_oalib(base_path): |
| oa_files = glob.glob(f"{base_path}/**/.oalib", recursive=True) |
| for oa_file in oa_files: |
| os.chmod(oa_file, S_IREAD|S_IRGRP|S_IROTH) |
| if os.geteuid() == 0: |
| os.system(f"chattr +i {oa_file}") |
| |
| def make_readagain_oalib(base_path): |
| oa_files = glob.glob(f"{base_path}/**/.oalib", recursive=True) |
| for oa_file in oa_files: |
| if os.geteuid() == 0: |
| os.system(f"chattr -i {oa_file}") |
| os.chmod(oa_file, S_IWRITE|S_IREAD|S_IRGRP|S_IROTH) |
| |
| def copy_files(base_path, copy_path, pattern): |
| files = glob.glob(f"{base_path}/**/{pattern}", recursive=True) |
| for file_i in files: |
| if copy_file(file_i, file_i.replace(str(base_path), str(copy_path))) == COPY_ERROR: |
| sys.stderr.write(f"Error copying {file_i} to {file_i.replace(str(base_path), str(copy_path))}\n") |
| sys.stderr.flush() |
| |
| |
| def copy_lib_defs(): |
| copy_dict = [{"from": f"{args.input}/lib.defs", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/lib.defs"}, |
| {"from": f"{args.input}/s8iom0s8/V0.2.0/oa/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/s8iom0s8/V0.2.0/oa/cds.lib"}, |
| {"from": f"{args.input}/s8iom0s8/V0.2.1/oa/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/s8iom0s8/V0.2.1/oa/cds.lib"}, |
| {"from": f"{args.input}/s8iom0s8/V0.0.0/oa/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/s8iom0s8/V0.0.0/oa/cds.lib"}, |
| {"from": f"{args.input}/s8iom0s8/V0.1.0/oa/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/s8iom0s8/V0.1.0/oa/cds.lib"}, |
| {"from": f"{args.input}/scs8hdll/V0.1.0/oa/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/scs8hdll/V0.1.0/oa/cds.lib"}, |
| {"from": f"{args.input}/scs8lp/V0.0.0/oa/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/scs8lp/V0.0.0/oa/cds.lib"}, |
| {"from": f"{args.input}/scs8ms/V0.0.0/oa/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/scs8ms/V0.0.0/oa/cds.lib"}, |
| {"from": f"{args.input}/scs8ls/V0.1.0/oa/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/scs8ls/V0.1.0/oa/cds.lib"}, |
| {"from": f"{args.input}/scs8hd/V0.0.1/oa/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/scs8hd/V0.0.1/oa/cds.lib"}, |
| {"from": f"{args.input}/scs8hs/V0.0.0/oa/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/scs8hs/V0.0.0/oa/cds.lib"}, |
| {"from": f"{args.input}/scs8hvl/V0.0.0/oa/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/scs8hvl/V0.0.0/oa/cds.lib"}, |
| {"from": f"{args.input}/scs8hvl/V0.0.1/oa/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/scs8hvl/V0.0.1/oa/cds.lib"}, |
| {"from": f"{args.input}/s8/V1.0.1/VirtuosoOA/examples/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/s8/V1.0.1/VirtuosoOA/examples/cds.lib"}, |
| {"from": f"{args.input}/s8/V1.1.0/VirtuosoOA/examples/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/s8/V1.1.0/VirtuosoOA/examples/cds.lib"}, |
| {"from": f"{args.input}/s8/V1.2.1/VirtuosoOA/examples/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/s8/V1.2.1/VirtuosoOA/examples/cds.lib"}, |
| {"from": f"{args.input}/s8/V1.2.0/VirtuosoOA/examples/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/s8/V1.2.0/VirtuosoOA/examples/cds.lib"}, |
| {"from": f"{args.input}/s8/V2.0.1/VirtuosoOA/examples/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/s8/V2.0.1/VirtuosoOA/examples/cds.lib"}, |
| {"from": f"{args.input}/s8/V1.0.0/VirtuosoOA/examples/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/s8/V1.0.0/VirtuosoOA/examples/cds.lib"}, |
| {"from": f"{args.input}/s8/V1.3.0/VirtuosoOA/examples/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/s8/V1.3.0/VirtuosoOA/examples/cds.lib"}, |
| {"from": f"{args.input}/s8/V2.0.0/VirtuosoOA/examples/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/s8/V2.0.0/VirtuosoOA/examples/cds.lib"}, |
| {"from": f"{args.input}/s8/V2.0.1/DRC/dev/libs/cds.lib", "to": f"{args.tmp}/{TMP_OARESULT_DIR}/s8/V2.0.1/DRC/dev/libs/cds.lib"} |
| ] |
| |
| for copy in copy_dict: |
| if not os.path.exists(copy["to"]): |
| if copy_file(copy["from"], copy["to"]) == COPY_ERROR: |
| print(f"Error copying {copy['from']} to {copy['to']}! Exiting.") |
| sys.exit(1) |
| |
| def make_lib_dirs(base_path): |
| folders = [ |
| f"{base_path}/{TMP_OARESULT_DIR}/s8iom0s8/V0.0.0/oa/s8io_m0s8", |
| f"{base_path}/{TMP_OARESULT_DIR}/s8iom0s8/V0.1.0/oa/s8io_m0s8", |
| f"{base_path}/{TMP_OARESULT_DIR}/s8iom0s8/V0.2.0/oa/s8io_m0s8", |
| f"{base_path}/{TMP_OARESULT_DIR}/s8iom0s8/V0.2.1/oa/s8io_m0s8", |
| f"{base_path}/{TMP_OARESULT_DIR}/scs8hdll/V0.1.0/oa/scs8_hdll", |
| f"{base_path}/{TMP_OARESULT_DIR}/scs8lp/V0.0.0/oa/scs8_lp", |
| f"{base_path}/{TMP_OARESULT_DIR}/scs8ms/V0.0.0/oa/scs8_ms", |
| f"{base_path}/{TMP_OARESULT_DIR}/scs8ls/V0.1.0/oa/scs8_ls", |
| f"{base_path}/{TMP_OARESULT_DIR}/scs8hd/V0.0.1/oa/scs8_hd", |
| f"{base_path}/{TMP_OARESULT_DIR}/scs8hs/V0.0.0/oa/scs8_hs", |
| f"{base_path}/{TMP_OARESULT_DIR}/scs8hvl/V0.0.0/oa/scs8_hvl", |
| f"{base_path}/{TMP_OARESULT_DIR}/scs8hvl/V0.0.1/oa/scs8_hvl", |
| f"{base_path}/{TMP_OARESULT_DIR}/s8/V2.0.1/DRC/dev/libs/QA_s8_drc", |
| ] |
| |
| for folder in folders: |
| os.makedirs(folder, exist_ok=True) |
| |
| if __name__ == "__main__": |
| import doctest |
| fails, _ = doctest.testmod() |
| if fails>0: |
| exit(1) |
| |
| parser = argparse.ArgumentParser() |
| parser.add_argument( |
| "input", |
| help="The path to the source directory/file", |
| type=Path) |
| parser.add_argument( |
| "output", |
| help="The path to the output directory", |
| type=Path) |
| parser.add_argument( |
| "tmp", |
| help="The path to the output directory", |
| type=Path) |
| parser.add_argument( |
| "--sourcetodests", |
| help="Mapping from source files to destination files", |
| type=Path) |
| parser.add_argument( |
| "--cpus", |
| help="Number od CPUs in the system", |
| type=int) |
| args = parser.parse_args() |
| |
| # Resolve paths to make sure we can find args.input/args.output/args.tmp in file paths |
| args.input = args.input.resolve() |
| args.output = args.output.resolve() |
| args.tmp = args.tmp.resolve() |
| |
| # FIXME: set sys.argv as common.files echos back command-line args if sys.argv is > 1 |
| sys.argv = [sys.argv[0]] |
| |
| |
| make_readonly_oalib(args.input) #OA tools is rewriting .oalib every time (even if file exists) so to run it in parallel, we are making them read-only |
| copy_files(args.input, f"{args.tmp}/{TMP_OARESULT_DIR}", ".oalib") |
| make_readonly_oalib(f"{args.tmp}/{TMP_OARESULT_DIR}") |
| copy_lib_defs() #Copy lib.defs and cds.lib to output directory |
| make_lib_dirs(args.tmp) #Make empty folders for libs defined in lib.defs |
| |
| #Copy technology database to output folder |
| copy_files(args.input, f"{args.tmp}/{TMP_OARESULT_DIR}", "tech.db") |
| copy_files(args.input, f"{args.tmp}/{TMP_OARESULT_DIR}", "*.layermap") |
| |
| os.system(f"find {args.tmp}/{TMP_OARESULT_DIR} -name cds.lib | xargs sed -i 's/\.\//$TMP_PDK_HOME\//g'") #Add $TMP_PDK_HOME to s8iom0s8, scs8hdll, scs8lp, scs8ms, scs8ls, scs8hd, scs8hs, scs8hvl to redirect output files |
| |
| #Not needed as QA_s8_drc is now skipped #145 |
| #copy_folder(str(args.input), f"{args.tmp}/skywater-src-nda") |
| #args_input_old = str(args.input) |
| #args.input = f"{args.tmp}/skywater-src-nda" |
| |
| # Move skywater-src-nda to tmp and rename files with # |
| #for file_path in common.files([".oa"]): |
| # if "#" in str(file_path): |
| # copy_file(file_path, str(file_path).replace(args_input_old, f"{str(args.tmp)}/skywater-src-nda").replace("#", "_")) |
| |
| begin_time = datetime.datetime.now() |
| |
| repack_sequence = [ |
| {"unpack": call_oa2verilog, "pack": call_verilog2oa, "name": "oa2verilog"}, |
| #{"unpack": call_oa2strm, "pack": call_strm2oa, "name": "oa2strm"}, |
| {"unpack": None, "pack": call_verilogAnnotate, "name": "verilogAnnotate"}, |
| {"unpack": call_oa2def, "pack": call_def2oa, "name": "oa2def"}, |
| {"unpack": call_oa2lef, "pack": call_lef2oa, "name": "oa2lef"} |
| ] |
| |
| max_parallel_jobs = 2 * args.cpus |
| current_parallel_jobs = max_parallel_jobs |
| |
| file_paths = [f for f in common.files([".oa"])] |
| # fix paths to point to tmp dir |
| #fixed_paths = [] |
| #for path in file_paths: |
| # fixed_paths.append(path.replace(args_input_old, f"{str(args.tmp)}/skywater-src-nda")) |
| |
| |
| csv_status = defaultdict(list) |
| |
| def run_unpack_job(file_path, sequence): |
| #if "#" in str(file_path): |
| # file_path = str(file_path).replace("#", "_") |
| return unpack(file_path, sequence["unpack"]) |
| |
| def run_pack_job(file_path, sequence): |
| #if "#" in str(file_path): |
| # file_path = str(file_path).replace("#", "_") |
| global TMP_OARESULT_DIR |
| if sequence["pack"] is call_verilog2oa: |
| TMP_OARESULT_DIR = f"oaresults{job}" |
| copy_lib_defs() |
| os.system(f"find {args.tmp}/{TMP_OARESULT_DIR} -name cds.lib | xargs sed -i 's/\.\//$TMP_PDK_HOME\//g'") #Add $TMP_PDK_HOME to s8iom0s8, scs8hdll, scs8lp, scs8ms, scs8ls, scs8hd, scs8hs, scs8hvl to redirect output files |
| else: |
| TMP_OARESULT_DIR = "oaresults" |
| return pack(file_path, sequence["pack"]) |
| |
| for sequence in repack_sequence: |
| errored = defaultdict(list) |
| skipped = defaultdict(list) |
| unpack_job_success = [] |
| last_print = 0 |
| |
| for oa_job in [run_unpack_job, run_pack_job]: |
| print(f"Starting {sequence['name']} {'pack' if oa_job is run_pack_job else 'unpack'} job") |
| files_left = len(file_paths) |
| all_files = files_left |
| if sequence['pack'] == call_strm2oa and oa_job is run_pack_job: |
| current_parallel_jobs = 1 |
| else: |
| current_parallel_jobs = max_parallel_jobs |
| running_jobs = [None for i in range(current_parallel_jobs)] |
| current_file = 0 |
| |
| error_count = 0 |
| good_count = 0 |
| skipped_count = 0 |
| |
| error_map = defaultdict(list) |
| job_to_file_path = [None for i in range(current_parallel_jobs)] |
| job_status = [None for i in range(current_parallel_jobs)] |
| |
| while files_left > 0: |
| for job in range(len(running_jobs)): |
| if current_file % 1000 == 0 and last_print != current_file: |
| sys.stderr.write(f"[OA] [{time.strftime('%H:%M:%S')}] Processed {current_file} of {all_files}\n") |
| sys.stderr.flush() |
| last_print = current_file |
| if running_jobs[job] is not None and running_jobs[job].poll() is not None: |
| #Job finished |
| files_left -= 1 |
| if running_jobs[job].returncode != 0: |
| error_count += 1 |
| stderr = running_jobs[job].stderr.read().decode('utf-8') |
| job_status[job] = errors_msg[decode_rc(check_status(stderr, sequence["unpack"] if oa_job is run_unpack_job else sequence["pack"]))] |
| error_map[job_status[job]].append({file_path: stderr}) |
| if decode_rc(check_status(stderr, sequence["unpack"] if oa_job is run_unpack_job else sequence["pack"])) == UNKNOWN_ERROR: |
| sys.stderr.write(f"Job: {job}: {stderr}\n") |
| sys.stderr.flush() |
| else: |
| good_count += 1 |
| if oa_job is run_unpack_job: |
| unpack_job_success.append(job_to_file_path[job]) |
| |
| # For verilog2oa we creating separate folder for each thread (to make sure tmp folders do not have collision in names) |
| # after generating oa file, we need to move it to origin result folder |
| if oa_job is run_pack_job and sequence["pack"] is call_verilog2oa and running_jobs[job].returncode == 0: # only copy if we produced correct oa file and sequence is verilog2oa |
| file_path_copy = job_to_file_path[job] |
| move_file_path = file_path_copy.replace(str(args.input), f"{args.tmp}/oaresults{job}") |
| move_output_path = file_path_copy.replace(str(args.input), f"{args.tmp}/oaresults") |
| if move_file(move_file_path, move_output_path) == COPY_ERROR: |
| error_map[errors_msg[MISSING_FILE]].append({move_file_path: move_output_path}) |
| |
| if job_status[job] is None: |
| job_status[job] = errors_msg[NO_ERROR] |
| csv_status[job_to_file_path[job]].append({f'{sequence["name"]} unpack' if oa_job is run_unpack_job else f'{sequence["name"]} pack': job_status[job]}) |
| |
| # When job is done, set it to None |
| running_jobs[job] = None |
| job_status[job] = None |
| |
| if running_jobs[job] is None: |
| #start a new job |
| if current_file < all_files: |
| file_path = file_paths[current_file] |
| job_to_file_path[job] = file_path |
| # For pack job, run only if unpack was successful, for verilogAnnotate, always run, as it does not have unpack job |
| if (oa_job is run_pack_job and file_path in unpack_job_success) or (oa_job is run_unpack_job) or (sequence['pack'] is call_verilogAnnotate): |
| running_jobs[job] = oa_job(file_path, sequence) |
| |
| current_file += 1 |
| # check if this job was skipped |
| if running_jobs[job] is None: |
| files_left -= 1 |
| skipped_count += 1 |
| job_status[job] = errors_msg[SKIPPED] |
| csv_status[job_to_file_path[job]].append({f'{sequence["name"]} unpack' if oa_job is run_unpack_job else f'{sequence["name"]} pack': job_status[job]}) |
| else: |
| running_jobs[job] = None |
| |
| runname = sequence["name"] |
| if oa_job == run_unpack_job: |
| runname += "[unpack]" |
| else: |
| runname += "[pack]" |
| |
| log_name = f"errored-{runname.replace('[', '-').replace(']', '')}.txt" |
| with open(log_name, mode='wt', encoding='utf-8') as myfile: |
| json.dump(error_map, myfile, indent=2) |
| |
| for error_type, errors in error_map.items(): |
| sys.stderr.write(f" >>> OA |{runname}|: [{time.strftime('%H:%M:%S')}] Error type: [{error_type}] count: [{len(errors)}]\n") |
| sys.stderr.flush() |
| |
| sys.stderr.write(" >>> OA |%s|: [%s] Error count = %d -- %.2f %%\n" % (runname,time.strftime('%H:%M:%S'),error_count,100.0 * error_count / all_files)) |
| sys.stderr.write(" >>> OA |%s|: [%s] Success count = %d -- %.2f %%\n" % (runname,time.strftime('%H:%M:%S'),good_count, 100.0 * good_count / all_files)) |
| sys.stderr.write(" >>> OA |%s|: [%s] Skipped count = %d -- %.2f %%\n" % (runname,time.strftime('%H:%M:%S'),skipped_count, 100.0 * skipped_count / all_files)) |
| sys.stderr.flush() |
| |
| with open(f'errored-{sequence["name"]}.txt', mode='wt', encoding='utf-8') as myfile: |
| json.dump(errored, myfile, indent=2) |
| |
| with open(f'skipped-{sequence["name"]}.txt', mode='wt', encoding='utf-8') as myfile: |
| json.dump(skipped, myfile, indent=2) |
| |
| with open('file_status.csv', 'w', newline='') as file: |
| writer = csv.writer(file) |
| writer.writerow(["File", "oa2verilog", "verilog2oa", "verilogAnnotate", "oa2def", "def2oa", "oa2lef", "lef2oa"]) |
| for file_str in csv_status: |
| row = [file_str.replace(str(args.input), ""), csv_status[file_str][0]["oa2verilog unpack"], csv_status[file_str][1]["oa2verilog pack"], csv_status[file_str][3]["verilogAnnotate pack"], csv_status[file_str][4]["oa2def unpack"], csv_status[file_str][5]["oa2def pack"], csv_status[file_str][6]["oa2lef unpack"], csv_status[file_str][7]["oa2lef pack"]] |
| writer.writerow(row) |
| |
| TMP_OARESULT_DIR = "oaresults" |
| copy_repacked_to_output() |
| |
| with open(args.sourcetodests, 'w') as srctodst: |
| json.dump(sourcetodests, srctodst, indent=2) |
| |
| print("Done!") |
| print(f"Took: {datetime.datetime.now() - begin_time}") |
| make_readagain_oalib(args.input) |
| make_readagain_oalib(f"{args.tmp}/{TMP_OARESULT_DIR}") |
