#!/usr/bin/env python3
#
# staging_install.py
#
# This file copies the staging area created by foundry_install.py
# into the target directory area, changing paths to match the target,
# and creating symbolic links where requested and allowed.
#
# Options:
#    -link_from <type>	Make symbolic links to vendor files from target
#			Types are: "none", "source", or a PDK name.
#			Default "none" (copy all files from source)
#    -ef_format		Use efabless naming (libs.ref/techLEF),
#			otherwise use generic naming (libs.tech/lef)
#
#    -staging <path>	Path to staging top level directory
#    -target <path>	Path to target top level directory
#    -local <path>	For distributed installs, this is the local
#			path to target top level directory.
#    -source <path>     Path to original source top level directory,
#                       if link_from is "source".  This option may
#                       be called multiple times if there are multiple
#                       sources.
#    -variable <name>	Specify a variable name that is used for the
#			target path.  This variable name must be enforced
#			in setup scripts like .magicrc

import re
import os
import sys
import glob
import stat
import shutil
import filecmp
import subprocess

# NOTE:  This version of copy_tree from distutils works like shutil.copytree()
# in Python 3.8 and up ONLY using "dirs_exist_ok=True".
from distutils.dir_util import copy_tree

def usage():
    print("staging_install.py [options...]")
    print("   -link_from <name> Make symbolic links from target to <name>")
    print("                     where <name> can be 'source' or a PDK name.")
    print("                     Default behavior is to copy all files.")
    print("   -copy             Copy files from source to target (default)")
    print("   -ef_format        Use efabless naming conventions for local directories")
    print("")
    print("   -staging <path>   Path to top of staging directory tree")
    print("   -target <path>    Path to top of target directory tree")
    print("   -local <path>	Local path to top of target directory tree for distributed install")
    print("")
    print(" If <target> is unspecified then <name> is used for the target.")

def makeuserwritable(filepath):
    if os.path.exists(filepath):
        st = os.stat(filepath)
        os.chmod(filepath, st.st_mode | stat.S_IWUSR)

# Filter files to replace all strings matching "stagingdir" with "localdir" for
# every file in "tooldir".  If "tooldir" contains subdirectories, then recursively
# apply the replacement filter to all files in the subdirectories.  Do not follow
# symbolic links.

def filter_recursive(tooldir, stagingdir, localdir):
    # Add any non-ASCII file types here
    bintypes = ['.gds', '.gds2', '.gdsii', '.png']

    if not os.path.exists(tooldir):
        return 0
    elif os.path.islink(tooldir):
        return 0

    toolfiles = os.listdir(tooldir)
    total = 0

    # Add any non-ASCII 
    binexts = ['.png']

    for file in toolfiles:
        # Do not attempt to do text substitutions on a binary file!
        if os.path.splitext(file)[1] in bintypes:
            continue

        filepath = tooldir + '/' + file
        if os.path.islink(filepath):
            continue
        elif os.path.isdir(filepath):
            total += filter_recursive(filepath, stagingdir, localdir)
        else:
            with open(filepath, 'r') as ifile:
                try:
                    flines = ifile.read().splitlines()
                except UnicodeDecodeError:
                    print('Failure to read file ' + filepath + '; non-ASCII content.')
                    continue

            # Make sure this file is writable (as the original may not be)
            makeuserwritable(filepath)

            modified = False
            with open(filepath, 'w') as ofile:
                for line in flines:
                    newline = line.replace(stagingdir, localdir)
                    print(newline, file=ofile) 
                    if newline != line:
                        modified = True

            if modified:
                total += 1
    return total
        
# To avoid problems with various library functions that copy hierarchical
# directory trees, remove all the files from the target that are going to
# be replaced by the contents of staging.  This avoids problems with
# symbolic links and such.

def remove_target(stagingdir, targetdir):

    slist = os.listdir(stagingdir)
    tlist = os.listdir(targetdir)

    for sfile in slist:
        if sfile in tlist:
            tpath = targetdir + '/' + sfile
            if os.path.islink(tpath):
                os.unlink(tpath)
            elif os.path.isdir(tpath):
                remove_target(stagingdir + '/' + sfile, targetdir + '/' + sfile)
            else:
                os.remove(tpath)

# Create a list of source files/directories from the contents of source.txt

def make_source_list(sources):
    sourcelist = []
    for source in sources:
        sourcelist.extend(glob.glob(source))
    return sourcelist

# Replace all files in list "libfiles" with symbolic links to files in
# "sourcelist", where the files are found to be the same.  If the entry
# in "libfiles" is a directory and the same directory is found in "sourcelist",
# then repeat recursively on the subdirectory.
#
# Because the installation may be distributed, there may be a difference
# between where the files to be linked to currently are (checklist)
# and where they will eventually be located (sourcelist).

def replace_with_symlinks(libfiles, sourcelist):
    # List of files that never get installed
    exclude = ['generate_magic.tcl', '.magicrc', 'sources.txt']
    total = 0
    for libfile in libfiles:
        if os.path.islink(libfile):
            continue
        else:
            try:
                sourcefile = next(item for item in sourcelist if os.path.split(item)[1] == os.path.split(libfile)[1])
            except:
                pass
            else:
                if os.path.isdir(libfile):
                    newlibfiles = glob.glob(libfile + '/*')
                    newsourcelist = glob.glob(sourcefile + '/*')
                    total += replace_with_symlinks(newlibfiles, newsourcelist)
                elif filecmp.cmp(libfile, sourcefile):
                    if not os.path.split(libfile)[1] in exclude:
                        os.remove(libfile)
                        # Use absolute path for the source file
                        sourcepath = os.path.abspath(sourcefile)
                        os.symlink(sourcepath, libfile)
                        total += 1
    return total

# Similar to the routine above, replace files in "libdir" with symbolic
# links to the files in "srclibdir", where the files are found to be the
# same.  The difference from the routine above is that "srclibdir" is
# another installed PDK, and so the directory hierarchy is expected to
# match that of "libdir" exactly, so the process of finding matches is
# a bit more straightforward.
#
# Because the installation may be distributed, there may be a difference
# between where the files to be linked to currently are (checklibdir)
# and where they will eventually be located (srclibdir).

def replace_all_with_symlinks(libdir, srclibdir, checklibdir):
    total = 0
    try:
        libfiles = os.listdir(libdir)
    except FileNotFoundError:
        print('Cannot list directory ' + libdir)
        print('Called: replace_all_with_symlinks(' + libdir + ', ' + srclibdir + ', ' + checklibdir + ')')
        return total

    try:
        checkfiles = os.listdir(checklibdir)
    except FileNotFoundError:
        print('Cannot list check directory ' + checklibdir)
        print('Called: replace_all_with_symlinks(' + libdir + ', ' + srclibdir + ', ' + checklibdir + ')')
        return total

    for libfile in libfiles:
        if libfile in checkfiles:
            libpath = libdir + '/' + libfile
            checkpath = checklibdir + '/' + libfile
            srcpath = srclibdir + '/' + libfile

            if os.path.isdir(libpath):
                if os.path.isdir(checkpath):
                    total += replace_all_with_symlinks(libpath, srcpath, checkpath)
            else:
                try:
                    if filecmp.cmp(libpath, checkpath):
                        os.remove(libpath)
                        os.symlink(srcpath, libpath)
                        total += 1
                except FileNotFoundError:
                    print('Failed file compare with libpath=' + libpath + ', checkpath=' + checkpath)

    return total

#----------------------------------------------------------------
# This is the main entry point for the staging install script.
#----------------------------------------------------------------

if __name__ == '__main__':

    if len(sys.argv) == 1:
        print("No options given to staging_install.py.")
        usage()
        sys.exit(0)
    
    optionlist = []
    newopt = []

    stagingdir = None
    targetdir = None
    link_from = None
    localdir = None
    variable = None

    ef_format = False
    do_install = True

    # Break arguments into groups where the first word begins with "-".
    # All following words not beginning with "-" are appended to the
    # same list (optionlist).  Then each optionlist is processed.
    # Note that the first entry in optionlist has the '-' removed.

    for option in sys.argv[1:]:
        if option.find('-', 0) == 0:
            if newopt != []:
                optionlist.append(newopt)
                newopt = []
            newopt.append(option[1:])
        else:
            newopt.append(option)

    if newopt != []:
        optionlist.append(newopt)

    # Check for option "ef_format" or "std_format"
    for option in optionlist[:]:
        if option[0] == 'ef_naming' or option[0] == 'ef_names' or option[0] == 'ef_format':
            optionlist.remove(option)
            ef_format = True
        elif option[0] == 'std_naming' or option[0] == 'std_names' or option[0] == 'std_format':
            optionlist.remove(option)
            ef_format = False
        elif option[0] == 'uninstall':
            optionlist.remove(option)
            do_install = False

    # Check for options "link_from", "staging", "target", and "local"

    link_name = None
    for option in optionlist[:]:
        if option[0] == 'link_from':
            optionlist.remove(option)
            if option[1].lower() == 'none':
                link_from = None
            elif option[1].lower() == 'source':
                link_from = 'source'
            else:
                link_from = option[1]
                link_name = os.path.split(link_from)[1]
        elif option[0] == 'staging' or option[0] == 'source':
            optionlist.remove(option)
            stagingdir = option[1]
        elif option[0] == 'target':
            optionlist.remove(option)
            targetdir = option[1]
        elif option[0] == 'local':
            optionlist.remove(option)
            localdir = option[1]
        elif option[0] == 'variable':
            optionlist.remove(option)
            variable = option[1]

    # Error if no staging or dest specified
    if not stagingdir:
        print("No staging directory specified.  Exiting.")
        sys.exit(1)

    if not targetdir:
        print("No target directory specified.  Exiting.")
        sys.exit(1)

    # If localdir is not specified, then it is the same as the parent
    # of the target (local installation assumed)
    if not localdir:
        localdir = targetdir

    # Take the target PDK name from the target path last component
    pdkname = os.path.split(targetdir)[1]

    # If link source is a PDK name, if it has no path, then pull the
    # path from the target name.

    if link_from:
        if link_from != 'source':
            if link_from.find('/', 0) < 0:
                link_name = link_from
                link_from = os.path.split(localdir)[0] + '/' + link_name
        else:
            # If linking from source, convert the source path to an
            # absolute pathname.
            stagingdir = os.path.abspath(stagingdir)

        # If link_from is the same as localdir, then set link_from to None
        if link_from == localdir:
            link_from = None

    # checkdir is the DIST target directory for the PDK pointed
    # to by link_name.  Files must be found there before creating
    # symbolic links to the (not yet existing) final install location.

    if link_name:
        checkdir = os.path.split(targetdir)[0] + '/' + link_name
    else:
        checkdir = ''

    # Diagnostic
    if do_install:
        print("Installing in target directory " + targetdir)
    else:
        print("Uninstalling from target directory " + targetdir)
        print("(Method not yet implemented)")

    # Create the top-level directories

    os.makedirs(targetdir, exist_ok=True)
    os.makedirs(targetdir + '/libs.tech', exist_ok=True)
    os.makedirs(targetdir + '/libs.ref', exist_ok=True)
    if os.path.isdir(stagingdir + '/libs.priv'):
        os.makedirs(targetdir + '/libs.priv', exist_ok=True)
        has_priv = True
    else:
        has_priv = False

    # Path to magic techfile depends on ef_format

    if ef_format == True:
        mag_current = '/libs.tech/magic/current/'
    else:
        mag_current = '/libs.tech/magic/'

    # First install everything by direct copy.  Keep the staging files
    # as they will be used to reference the target area to know which
    # files need to be checked and/or modified.

    if not os.path.isdir(targetdir):
        try:
            os.makedirs(targetdir, exist_ok=True)
        except:
            print('Fatal error:  Cannot make target directory ' + targetdir + '!')
            exit(1)

    # Remove any files from the target directory that are going to be replaced
    print('Removing files from target')
    remove_target(stagingdir, targetdir)

    print('Copying staging files to target')
    # print('Diagnostic:  copy_tree ' + stagingdir + ' ' + targetdir)
    copy_tree(stagingdir, targetdir, preserve_symlinks=True)
    print('Done.')

    # Magic and qflow setup files have references to the staging area that have
    # been used by the vendor install;  these need to be changed to the target
    # directory.

    print('Changing local path references from ' + stagingdir + ' to ' + localdir)
    print('Part 1:  Tools')

    needcheck = ['ngspice']
    techdirs = ['/libs.tech/']
    if has_priv:
        techdirs.append('/libs.priv/')

    for techdir in techdirs:
        tools = os.listdir(targetdir + techdir)
        for tool in tools:
            tooldir = targetdir + techdir + tool

            # There are few enough tool setup files that they can just all be
            # filtered directly.  This code only looks in the directory 'tooldir'.
            # If there are files is subdirectories of 'tooldir' that require
            # substitution, then this code needs to be revisited.

            # Note that due to the low overhead of tool setup files, there is
            # no attempt to check for possible symlinks to link_from if link_from
            # is a base PDK.

            total = filter_recursive(tooldir, stagingdir, localdir)
            if total > 0:
                substr = 'substitutions' if total > 1 else 'substitution'
                print('      ' + tool + ' (' + str(total) + ' ' + substr + ')')

    # If "link_from" is another PDK, then check all files against the files in
    # the other PDK, and replace the file with a symbolic link if the file contents
    # match (Note:  This is done only for ngspice model files;  other tool files are
    # generally small and deemed unnecessary to make symbolic links).

    if link_from not in ['source', None]:
        thispdk = os.path.split(targetdir)[1]

        # Only create links for PDKs other than the one we are making links to.
        if thispdk != link_from:
            print('Replacing files with symbolic links to ' + link_from + ' where possible.')
            for techdir in techdirs:
                for tool in needcheck:
                    tooldir = targetdir + techdir + tool
                    srctooldir = link_from + techdir + tool
                    if checkdir != '':
                        checktooldir = checkdir + techdir + tool
                    else:
                        checktooldir = srctooldir
                    if os.path.exists(tooldir):
                        total = replace_all_with_symlinks(tooldir, srctooldir, checktooldir)
                        if total > 0:
                            symstr = 'symlinks' if total > 1 else 'symlink'
                            print('      ' + tool + ' (' + str(total) + ' ' + symstr + ')')

    # In .mag files in mag/ and maglef/, also need to change the staging
    # directory name to localdir.  If "-variable" is specified in the options,
    # the replace the staging path with the variable name, not localdir.

    if variable:
        localname = '$' + variable
    else:
        localname = localdir

    needcheck = ['mag', 'maglef']
    refdirs = ['/libs.ref/']
    if has_priv:
        refdirs.append('/libs.priv/')

    if ef_format:
        print('Part 2:  Formats')
        for refdir in refdirs:
            for filetype in needcheck:
                print('   ' + filetype)
                filedir = targetdir + refdir + filetype
                if os.path.isdir(filedir):
                    libraries = os.listdir(filedir)
                    for library in libraries:
                        libdir = filedir + '/' + library
                        total = filter_recursive(libdir, stagingdir, localname)
                        if total > 0:
                            substr = 'substitutions' if total > 1 else 'substitution'
                            print('      ' + library + ' (' + str(total) + ' ' + substr + ')')
    else:
        print('Part 2:  Libraries')
        for refdir in refdirs:
            libraries = os.listdir(targetdir + refdir)
            for library in libraries:
                print('   ' + library)
                for filetype in needcheck:
                    filedir = targetdir + refdir + library + '/' + filetype
                    total = filter_recursive(filedir, stagingdir, localname)
                    if total > 0:
                        substr = 'substitutions' if total > 1 else 'substitution'
                        print('      ' + filetype + ' (' + str(total) + ' ' + substr + ')')
        
    # If "link_from" is "source", then check all files against the source
    # directory, and replace the file with a symbolic link if the file
    # contents match.  The "foundry_install.py" script should have added a
    # file "sources.txt" with the name of the source directories for each
    # install directory.

    if link_from not in ['source', None]:
        print('Replacing files with symbolic links to source where possible.')
        for refdir in refdirs:
            if ef_format:
                filedirs = os.listdir(targetdir + refdir)
                for filedir in filedirs:
                    print('   ' + filedir)
                    dirpath = targetdir + refdir + filedir
                    if os.path.isdir(dirpath):
                        libraries = os.listdir(dirpath)
                        for library in libraries:
                            libdir = targetdir + refdir + filedir + '/' + library
                            libfiles = os.listdir(libdir)
                            if 'sources.txt' in libfiles:
                                libfiles = glob.glob(libdir + '/*')
                                libfiles.remove(libdir + '/sources.txt')
                                with open(libdir + '/sources.txt') as ifile:
                                    sources = ifile.read().splitlines()
                                sourcelist = make_source_list(sources)
                                total = replace_with_symlinks(libfiles, sourcelist)
                                if total > 0:
                                    symstr = 'symlinks' if total > 1 else 'symlink'
                                    print('      ' + library + ' (' + str(total) + ' ' + symstr + ')')
            else:
                libraries = os.listdir(targetdir + refdir)
                for library in libraries:
                    print('   ' + library)
                    filedirs = os.listdir(targetdir + refdir + library)
                    for filedir in filedirs:
                        libdir = targetdir + refdir + library + '/' + filedir
                        if os.path.isdir(libdir):
                            libfiles = os.listdir(libdir)
                            if 'sources.txt' in libfiles:
                                # List again, but with full paths.
                                libfiles = glob.glob(libdir + '/*')
                                libfiles.remove(libdir + '/sources.txt')
                                with open(libdir + '/sources.txt') as ifile:
                                    sources = ifile.read().splitlines()
                                sourcelist = make_source_list(sources)
                                total = replace_with_symlinks(libfiles, sourcelist)
                                if total > 0:
                                    symstr = 'symlinks' if total > 1 else 'symlink'
                                    print('      ' + filedir + ' (' + str(total) + ' ' + symstr + ')')

    # Otherwise, if "link_from" is another PDK, then check all files against
    # the files in the other PDK, and replace the file with a symbolic link
    # if the file contents match.

    elif link_from:
        thispdk = os.path.split(targetdir)[1]

        # Only create links for PDKs other than the one we are making links to.
        if thispdk != link_from:

            print('Replacing files with symbolic links to ' + link_from + ' where possible.')

            for refdir in refdirs:
                if ef_format:
                    filedirs = os.listdir(targetdir + refdir)
                    for filedir in filedirs:
                        print('   ' + filedir)
                        dirpath = targetdir + refdir + filedir
                        if os.path.isdir(dirpath):
                            libraries = os.listdir(dirpath)
                            for library in libraries:
                                libdir = targetdir + refdir + filedir + '/' + library
                                srclibdir = link_from + refdir + filedir + '/' + library
                                if checkdir != '':
                                    checklibdir = checkdir + refdir + filedir + '/' + library
                                else:
                                    checklibdir = srclibdir
                                if os.path.exists(libdir):
                                    total = replace_all_with_symlinks(libdir, srclibdir, checklibdir)
                                    if total > 0:
                                        symstr = 'symlinks' if total > 1 else 'symlink'
                                        print('      ' + library + ' (' + str(total) + ' ' + symstr + ')')
                else:
                    libraries = os.listdir(targetdir + refdir)
                    for library in libraries:
                        print('   ' + library)
                        filedirs = os.listdir(targetdir + refdir + library)
                        for filedir in filedirs:
                            libdir = targetdir + refdir + library + '/' + filedir
                            srclibdir = link_from + refdir + library + '/' + filedir
                            if checkdir != '':
                                checklibdir = checkdir + refdir + library + '/' + filedir
                            else:
                                checklibdir = srclibdir
                            if os.path.exists(libdir):
                                total = replace_all_with_symlinks(libdir, srclibdir, checklibdir)
                                if total > 0:
                                    symstr = 'symlinks' if total > 1 else 'symlink'
                                    print('      ' + filedir + ' (' + str(total) + ' ' + symstr + ')')

    # Remove temporary files:  Magic generation scripts, sources.txt
    # file, and magic extract files.

    print('Removing temporary files from destination.')

    for refdir in refdirs:
        if ef_format:
            filedirs = os.listdir(targetdir + refdir)
            for filedir in filedirs:
                if os.path.islink(filedir):
                    continue
                elif os.path.isdir(filedir):
                    libraries = os.listdir(targetdir + refdir + filedir)
                    for library in libraries:
                        libdir = targetdir + refdir + filedir + '/' + library
                        libfiles = os.listdir(libdir)
                        for libfile in libfiles:
                            filepath = libdir + '/' + libfile
                            if os.path.islink(filepath):
                                continue
                            elif libfile == 'sources.txt':
                                os.remove(filepath)
                            elif libfile == 'generate_magic.tcl':
                                os.remove(filepath)
                            elif os.path.splitext(libfile)[1] == '.ext':
                                os.remove(filepath)
        else:
            libraries = os.listdir(targetdir + refdir)
            for library in libraries:
                filedirs = os.listdir(targetdir + refdir + library)
                for filedir in filedirs:
                    filepath = targetdir + refdir + library + '/' + filedir
                    if os.path.islink(filepath):
                        continue
                    elif os.path.isdir(filepath):
                        libfiles = os.listdir(filepath)
                        for libfile in libfiles:
                            libfilepath = filepath + '/' + libfile
                            if os.path.islink(libfilepath):
                                continue
                            elif libfile == 'sources.txt':
                                os.remove(libfilepath)
                            elif libfile == 'generate_magic.tcl':
                                os.remove(libfilepath)
                            elif os.path.splitext(libfile)[1] == '.ext':
                                os.remove(libfilepath)
        
    print('Done with PDK migration.')
    sys.exit(0)