#!/usr/bin/env pythonimport os, os.path, re, socket, sysfrom os import environ as env, listdirfrom os.path import basename, isdir, isfile, islink, join as joinpath, normpathfrom filecmp import cmp as filecmpfrom shutil import copydef nfspath(dir):    if dir.startswith('/.automount/'):        dir = '/n/%s' % dir[12:]    elif not dir.startswith('/n/'):        dir = '/n/%s%s' % (socket.gethostname().split('.')[0], dir)    return dirdef syncdir(srcdir, destdir):    srcdir = normpath(srcdir)    destdir = normpath(destdir)    if not isdir(destdir):        sys.exit('destination directory "%s" does not exist' % destdir)            for root, dirs, files in os.walk(srcdir):        root = normpath(root)        prefix = os.path.commonprefix([root, srcdir])        root = root[len(prefix):]        if root.startswith('/'):            root = root[1:]        for rem in [ d for d in dirs if d.startswith('.') or d == 'SCCS']:            dirs.remove(rem)        for entry in dirs:            newdir = joinpath(destdir, root, entry)            if not isdir(newdir):                os.mkdir(newdir)                print 'mkdir', newdir        for i,d in enumerate(dirs):            if islink(joinpath(srcdir, root, d)):                dirs[i] = joinpath(d, '.')        for entry in files:            dest = normpath(joinpath(destdir, root, entry))            src = normpath(joinpath(srcdir, root, entry))            if not isfile(dest) or not filecmp(src, dest):                print 'copy %s %s' % (dest, src)                copy(src, dest)progpath = nfspath(sys.path[0])progname = basename(sys.argv[0])usage = """\Usage:    %(progname)s [-c] [-e] [-f] [-j <jobfile>] [-q queue] [-v] <regexp>    -c           clean directory if job can be run    -e           only echo pbs command info, don't actually send the job    -f           force the job to run regardless of state    -q <queue>   submit job to the named queue    -j <jobfile> specify the jobfile (default is <basedir>/test.py)    -v           be verbose    %(progname)s [-j <jobfile>] -l [-v] <regexp>    -j <jobfile> specify the jobfile (default is <basedir>/test.py)    -l           list job names, don't submit    -v           be verbose (list job parameters)    %(progname)s -h    -h           display this help""" % locals()try:    import getopt    opts, args = getopt.getopt(sys.argv[1:], '-CRcd:efhj:lq:v')except getopt.GetoptError:    sys.exit(usage)clean = Falseonlyecho = Falseexprs = []force = Falselistonly = Falsequeue = ''verbose = Falsejfile = 'Base/test.py'docpts = Falsedoruns = Truerunflag = Falsefor opt,arg in opts:    if opt == '-C':        docpts = True    if opt == '-R':        runflag = True    if opt == '-c':        clean = True    if opt == '-e':        onlyecho = True    if opt == '-f':        force = True    if opt == '-h':        print usage        sys.exit(0)    if opt == '-j':        jfile = arg    if opt == '-l':        listonly = True    if opt == '-q':        queue = arg    if opt == '-v':        verbose = Trueif docpts:    doruns = runflagfor arg in args:    exprs.append(re.compile(arg))import jobfile, pbsfrom job import JobDir, dateconf = jobfile.JobFile(jfile)if not listonly and not onlyecho and isdir(conf.linkdir):    if verbose:        print 'Checking for outdated files in Link directory'    syncdir(conf.linkdir, conf.basedir)jobnames = {}joblist = []if docpts and doruns:    gen = conf.alljobs()elif docpts:    gen = conf.checkpoints()elif doruns:    gen = conf.jobs()for job in gen:    if job.name in jobnames:        continue    if exprs:        for expr in exprs:            if expr.match(job.name):                joblist.append(job)                break    else:        joblist.append(job)if listonly:    if verbose:        for job in joblist:            job.printinfo()    else:        for job in joblist:            print job.name    sys.exit(0)if not onlyecho:    newlist = []    for job in joblist:        jobdir = JobDir(joinpath(conf.rootdir, job.name))        if jobdir.exists():            if not force:                status = jobdir.getstatus()                if status == 'queued':                    continue                if status == 'running':                    continue                if status == 'success':                    continue            if not clean:                sys.exit('job directory %s not clean!' % jobdir)            jobdir.clean()        newlist.append(job)    joblist = newlistclass NameHack(object):    def __init__(self, host='pbs.pool', port=24465):        self.host = host        self.port = port        self.socket = None    def setname(self, jobid, jobname):        try:            jobid = int(jobid)        except ValueError:            jobid = int(jobid.strip().split('.')[0])        jobname = jobname.strip()        # since pbs can handle jobnames of 15 characters or less,        # don't use the raj hack.        if len(jobname) <= 15:            return        if self.socket is None:            import socket            self.socket = socket.socket()            # Connect to pbs.pool and send the jobid/jobname pair to port            # 24465 (Raj didn't realize that there are only 64k ports and            # setup inetd to point to port 90001)            self.socket.connect((self.host, self.port))        self.socket.send("%s %s\n" % (jobid, jobname))namehack = NameHack()for job in joblist:    jobdir = JobDir(joinpath(conf.rootdir, job.name))        if not onlyecho:        jobdir.create()            print 'Job name:       %s' % job.name    print 'Job directory:  %s' % jobdir    qsub = pbs.qsub()    qsub.pbshost = 'simpool.eecs.umich.edu'    qsub.stdout = jobdir.file('jobout')    qsub.name = job.name[:15]    qsub.join = True    qsub.node_type = 'FAST'    qsub.env['ROOTDIR'] = conf.rootdir    qsub.env['JOBNAME'] = job.name    if len(queue):        qsub.queue = queue    qsub.build(joinpath(progpath, 'job.py'))    if verbose:        print 'PBS Command:    %s' % qsub.command    if not onlyecho:        ec = qsub.do()        if ec == 0:            jobid = qsub.result            print 'PBS Jobid:      %s' % jobid            namehack.setname(jobid, job.name)            queued = date()            jobdir.echofile('.queued', queued)            jobdir.setstatus('queued on %s' % queued)        else:            print 'PBS Failed'