#!/usr/bin/env python
#
# @package      hubzero-submit-client
# @file         submit
# @author       Rick Kennell <kennell@purdue.edu>
# @copyright    Copyright (c) 2004-2012 Purdue University. All rights reserved.
# @license      http://www.gnu.org/licenses/lgpl-3.0.html LGPLv3
#
# Copyright (c) 2004-2012 Purdue University
# All rights reserved.
#
# This file is part of: The HUBzero(R) Platform for Scientific Collaboration
#
# The HUBzero(R) Platform for Scientific Collaboration (HUBzero) is free
# software: you can redistribute it and/or modify it under the terms of
# the GNU Lesser General Public License as published by the Free Software
# Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# HUBzero is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# HUBzero is a registered trademark of Purdue University.
#
from OpenSSL import SSL
import sys
import os
import select
import socket
import signal
import resource
import time
import getpass
import errno
import exceptions

#=============================================================================
# Set default values.
#=============================================================================
configdirs = [ ".", "/etc/submit", "/apps/submit/etc", "/tmp/submit" ]
configdir = None
connect_list = [ ]
username = getpass.getuser()
debug_flag = 0
help_flag = 0
print_metrics = 0
logfile = None

readers = {}
writers = {}
writeclose = {}

inbuf = ""
old_inbuf = ""
inoffset = 0
old_inoffset = 0
input_files = {}
output_files = {}

testfile_file = None
testfile_inode = None
token = None
jobid = None

command_args = None
validate_args = None
local_flag = None
child_pid = None

ready_to_exit = None
child_has_exited = None
exit_status = None

#=============================================================================
# Debugger support.
#=============================================================================
def info(type, value, tb):
    if hasattr(sys, 'ps1') or not sys.stderr.isatty():
        # You are in interactive mode or don't have a tty-like
        # device, so call the default hook
        sys.__exechook__(type, value, tb)
    else:
        import traceback, pdb
        # You are not in interactive mode; print the exception
        traceback.print_exception(type, value, tb)
        print
        # ... then star the debugger in post-mortem mode
        pdb.pm()

#=============================================================================
# Log a message.
#=============================================================================
def log(msg):
  global logfile
  if (os.isatty(2)):
    sys.stderr.write(msg + "\n")
  else:
    if not logfile:
      try:
        logfile = open(".submit.log","w")
      except:
        return
    logfile.write(str(msg) + "\n")
    logfile.flush()

#=============================================================================
# Log a message if debug_flag is set.
#=============================================================================
def debug(msg):
  if debug_flag:
    try:
      log(msg)
    except:
      pass

#=============================================================================
# Write to the tty.  Or log the message if that fails.
#=============================================================================
def ttyWrite(msg):
  try:
    f=open("/dev/tty","w")
    f.write(msg + "\n")
    f.close()
  except:
    log(msg)

#=============================================================================
# Verify the certificate
#=============================================================================
def verify_cb(conn, cert, errnum, depth, ok):
  #print 'Got certificate: %s' % cert.get_subject()
  #print 'Issuer: %s' % cert.get_issuer()
  #print "Depth = %s" % str(depth)
  return ok

#=============================================================================
# Things to do file/name mapping.
#=============================================================================
nametofile = {}
filetoname = {}

def mapfile(file,name):
  nametofile[name] = file
  filetoname[file] = name


#=============================================================================
# Socket things.
#=============================================================================
def parseURL(item):
  try:
    temp = item
    colon = temp.index(":")
    proto = temp[0:colon]
    temp = temp[colon+1:]

    slashes = temp.index("//")
    temp = temp[slashes+2:]

    colon = temp.index(":")
    host = temp[0:colon]
    temp = temp[colon+1:]

    try:
      slash = temp.index("/")
      port = int(temp[0:slash])
    except:
      port = int(temp)

  except:
    log("Improper network specification: %s" % item)
    return ("", "", 0)

  return (proto, host, port)


#=============================================================================
# Send and receive a string so we know we have a valid connection.
#=============================================================================
def handshake(f):
  string = "Hello.\n"
  reply = ""

  try:
    # Write the string.
    f.send(string)

    # Expect the same string back.
    reply = f.recv(len(string))
    if reply == string:
      return

  except Exception, err:
    debug("handshake: %s" % reply)
    debug("err = %s" % str(err))
    pass

  debug("Connection handshake failed.  Protocol mismatch?")
  raise IOError()


def setupSocket():
  global sock
  delay = 5

  if len(connect_list) == 0:
    ttyWrite("Server connection list empty. Check resources file.")
    sys.exit(1)

  while 1:
    for item in connect_list:
      (proto,host,port) = parseURL(item)

      if proto.lower() == "tls":
        sock = SSL.Connection(ctx,socket.socket(socket.AF_INET,socket.SOCK_STREAM))
      elif proto.lower() == "tcp":
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

      else:
        log("Unknown protocol: %s" % item)
        continue
    
      try:
        sock.connect( (host,port) )
        handshake(sock)
        sock.setblocking(0)
        mapfile(sock,"!socket")
        readers[sock] = "!socket"
        writers[sock] = ""
        return
      except exceptions.SystemExit:
        sys.exit(1)
      except:
        ttyWrite("Unable to connect to server %s.  Retrying." % item)
    
    time.sleep(delay)


def restoreSocket():
  global sock
  global inbuf
  global inoffset
  global old_inbuf
  global old_inoffset

  closeFile(sock)
  sock=None

  if token and jobid and not ready_to_exit:
    # Suspend input handling until connection is restored...
    try:
      del readers[nametofile["!stdin"]]
    except:
      pass
    setupSocket()
    lastreceived = inoffset + len(inbuf)
    old_inbuf = inbuf
    old_inoffset = inoffset
    inbuf = ""
    inoffset = 0
    # Don't checkpoint the following write...
    queueCommand("resume %s %d %d\n" % (token, jobid, lastreceived), 0)
  else:
    sys.exit(1)

#=============================================================================
# Functions to deal with reader/writer queues.
#=============================================================================
cpbuf=""
cpoffset=0
def checkpoint(str):
  global cpbuf
  cpbuf = cpbuf + str

def advance(offset):
  global cpbuf
  global cpoffset
  
  if offset < cpoffset:
    print "Big advance problem: offset(%d) < cpoffset(%d)" % (offset,cpoffset)
  elif offset > cpoffset + len(cpbuf):
    print "Big advance problem: offset(%d) > cpoffset(%d) + len(%d)" % (offset, cpoffset, len(cpbuf))
  else:
    diff = offset - cpoffset
    cpbuf = cpbuf[diff:]
    cpoffset = offset
    #if cpbuf == "":
    #  print "cpbuf empty. good."
    #else:
    #  print "cpbuf len = %d" % len(cpbuf)

def replay(offset):
  global cpbuf
  global cpoffset

  if offset < cpoffset:
    print "Big replay problem: offset(%d) < cpoffset(%d)" % (offset,cpoffset)
  elif offset > cpoffset + len(cpbuf):
    print "Big replay problem: offset(%d) > cpoffset(%d) + len(%d)" % (offset, cpoffset, len(cpbuf))
  else:
    diff = offset - cpoffset
    cpbuf = cpbuf[diff:]
    cpoffset = offset
    # Don't checkpoint this write:
    queueCommand("replay %d %d\n" % (cpoffset,len(cpbuf)), 0)
    queueCommand(cpbuf,0)

def queueWrite(x, str):
  global writers
  if not writers.has_key(x):
    writers[x] = ""
  writers[x] += str

def queueCommand(str, cpt=1):
  debug("queueCommand '%s'" % str)
  if sock:
    queueWrite(sock, str)
  if cpt:
    checkpoint(str)

def deleteReader(i):
  if readers.has_key(i):
    del readers[i]

def deleteWriter(i):
  if writers.has_key(i):
    del writers[i]

#=============================================================================
# Stop the application.
#=============================================================================
def appExit(status, errors=None):
  try:
    debug("appExit(%d,%s)" % (status,errors))
    sock.send("exit\n")
    sock.send("ackexit\n")
    sock.close()
  except:
    pass
  if errors:
    ttyWrite(errors)
  for name in [ "!stdin", "!stdout", "!stderr" ]:
    try:
      closeFile(nametofile[name])
    except:
      pass
  sys.exit(status)

def handleSignal(sig, frame):
  #print "Caught signal %d.  Clean shutdown." % sig
  global child_has_exited
  global ready_to_exit

  # If this is a SIGCHLD, its only purpose is to interupt select().
  if sig == signal.SIGCHLD:
    child_has_exited
    return

  if local_flag:
    if child_pid:
      try:
        os.kill(child_pid, signal.SIGHUP)
        time.sleep(5)
        os.kill(child_pid, signal.SIGTERM)
        time.sleep(5)
        os.kill(child_pid, signal.SIGKILL)
      except:
        pass
  else:
    ready_to_exit = 1
    queueCommand("localexit 1 0 0\n")
    queueCommand("ackexit\n")
    appExit(0)

def syntheticNetworkPartition(sig, frame):
  ttyWrite("Caught signal %d.  Simulating a network failure." % sig)
  try:
    sock.close()
  except:
    pass

#=============================================================================
# Update the resources file.  (to set a session token)
#=============================================================================
def updateResources():
  try:
    home=os.environ["HOME"]
    fname=os.path.join(home,".default_resources")
    f=open(fname,"w")
    f.write("# This file was created by 'submit'\n")
    f.write("username %s\n" % username)
    f.write("session_token %s\n" % token)
    f.close()
    ttyWrite("Updated %s" % fname)
  except:
    pass

def getUsername():
  global username

  username = getpass._raw_input("Username: (%s) " % username)
  if username == "":
    username = getpass.getuser()

def getPassword():
  global password

  password = getpass.getpass("Password: ")

def signon():
  global password
  queueCommand("username %s\n" % username)
  if token:
    queueCommand("token %s\n" % token)
  else:
    # Don't send an empty password.
    if password == "":
      password = "*"
    queueCommand("password %s\n" % password)
  queueCommand("signon\n")

#=============================================================================
# Move output files back from the server.
#=============================================================================
def moveOutputFiles():
  for name in output_files.keys():
    try:
      debug("pulling file %s" % name)
      dir=os.path.dirname(name)
      try:
        os.path.getctime(dir)
      except:
        try:
          os.makedirs(dir)
        except:
          log("Unable to create directory: %s" % dir)
          continue
      f=open(name,"w")
      mapfile(f,name)
      output_files[name] = f
    except:
      ttyWrite("Unable to open %s for transfer: %s" % (name,sys.exc_info()[0]))
    else:
      debug("exportfile %s" % name)
      queueCommand("exportfile %s\n" % name)

#=============================================================================
# Interpret a command from the network.
#=============================================================================
def eat(ilen):
  global inoffset
  global inbuf
  inoffset = inoffset + ilen
  inbuf = inbuf[ilen:]

def uneat(str):
  global inoffset
  global inbuf
  #print "UNEAT", str
  inoffset = inoffset - len(str)
  inbuf = str + inbuf

def doCommand(more):
  global inbuf
  global old_inbuf
  global inoffset
  global ready_to_exit
  global exit_status
  global token
  global jobid

  debug("doCommand +%d" % len(more))

  inbuf = inbuf + more
  cmdcount = 0
  ackcount = 0
  while 1:
    cmdcount = cmdcount + 1

    try:
      nl=inbuf.index("\n")
    except:
      break

    arg=inbuf[0:nl].split()
    line = inbuf[0:nl]
    eat(nl+1)

    debug("Looking at " + line)

    try:
      if arg[0] == "null":
        queueCommand("null\n")
        debug("null")

      elif arg[0] == "authz":
        authz = int(arg[1])
        if authz == 0:
          log("Session authentication failed")
          token=None
          getUsername()
          getPassword()
          signon()
        elif authz == -1:
          appExit(1)
        else:
          if local_flag:
            queueCommand("startlocal\n")
          else:
            queueCommand("startcmd\n")

      elif arg[0] == "jobid":
        jobid = int(arg[1])
        if jobid == 0:
          sys.exit(1)
        if print_metrics:
          print "=SUBMIT-METRICS=> job=%d" % jobid

      elif arg[0] == "token":
        token = arg[1]
        updateResources()

      elif arg[0] == "server_ready_for_io":
        if local_flag:
          startLocal()
        else:
          readers[sys.stdin] = "!stdin"
          mapfile(sys.stdin,  "!stdin")
          mapfile(sys.stdout, "!stdout")
          mapfile(sys.stderr, "!stderr")

      elif arg[0] == "importfile":
        name = arg[1]
        if not input_files.has_key(name) and name != command_args[0]:
          log("Improper attempt by server to read %s" % name)
        else:
          f = open(name,"r")
          mapfile(f,name)
          readers[f] = name

      elif arg[0] == "message":
        mlen = int(arg[1])
        if len(inbuf) < mlen:
          uneat(line + "\n")
          debug("Had to uneat it.")
          ackcount=0
          break
        else:
          debug("Eat it.")
          buf = inbuf[0:mlen]
          eat(mlen)
          ttyWrite(buf)

      elif arg[0] == "write":
        name = arg[1]
        ilen = int(arg[2])
        if len(inbuf) < ilen:
          uneat(line + "\n")
          debug("Had to uneat it.")
          ackcount=0
          break
        else:
          debug("Eat it.")
          buf = inbuf[0:ilen]
          eat(ilen)
          if not nametofile.has_key(name):
            debug("file %s was closed too early" % name)
          else:
            queueWrite(nametofile[name], buf)

      elif arg[0] == "close":
        name = arg[1]
        writeclose[nametofile[name]] = 1
        #print "Set to close", name

      elif arg[0] == "ack":
        sendoffset = int(arg[1])
        recvoffset = int(arg[2])
        ackcount = ackcount + 1
        #print "ack %d" % offset
        if recvoffset != inoffset - (len(line) + 1):
          print "INBUF IS OUT OF SYNC"
          print "I think recvoffset=%d" % (inoffset - (len(line) + 1))
          print "However recvoffset=%d" % recvoffset
        advance(sendoffset)

      elif arg[0] == "resume":
        offset = int(arg[1])
        replay(offset)
        try:
          readers[nametofile["!stdin"]] = "!stdin"
        except:
          pass

      elif arg[0] == "replay":
        roffset = int(arg[1])
        rlen = int(arg[2])
        if len(inbuf) < rlen:
          uneat(line + "\n")
          cmdcount = -1 # Make sure we don't send an "ack"
          break
        else:
          buf = inbuf[0:rlen]
          eat(rlen)
          #print "Old inoffset is %d" % old_inoffset
          #print "Replay offset is %d" % roffset
          #print "Old inbuf is '%s'" % old_inbuf
          #print "Replay is '%s'" % buf
          #print "New inbuf is '%s'" % inbuf
          #print "================="
          old_inbuf = old_inbuf + buf + inbuf
          inbuf = old_inbuf
          inoffset = roffset
          old_inbuf = ""

      elif arg[0] == "childexit":
        status = int(arg[1])
        #print "Exiting %d" % status
        for w in writers.keys():
          #print "Flushing ", w
          writeOutput(w)
        exit_status = status

      elif arg[0] == "declare_output_file":
        output_files[arg[1]] = 0

      elif arg[0] == "get_output_files":
        moveOutputFiles()

      elif arg[0] == "exit":
        appExit(exit_status)

      elif arg[0] == "ackexit":
        appExit(exit_status)

      else:
        print "Unrecognized client command:", line

    except (IndexError,ValueError):
      print "Unknown client command:", line

  if ackcount < cmdcount:
    recvoffset = inoffset + len(inbuf)
    sendoffset = cpoffset + len(cpbuf)
    debug("Sending ack %d %d" % (recvoffset,sendoffset))
    queueCommand("ack %d %d\n" % (recvoffset,sendoffset))

#=============================================================================
#=============================================================================
def closeFile(f):
  if filetoname.has_key(f):
    name = filetoname[f]
    debug("Closing %s" % name)
    if (print_metrics or debug_flag) and (name=="!stdout" or name=="!stderr"):
      debug("Must keep outputs open.")
      return
    del filetoname[f]
  else:
    name = "!unknown"
  if writers.has_key(f):
    if writers[f] != "":
      try:
        os.write(f.fileno(), writers[f])
      except:
        pass
    del writers[f]
  if readers.has_key(f):
    del readers[f]
  if nametofile.has_key(name):
    del nametofile[name]
  if input_files.has_key(name):
    del input_files[name]
  if output_files.has_key(name):
    del output_files[name]
    if output_files.keys() == []:
      appExit(exit_status)
  try:
    f.close()
  except:
    pass

#=============================================================================
# Read an input.
#=============================================================================
def readInput(i):
  ret = ""

  if i == sock:
    while 1:
      ret = ""
      try:
        ret = i.recv(1024)
        if ret == "":
          closeFile(i)
          return
      except socket.error, err:
        # Happens on non-blocking TCP socket when there's nothing to read
        #print "Client read err: %s" % str(err)
        break
      except (SSL.WantReadError, SSL.WantWriteError, SSL.WantX509LookupError):
        break
      except SSL.ZeroReturnError, errors:
        break
      except SSL.Error, errors:
        break
      else:
        doCommand(ret)

  else:
    try:
      ret = os.read(i.fileno(), 1024)
      if ret == "":
        queueCommand("close %s\n" % readers[i])
        closeFile(i)
        return
    except:
      appExit(1,"Read failed: %s" % str(i))
    else:
      if ret == "":
        queueCommand("close %s\n" % readers[i])
        deleteReader(i)
        return
      queueCommand("read %s %d\n" % (readers[i],len(ret)))
      queueCommand(ret)

#=============================================================================
# Write to an output that's ready.
#=============================================================================
def writeOutput(f):
  ret = ""

  name = filetoname[f]

  if name == "!socket":
    try:
      ret = sock.send(writers[f])
    except socket.error, err:
      # Happens on non-blocking TCP socket when there's nothing to read
      #print "Write error: %s" % str(err)
      pass
    except (SSL.WantReadError, SSL.WantWriteError, SSL.WantX509LookupError):
      pass
    except SSL.ZeroReturnError:
      appExit(1,"Write failed: %s" % str(f))
    except SSL.Error, errors:
      log("Connection died unexpectedly.")
      appExit(1,str(errors))
    else:
      writers[f] = writers[f][ret:]
      if writers[f] == "":
        del writers[f]

  else:
    if writers[f] == "":
      closeFile(f)
      return

    try:
      ret = os.write(f.fileno(), writers[f])
    except KeyError:
      # This happens at exit, when the output is closed.  Ignore it.
      debug("Bad writer key:" + str(f))
      pass
    except exceptions.SystemExit:
      # This happens at exit, after closing the last output_file.
      sys.exit(exit_status)
    except exceptions.OSError:
      log("Error: %s for %s" % (sys.exc_info()[0], str(f)))
      handleSignal(signal.SIGINT, None)  # This will clean up and exit.
    except:
      log("Error: %s for %s" % (sys.exc_info()[0], str(f)))
    else:
      if writers.has_key(f):
        writers[f] = writers[f][ret:]
        if writers[f] == "":
          del writers[f]

#=============================================================================
# Parse interesting things out of the resources file.
#=============================================================================
def parseResources(rfile):
  global token
  global testfile_file
  global testfile_inode
  global connect_list

  try:
    f=open(rfile)
    while 1:
      line=f.readline()
      if line=="":
        break
      try:
        if line.startswith("session_token "):
          token=line.split()[1]
        if line.startswith("username"):
          username=line.split()[1]
        if line.startswith("submit_target "):
          #print "Appending " + line.split()[1]
          connect_list.append(line.split()[1])
      except:
        pass
    f.close()
    testfile_file = rfile
    (mode,inode,dev,nlinks,uid,gid,size,a,m,c)=os.lstat(rfile)
    testfile_inode = inode
  except:
    pass

  return token

#=============================================================================
# Collect all the process' context (args, env, pwd) to be sent over.
#=============================================================================
def sendContext():
  if testfile_inode:
    queueCommand("testfile %d %s\n" % (testfile_inode, testfile_file))

  if not token:
    getUsername()
    getPassword()

  for a in sys.argv[0:]:
    queueCommand("arg %d\n" % len(a))
    queueCommand(a)

  for e in os.environ:
    queueCommand("var %d %d\n" % (len(e), len(os.environ[e])))
    queueCommand(e + os.environ[e])

  umask = os.umask(0)
  os.umask(umask)
  queueCommand("umask 0%o\n" % umask)
  queueCommand("pwd %s\n" % os.getcwd())
  signon()

#=============================================================================
# Look at the command line arguments.
#=============================================================================
def parseArgs():
  global local_flag
  global command_args
  global validate_args
  global debug_flag
  global help_flag
  global print_metrics

  #print "sys.argv=" + str(sys.argv)

  local_flag=0
  i=0
  while i+1 < len(sys.argv):
    i=i+1
    #print "Parsing argv[%d]=%s" % (i,sys.argv[i])
    if sys.argv[i] == "-d" or sys.argv[i] == "--debug":
      debug_flag = 1
      del sys.argv[i]  # Make it look like the argument is not there
      i=i-1            # Retreat the argument array index
    elif sys.argv[i] == "-e":
      i=i+1
    elif sys.argv[i] == "-h" or sys.argv[i] == "--help":
      help_flag=1
    elif sys.argv[i] == "-i" or sys.argv[i] == "--inputfile":
      i=i+1
      input_files[sys.argv[i]] = 0
    elif sys.argv[i] == "-l" or sys.argv[i] == "--local":
      local_flag=1
    elif sys.argv[i] == "-m" or sys.argv[i] == "--manager":
      i=i+1
    elif sys.argv[i] == "-M" or sys.argv[i] == "--metrics":
      print_metrics = 1
    elif sys.argv[i] == "-N" or sys.argv[i] == "--ppn":
      i=i+1
    elif sys.argv[i] == "-n" or sys.argv[i].lower() == "--ncpus":
      i=i+1
    #elif sys.argv[i] == "-o":
    #  i=i+1
    #  output_files[sys.argv[i]] = 0
    elif sys.argv[i] == "-v" or sys.argv[i] == "--venue":
      i=i+1
    elif sys.argv[i] == "-W" or sys.argv[i] == "--wait":
      pass
    elif sys.argv[i] == "-w" or sys.argv[i] == "--walltime":
      i=i+1
    elif sys.argv[i] == "-r" or sys.argv[i] == "--redundancy":
      i=i+1
    else:
      if sys.argv[i].startswith("-"):
        appExit(1,"Argument '%s' is not recognized" % sys.argv[i])
      else:
        command_args = sys.argv[i:]
        validate_args = sys.argv[0:i]
        return

#=============================================================================
# Check the arguments for correctness.
#=============================================================================
def validateArgs():
  if not help_flag:
    if command_args == None:
      appExit(1, "No command.")
    for arg in validate_args:
      if local_flag and (arg == "-v" or arg == "--venue"):
        appExit(1, "Combining the --local flag with --venue is not allowed.")

#=============================================================================
# Start a local command.
#=============================================================================
def startLocal():
  global child_pid
  global start_time
  
  #print "Starting: " + str(command_args)
  start_time = time.time()
  child_pid = os.fork()
  if child_pid == 0:
    sock.close()
    os.environ['SUBMIT_JOB'] = "%s" % jobid
    try:
      os.execvpe(command_args[0], command_args, os.environ)
    except OSError, e:
      log("Failed to invoke " + command_args[0] + ': ' + e.args[1])
      sys.exit(126)
  else:
    debug("Child pid is %d" % child_pid)

#=============================================================================
# Main program begins here...
#=============================================================================
sys.exechook = info

#=============================================================================
# Load configuration values
#=============================================================================
for configdir in configdirs:
  try:
    execfile(os.path.join(configdir,"config"))
    debug("Using configdir %s" % configdir)
    break
  except:
    pass


#=============================================================================
# Parse arguments and resource file.
#=============================================================================
parseArgs()
validateArgs()

resource_search_list = [ ]

if os.environ.has_key("SESSIONDIR"):
  resource_search_list.append(os.environ["SESSIONDIR"])

# Why add current working directory?
resource_search_list.append(os.getcwd())

res_found=0
for resource_dir in resource_search_list:
  if parseResources(os.path.join(resource_dir, "resources")):
    res_found=1
    break

if res_found==0:
  if parseResources(os.path.join( os.environ["HOME"], ".default_resources")):
    res_found=1

if not res_found:
  if not sys.stdin.isatty():
    log("Unable to find a resources file.")
    sys.exit(1)

#=============================================================================
# Set up SSL context
#=============================================================================
ctx = SSL.Context(SSL.TLSv1_METHOD)
ctx.set_verify(SSL.VERIFY_PEER, verify_cb) # Demand a certificate
rfile = os.path.join(configdir, 'submit_server_ca.crt')
if os.access(rfile, os.R_OK):
  ctx.load_verify_locations(rfile)

#=============================================================================
# Start the client and send parameters to the server.
#=============================================================================
setupSocket()
sendContext()

signal.signal(signal.SIGHUP, handleSignal)
signal.signal(signal.SIGINT, handleSignal)  # ctrl-c
signal.signal(signal.SIGTERM, handleSignal) # signal 15
signal.signal(signal.SIGQUIT, syntheticNetworkPartition) # ctrl-\
signal.signal(signal.SIGCHLD, handleSignal)

#=============================================================================
# Main loop.  Wait for events.
#=============================================================================
while 1:
  for x in writeclose.keys():
    if not writers.has_key(x):
      #print "Marking for close: " + str(x)
      writers[x] = ""
      del writeclose[x]
    else:
      #print "Can't mark for close yet:" + str(x)
      pass

  try:
    #print "select", readers.keys(), writers.keys()
    if child_has_exited:
      select_timeout = .1
    else:
      select_timeout = 15*60
    r,w,_ = select.select(readers.keys(), writers.keys(), [], select_timeout)
    #
    # If the timeout has occurred (nothing to read/write) send a keepalive.
    #
    if r == [] and w == []:
      if writers.has_key("!socket"):
        queueCommand("null\n")
      continue
  except exceptions.SystemExit:
    pass
  except select.error, err:
    # benign
    r = []
    w = []
    pass
  except socket.error, err:
    r = []
    w = []
    if err[0] == errno.EBADF:
      restoreSocket()
      pass
    else:
      log("Unknown select error:", err)
      break
  except:
    log("Select exception: %s" % sys.exc_info()[0])
    break
  
  for i in r:
    readInput(i)

  for o in w:
    writeOutput(o)

  if ready_to_exit:
    appExit(exit_status)

  if child_pid:
    try:
      (pid,status) = os.waitpid(child_pid, os.WNOHANG)
    except:
      pass

    if pid != 0:
      status = status >> 8
      debug("child exited with status 0x%x" % status)
      child_pid = None
      exit_status = status
      rusage = resource.getrusage(resource.RUSAGE_CHILDREN)
      realtime = time.time() - start_time
      cputime = rusage[0] + rusage[1]
      debug("localexit %d %f %f" % (status, cputime, realtime))
      queueCommand("localexit %d %f %f\n" % (status, cputime, realtime))
      if print_metrics:
        msg="=SUBMIT-METRICS=>"
        msg += " job=%d" % jobid
        msg += " venue=local"
        msg += " status=%d" % status
        msg += " cputime=%f" % cputime
        msg += " realtime=%f" % realtime
        print msg

#=============================================================================
# Done with main loop.  Exit.
#=============================================================================
appExit(1, "Fell out of main loop.")
