#!/usr/bin/python


# ============================================================================
#
#                               Preamble
#
# ============================================================================

from optparse import OptionParser
try:
    import sqlite3
except ImportError:
    # pre 2.5.x
    from pysqlite2 import dbapi2 as sqlite3
import sys, os, math, re
import numpy

from glue import git_version
from glue.ligolw import table
from glue.ligolw import lsctables
from glue.ligolw import dbtables

from pylal import InspiralUtils
from pylal import ligolw_sqlutils as sqlutils
from pylal import printutils
from pylal.ligolw_dataUtils import get_row_stat

__author__ = "Sarah Caudill <sarah.caudill@ligo.org>"
__prog__ = "ligolw_cbc_plotsngl"

description = \
"Plots single-ifo triggers from a search results database"

# ============================================================================
#
#                               Set Options
#
# ============================================================================

def parse_command_line():
    """
    Parser function dedicated
    """
    parser = OptionParser(
        version = git_version.verbose_msg,
        usage   = "%prog -x var1 [options] file1.sqlite file2.sqlite ...",
        description = description
        )
    # following are related to file input and output naming
    parser.add_option( "-t", "--tmp-space", action = "store", type = "string", default = None,
        metavar = "PATH",
        help =
            "Path to the local disk for operating on .sqlite files. Optional, "
            "used to enhance performance in a networked environment. "
        )
    parser.add_option( "-P", "--output-path", action = "store", type = "string",
        default = os.getcwd(), metavar = "PATH",
        help = 
            "Optional. Path where the figures should be stored. Default is current directory."
        )
    parser.add_option( "-O", "--enable-output", action = "store_true",
        default =  False, metavar = "OUTPUT",
        help = 
            "Enable the generation of html and cache documents"
        )
    parser.add_option( "-u", "--user-tag", action = "store", type = "string",
        help =
            "Set a user-tag for plot and html naming."
        )
    parser.add_option( "-d", "--datatypes", action = "store", type = "string",
        help =
            "Required. What datatype to plot. Can be either 'injection', 'slide', 'all_data', 'playground', "
            "or 'exclude_play'. If 'injection', --simulation-table must be specified and found injections"
            "will be plotted. To specify multiple datatypes, comma separate the types."
        )
    parser.add_option( "-x", "--x-var", action = "store", type = "string",
        metavar = "x_var[:label]",
        help = 
            "Required. What variable to plot on the x-axis. Any column in the single table can be used. "
            "Math operations are also supported, e.g. 'chisq/chisq_dof'."
            "All functions in the python math module can be used. Syntax is standard python; e.g., log(snr, 10) "
            "will return the log10 of the snr. To specify a label for the axis, type :label after the variable, e.g., "
            "'chisq: $\chi^2$'. LaTeX is supported when placed between two $ signs (if using a bash terminal, "
            "escape each dollar sign appropriately). "
            "One special label is gps_(s|min|hr|days|yr). If given, the stat will be assumed to be a gps time and the units will be plotted in "
            "(s|min|hr|days|yr) since the earliest experiment start time in the databases. For example, if you type end_time:gps_days, "
            "and the earilest start time is 987654321, the x-axis will be 'End Time ($987654320 + days$)'. "
        )
    parser.add_option( "-y", "--y-var", action = "store", type = "string",
        metavar = "y_var[:label]",
        help = 
            "Required. What variable to plot on the y-axis. Any column in the single table can be used. "
            "Math operations are also supported. Syntax is the same as --x-var option."
        )
    parser.add_option( "", "--x-ifo", action = "store", type = "string",
        help = 
            "Required. What ifo to plot on the x-axis."
        )
    parser.add_option( "", "--y-ifo", action = "store", type = "string",
        help = 
            "What ifo to plot on the y-axis. If none specified, then x-ifo will be used."
        )
    parser.add_option( "-r", "--ranking-stat", action = "store", default = "snr",
        help =
            "Stat to rank found injections with. The stat must be a column in the recovery table; default is snr. "
            "This is needed to select the best match if multiple events are mapped to the same injection."
        )
    parser.add_option( "-b", "--rank-by", action = "store", default = "MAX",
        help =
            "How to rank the injections. Options are 'MAX' or 'MIN', default is 'MIN'. If set to 'MIN', any trigger "
            "with a 0 ranking_stat value will be plotted with a star as opposed to a circle. If set to 'MAX', any "
            "trigger with an infinite value will be plotted with a star."
        )
    parser.add_option( "-p", "--param-name", metavar = "PARAMETER[:label]",
        action = "store", default = None,
        help =
            "Specifying this and param-ranges will only select triggers that fall within the given range for each plot. "
            "Any column in the single table may be used. "
            "As with x-var, math operations are permitted between columns, e.g. "
            "mass1+mass2. Any function in the python math module may be used; syntax is python. The parameter name "
            "will be in the title of the plot; to give a label for the name, add a colon and the label. If no label is specified, "
            "the parameter name as given will be used."
        )
    parser.add_option( "-q", "--param-ranges", action = "store", default = None,
        metavar = " [ LOW1, HIGH1 ); ( LOW2, HIGH2]; !VAL3; etc.",
        help = 
            "Requires --param-name. Specify the parameter ranges "
            "to select triggers in. A '(' or ')' implies an open "
            "boundary, a '[' or ']' a closed boundary. To specify "
            "multiple ranges, separate each range by a ';'. To "
            "specify a single value, just type that value with no "
            "parentheses or brackets. To specify not equal to a single "
            "value, put a '!' before the value. If "
            "multiple ranges are specified, a separate plot for each range will be generated."
        )
    parser.add_option( "-s", "--sim-tag", action = "store", type = "string", default = 'ALLINJ',
        help =
            "If plotting injections, "
            "specify the simulation type to plot, e.g., 'BNSLININJ'. To plot multiple injection types together "
            "separate tags by a '+', e.g., 'BNSLOGINJA+BNSLOGINJB'. "
            "If not specified, will group all injections together (equivalent to specifying 'ALLINJ')."
        )
    parser.add_option('-X', '--logx', action = 'store_true', default = False,
        help =
            'Make x-axis logarithmic. Note: this will apply to all plots generated.'
        )
    parser.add_option('-Y', '--logy', action = 'store_true', default = False,
        help =
            'Make y-axis logarithmic. Note: this will apply to all plots generated.'
        )
    parser.add_option('', '--xmin', action = 'store', type = 'float', default = None,
        help =
            'Set a minimum value for the x-axis. If logx set, must be > 0. This will apply to all plots generated.'
        )
    parser.add_option('', '--xmax', action = 'store', type = 'float', default = None,
        help =
            'Set a maximum value for the x-axis. If logx set, must be > 0. This will apply to all plots generated.'
        )
    parser.add_option('', '--ymin', action = 'store', type = 'float', default = None,
        help =
            'Set a minimum value for the y-axis. If logy set, must be > 0. This will apply to all plots generated.'
        )
    parser.add_option('', '--ymax', action = 'store', type = 'float', default = None,
        help =
            'Set a maximum value for the y-axis. If logy set, must be > 0. This will apply to all plots generated.'
        )
    parser.add_option('-f', '--plot-x-function', action = 'append', default = [], metavar = 'f(x)[:label]',
        help =
            "Plot the curve y=f(x) where f(x) is the given function of x. As with --x-var, any function in the python math module may "
            "be used; syntax is python; use the variable 'x'. The domain is evenly "
            "spaced across the space spanned by the x-axis. Ex.: to plot the sine of pi times x, "
            "type 'sin(pi*x)'. To plot 'log10(x)' type 'log(x, 10)'. To plot y=x type 'x'. To plot a horizontal line "
            "at some value, say y=6, just type '6'. To plot multiple functions, give the argument multiple times. To put a label "
            "for the function on the plot(s), add a colon and the label. As with variables, LaTeX is supported by enclosing in $ signs."
        )
    parser.add_option('-g', '--plot-y-function', action = 'append', default = [], metavar = 'g(y)[:label]',
        help =
            "Plot the curve x=g(y) where g(y) is the given function of y. Syntax as for plot-x-function; "
            "use the variable 'y'. The domain is evenly spaced across the space spanned "
            "by the y-axis. Ex.: to plot a vertical line at x=6, type '6'."
        )
    parser.add_option( "-S", "--single-table", action = "store", type = "string", default = None,
        help =
            "Required. Table to look in for single parameters. "
            "Can be any lsctable with a event_id and ifo column."
        )
    parser.add_option( "", "--simulation-table", action = "store", type = "string", default = None,
        help =
            "Required if plotting injections. Table to look in for injection parameters. "
            "Can be any lsctable with a simulation_id."
        )
    parser.add_option( "-R", "--recovery-table", action = "store", type = "string",
        help =
            "Table to look in for recovered injections. "
            "Can be any lsctable with a coinc_event_id."
        )
    parser.add_option( "-M", "--map-label", action = "store", type = "string", default = None,
        help =
            "Required if plotting injections. Name of the type of mapping that was used for finding injections."
        )
    parser.add_option( "", "--show-plot", action = "store_true", default = False,
        help = 
            "Display the plots on the terminal"
        )
    parser.add_option( "", "--dpi", action = "store", type = "int", default = 150,
        help = 
            "Set the plots' dpi. Default is 150."
        )
    parser.add_option( "-v", "--verbose", action = "store_true", default = False,
        help = 
            "Be verbose."
        )

    (options,filenames) = parser.parse_args()

    #check if required options specified and for self-consistency
    required_options = ["x_var", "y_var", "x_ifo", "datatypes"]
    missing_options = [option for option in required_options if getattr(options, option) is None]
    if missing_options:
        raise ValueError, "missing required option(s) %s" % ", ".join("--%s" % option.replace("_", "-") for option in missing_options)

    options.datatypes = options.datatypes.split(",")
    if 'injection' in options.datatypes and options.simulation_table is None:
        raise ValueError, "Must specify --simulation-table for injection datatype."

    if options.logx and (options.xmin > 0 and options.xmax > 0):
        raise ValueError, "If --logx desired, x-limits must be > 0."
    if options.logy and (options.ymin > 0 and options.ymax > 0):
        raise ValueError, "If --logy desired, y-limits must be > 0."

    return options, filenames, sys.argv[1:]

# =============================================================================
#
#                       Function Definitions
#
# =============================================================================

def select_best_inj_match(found_table, selection_stat, select_by, use_match_rank = True):
    """
    If an injection is mapped to multiple events, picks out the best match via
    some selection stat. If use_match_rank is True, will always use the match_rank
    column first, thus only using the selection_stat as a tie-breaker. For example,
    if the found_table was generated using combined_far, combined_far determines the
    match_rank entry and therefore is the primary selection criteria. In the event
    that two events have the same match rank -- they had the same far -- the selection_stat
    is then used to pick one of the two. In the event that both triggers have the same
    selection stat, the first one in the table is used.

    @found_table: a SelectedFound table created by printsims
    @selection_stat: any recovered stat in the found_table, e.g., recovered_snr
    @select_by: must be 'MAX' or 'MIN'; whether to use max or min values of selection_stat
    @use_match_rank: toggle whether or not to use match_rank
    """

    # cycle through the table, getting coinc_event_ids of events to delete
    delete_ceids = []
    # get all events with multiple matches
    simid_index = dict([ [row.simulation_id, [all_rows for all_rows in found_table if all_rows.simulation_id == row.simulation_id]]
        for row in found_table ])

    for sim_id, matchlist in simid_index.items():
        if len(matchlist) > 1:
            # apply selection criteria
            delete_these = [ row.coinc_event_id for row in matchlist
                if use_match_rank and row.recovered_match_rank != 1
                or select_by == "MAX" and getattr(row, 'recovered_'+selection_stat) != max([getattr(x, 'recovered_'+selection_stat) for x in matchlist])
                or select_by == "MIN" and getattr(row, 'recovered_'+selection_stat) != min([getattr(x, 'recovered_'+selection_stat) for x in matchlist])
                ]
            # check that only have one event left, if not, just keep the first non-deleted event in the list
            if len(matchlist) - len(delete_these) > 1:
                add_these = [ match.coinc_event_id for match in matchlist if match.coinc_event_id not in delete_these ]
                add_these.pop(0)
                delete_these.extend( add_these )
            # add to list to be deleted
            delete_ceids.extend( delete_these )

    # remove the deletes
    [ found_table.remove(row) for row in found_table[::-1] if row.coinc_event_id in delete_ceids ]

# ============================================================================
#
#                                 Main
#
# ============================================================================

#
#   Generic Initialization
#

# parse command line
opts, filenames, args = parse_command_line()
# parse the variables
x=opts.x_var
y=opts.y_var
x_var, xlabel = len(x.split(':')) == 2 and x.split(':') or [x.split(':')[0], x.split(':')[0].replace('_', ' ').title() ]
y_var, ylabel = len(y.split(':')) == 2 and y.split(':') or [y.split(':')[0], y.split(':')[0].replace('_', ' ').title() ]
xlabel, ylabel = xlabel.strip(), ylabel.strip()

xifo = opts.x_ifo
if opts.y_ifo is None:
    yifo = opts.x_ifo
else:
    yifo = opts.y_ifo

sngl_table = sqlutils.validate_option(opts.single_table)

# get rank_by
rank_by = sqlutils.validate_option(opts.rank_by, lower = False).upper()
# check if binning by param-range at all
if opts.param_name is not None:
    param_name, param_label = len(opts.param_name.split(':')) == 2 and opts.param_name.split(':') or [opts.param_name, opts.param_name.replace('_', ' ').title()]
    param_name = sqlutils.validate_option( param_name, lower = False )
    param_label = param_label.strip()
    param_parser = sqlutils.parse_param_ranges('', param_name, opts.param_ranges, verbose = opts.verbose)
    num_subgroups = len(param_parser.param_ranges)
else:
    param_name = None
    num_subgroups = 1

sqlite3.enable_callback_tracebacks(True)

#
#   Plotting Initialization
#

# Change to Agg back-end if show() will not be called to avoid display problem
if not opts.show_plot:
  import matplotlib
  matplotlib.use('Agg')
import pylab
pylab.rc('text', usetex=True)

#
#   Program-specific Initialization
#

# get available instrument times
experiments = {}
if opts.verbose:
    print >> sys.stderr, "Opening database(s) and checking for instrument times..."
for filename in filenames:
    working_filename = dbtables.get_connection_filename(
        filename, tmp_path = opts.tmp_space, verbose = opts.verbose )
    connection = sqlite3.connect( working_filename )
    if opts.tmp_space:
        dbtables.set_temp_store_directory(connection, opts.tmp_space, verbose = opts.verbose)
    sqlquery = "SELECT DISTINCT instruments, gps_start_time, gps_end_time FROM experiment"
    for on_insts, gps_start, gps_end in connection.cursor().execute(sqlquery):
        current_times = experiments.setdefault( frozenset(lsctables.instrument_set_from_ifos(on_insts)), (int(gps_start), int(gps_end)) )
        if current_times is not None:
            update_start, update_end = current_times
            if update_start > int(gps_start):
                update_start = int(gps_start)
            if update_end < int(gps_end):
                update_end = int(gps_end)
            experiments[ frozenset(lsctables.instrument_set_from_ifos(on_insts)) ] = (update_start,update_end)
    # only close the connection if there are more than one database being used
    if len(filenames) > 1:
        connection.close()
        dbtables.discard_connection_filename( filename, working_filename, verbose = opts.verbose)

# cycle over available instrument times
fignum = 0
fnameList = []
tagList = []
for on_instruments, (gps_start_time, gps_end_time) in experiments.items():
    on_instr = r','.join(sorted(on_instruments))
    if opts.verbose:
        print >> sys.stderr, "Creating plots for %s time..." % on_instr

    # dictionary to store statistics
    stats = {}

    for filename in filenames:
        if len(filenames) > 1:
            working_filename = dbtables.get_connection_filename(
                filename, tmp_path = opts.tmp_space, verbose = opts.verbose )
            connection = sqlite3.connect( working_filename )
            if opts.tmp_space:
                dbtables.set_temp_store_directory(connection, opts.tmp_space, verbose = opts.verbose)

        # create a class to store values from the table
        # get sngl table columns
        sngl_table_cols = sqlutils.get_column_names_from_table( connection, sngl_table )
        class StorageTable(table.Table):
            tableName = "storage:table"
            validcolumns = {}
            for col_name in sngl_table_cols:
                if col_name in lsctables.TableByName[sngl_table].validcolumns.keys():
                    validcolumns[col_name] = lsctables.TableByName[sngl_table].validcolumns[col_name]
                # if custom columns exist in the database, just set them to lstrings
                else:
                    validcolumns[col_name] = "lstring"

        class Storage(object):
            __slots__ = StorageTable.validcolumns.keys()

            def get_effective_snr(self, fac=250.0):
                snr = self.snr
                rchisq = float(self.chisq) / (2.*self.chisq_dof - 2.)
                return snr/ (1 + snr**2/fac)**(0.25) / rchisq**(0.25)

            def get_new_snr(self,index=6.0):
                rchisq = float(self.chisq) / (2.*self.chisq_dof - 2.)
                nhigh = 2.
                if rchisq > 1.:
                    return self.snr/ ((1+rchisq**(index/nhigh))/2)**(1./index)
                else:
                    return self.snr

            def get_gps_time(self):
                if 'start_time' in self.__slots__:
                    return self.start_time
                elif 'end_time' in self.__slots__:
                    return self.end_time
                else:
                    raise AttributeError, "could not find a coinc table start_time or end_time"

            @property
            def new_snr(self):
                return self.get_new_snr()

            @new_snr.setter
            def new_snr(self):
                err_msg = "The new_snr property cannot be set directly."
                raise ValueError(err_msg)

            @property
            def effective_snr(self):
                return self.get_effective_snr()

            @effective_snr.setter
            def effective_snr(self):
                err_msg = "The effective_snr property cannot be set directly."
                raise ValueError(err_msg)


            def get_pyvalue(self):
                if self.value is None:
                    return None
                return ligolwtypes.ToPyType[self.type or "lstring"](self.value)

        # connect the row to the table
        StorageTable.RowType = Storage

        for datatype in opts.datatypes:
            if opts.verbose:
                print >> sys.stderr, "\tgetting %s..." % datatype
            stats.setdefault(datatype, [])
            if datatype == "injection":

                    # get found table
                    if opts.verbose:
                        print >> sys.stderr, "\tgetting 'found' triggers from %s..." % filename

                    found_table = \
                        printutils.printsims(connection, opts.simulation_table, opts.recovery_table, opts.map_label,
                        opts.ranking_stat, opts.rank_by,
                        "slide", param_name = None, param_ranges = None,
                        include_only_coincs = '[ALL in %s]' % on_instr,
                        sim_tag = opts.sim_tag, verbose = False)

                    # select best matches using snr
                    select_best_inj_match(found_table, opts.ranking_stat, rank_by, use_match_rank = True)

                    found_table = printutils.get_sngl_info(connection, found_table, sngl_table, verbose = opts.verbose)

                    # the single-table columns will have 'sngl_' appended to them; we need to remove this for get_row_stat to work
                    for thisrow in found_table:
                        cprow = Storage()
                        [setattr(cprow, col_name, getattr(thisrow, 'sngl_'+col_name)) for col_name in sngl_table_cols]
                        xval = get_row_stat(cprow, x_var)
                        yval = get_row_stat(cprow, y_var)
                        if opts.param_name is not None:
                            pval = get_row_stat(cprow, opts.param_name)
                        else:
                            pval = None
                        stats[datatype].append( (cprow.ifo, pval, xval, yval) )

            else:
                sqlquery = ''.join(['''
                    SELECT
                        ''',sngl_table,'''.*
                    FROM
                        ''',sngl_table,'''
                    JOIN
                        experiment,experiment_summary,experiment_map,coinc_event_map
                    ON(
                        experiment.experiment_id == experiment_summary.experiment_id AND
                        experiment_summary.experiment_summ_id == experiment_map.experiment_summ_id AND
                        experiment_map.coinc_event_id == coinc_event_map.coinc_event_id AND
                        coinc_event_map.event_id == ''',sngl_table,'''.event_id)
                    WHERE
                        experiment.instruments == "''',on_instr,'''" AND
                        experiment_summary.datatype == "''',datatype,'"'])
                for data in connection.cursor().execute(sqlquery).fetchall():
                    this_row = Storage()
                    [setattr(this_row, col_name, x) for (col_name,x) in zip(sngl_table_cols,data)]
                    xval = get_row_stat(this_row, x_var)
                    yval = get_row_stat(this_row, y_var)
                    if opts.param_name is not None:
                        pval = get_row_stat(this_row, opts.param_name)
                    else:
                        pval = None
                    stats[datatype].append( (this_row.ifo, pval, xval, yval) )
        #
        # Plot
        #
        # set InspiralUtils options for file and plot naming
        if opts.verbose:
            print >> sys.stderr, "\tplotting..."
        opts.gps_start_time = int(gps_start_time)
        opts.gps_end_time = int(gps_end_time)
        opts.ifo_times = ''.join(sorted(on_instruments))
        opts.ifo_tag = ''
        InspiralUtilsOpts = InspiralUtils.initialise( opts, __prog__, git_version.verbose_msg )

        pylab.figure(fignum)

        for datatype,thesestats in stats.items():
            thesestats = [val for val in thesestats if val[0] == xifo or val[0] == yifo]
            # get desired plot values
            plotvals=[(val[2],val[3]) for val in thesestats]

            # if logx, remove any 0 valued things
            if opts.logx:
                popis = [ j for j, (statval, _) in enumerate(plotvals) if statval == 0 ]
                for j in popis[::-1]:
                    del plotvals[j]
            # ditto logy
            if opts.logy:
                popis = [ j for j, (_, statval) in enumerate(plotvals) if statval == 0 ]
                for j in popis[::-1]:
                    del plotvals[j]

            # set time unit
            is_time_x = re.match(r'gps_(s|min|hr|days|yr)', xlabel)
            if is_time_x is not None:
                plotvals = [ (sqlutils.convert_duration( x - gps_start_time + 1, is_time_x.group(1) ), y) for (x,y) in plotvals ]
            is_time_y = re.match(r'gps_(s|min|hr|days|yr)', ylabel)
            if is_time_y is not None:
                plotvals = [ (x, sqlutils.convert_duration( y - gps_start_time + 1, is_time_y.group(1) )) for (x,y) in plotvals ]

            xval = [val[0] for val in plotvals]
            yval = [val[1] for val in plotvals]

            if datatype == "injection":
                c = 'red'
                marker = '+'
            if datatype == "slide":
                c = 'blue'
                marker = 'x'
            if datatype == "all_data":
                c = 'black'
                marker = 'x'
            if datatype == "playground":
                c = 'green'
                marker = 'x'
            if datatype == "exclude_play":
                c = 'lightcyan'
                marker = 'x'

            pylab.scatter( xval, yval, c=c, marker=marker, s = 40, linewidth = .5, alpha = 1., label = '_nolegend_' )
            if opts.xmax is not None:
                xmax = opts.xmax
            else:
                xmax = max(xval)
            if opts.xmin is not None:
                xmin = opts.xmin
            else:
                xmin = min(xval)
            if opts.ymax is not None:
                ymax = opts.ymax
            else:
                try:
                    ymax = max(yval)
                except:
                    raise
            if opts.ymin is not None:
                ymin = opts.ymin
            else: 
                ymin = min(yval)

            # if nothing to plot, just plot a warning message
            if plotvals == []:
                pylab.text(0.5,0.5, 'Nothing to plot.')
                xmin = ymin = 0
                xmax = ymax = 1
                no_data = True
            else:
                no_data = False

            # FIXME
            # In a bunch of places below two variables are used extensively,
            # but are undefined. No idea what they are supposed to mean so
            # setting these to False.
            x_is_string = False
            y_is_string = False

            # plot any x-functions
            if opts.plot_x_function != [] and not no_data:
                safe_dict = dict([ [name,val] for name,val in math.__dict__.items() if not name.startswith('__') ])
                if opts.logx and not x_is_string:
                    x_vals = numpy.logspace( math.log(xmin,10), math.log(xmax,10), num=1000, endpoint=True, base=10.0 )
                    xtext = xmax * 10**-.5
                else:
                    x_vals = numpy.linspace( xmin, xmax, num=1000, endpoint = True )
                    xtext = xmax-.1*(xmax-xmin)
                for f in opts.plot_x_function:
                    f, flabel = len(f.split(':')) == 2 and f.split(':') or [f, False]
                    y_vals = [eval( f, {"__builtins__":None, 'x':x}, safe_dict ) for x in x_vals]
                    pylab.plot( x_vals, y_vals, 'k--' )
                    ymin = min(y_vals) < ymin and min(y_vals) or ymin
                    ymax = max(y_vals) > ymax and max(y_vals) or ymax
                    if flabel:
                        x,y = ( x_vals[len(y_vals)-y_vals[::-1].index(max(y_vals))-1], max(y_vals) )
                        if opts.logy and not y_is_string:
                            ytext = max(y_vals) * 10**.25
                        else:
                            ytext = max(y_vals)+.05*(ymax-ymin)
                        pylab.annotate( '$y=$'+flabel, xy=(x,y), xytext = (xtext,ytext), arrowprops=dict(arrowstyle='->'))

            # plot any y-functions
            if opts.plot_y_function != [] and not no_data:
                safe_dict = dict([ [name,val] for name,val in math.__dict__.items() if not name.startswith('__') ])
                if opts.logy and not y_is_string:
                    y_vals = numpy.logspace( math.log(ymin,10), math.log(ymax,10), num=1000, endpoint=True, base=10.0 )
                    ytext = ymax * 10**.25
                else:
                    y_vals = numpy.linspace( ymin, ymax, num=1000, endpoint = True )
                    ytext = ymax+.05*(ymax-ymin)
                for g in opts.plot_y_function:
                    g, glabel = len(g.split(':')) == 2 and g.split(':') or [g, False]
                    x_vals = [eval( g, {"__builtins__":None, 'y':y}, safe_dict ) for y in y_vals]
                    pylab.plot( x_vals, y_vals, 'k--' )
                    xmin = min(x_vals) < xmin and min(x_vals) or xmin
                    xmax = max(x_vals) > xmax and max(x_vals) or xmax
                    if glabel:
                        x,y = ( max(x_vals), y_vals[len(x_vals)-x_vals[::-1].index(max(x_vals))-1] )
                        if opts.logx and not x_is_string:
                            xtext = max(x_vals) * 10**-.5
                        else:
                            xtext = max(x_vals)-.1*(xmax-xmin)
                        pylab.annotate( '$x=$'+glabel, xy=(x,y), xytext = (xtext,ytext), arrowprops=dict(arrowstyle='->'))

            #
            # Set plot parameters
            #

            t = "%s Time: %s" % (on_instr, opts.sim_tag.replace('_', '\_'))
            if param_name is not None:
                t = "%s %s %s" % (t, param_label, param_parser.param_range_by_group(n))
            pylab.title( t )

            # set x-axis parameters
            if is_time_x is not None:
                lbl = '%s ($%i + %s$)' % (xlabel.replace('_', ' ').title(), gps_start_time-1, is_time_x.group(1))
            else:
                lbl = xlabel
            pylab.xlabel(lbl)

            if opts.logx and not no_data and not x_is_string:
                pylab.gca().semilogx()

            # set y-axis parameters
            if is_time_y is not None:
                lbl = '%s ($%i + %s$)' % (ylabel.replace('_', ' ').title(), gps_start_time-1, is_time_y.group(1))
            else:
                lbl = ylabel
            pylab.ylabel(lbl)

            if opts.logy and not no_data and not y_is_string:
                pylab.gca().semilogy()

            # set x/y limits
            if opts.logx and not no_data and not x_is_string:
                xmin, xmax = ( xmin * 10**-.5, xmax * 10**.5 )
            elif x_is_string:
                xmin = min(tagdictx.values()) - 1
                xmax = max(tagdictx.values()) + 1
            else:
                xmin = opts.xmin is not None and not no_data and opts.xmin or xmin-.1*(xmax-xmin)
                xmax = opts.xmax is not None and not no_data and opts.xmax or xmax+.1*(xmax-xmin)

            pylab.xlim( xmin, xmax )

            if x_is_string:
                pylab.gca().set_xticks( [xmin]+sorted(tagdictx.values())+[xmax] )
                pylab.gca().set_xticklabels([n in rev_tagdictx and rev_tagdictx[n].replace('_', '\_') or '' for n in pylab.gca().get_xticks()],
                    size = 'small', rotation = 45)

            if opts.logy and not no_data:
                ymin, ymax = ( ymin * 10**-.5, ymax * 10**.5 )
            elif y_is_string:
                ymin = min(tagdicty.values()) - 1
                ymax = max(tagdicty.values()) + 1
            else:
                ymin = opts.ymin is not None and not no_data and opts.ymin or ymin-.1*(ymax-ymin)
                ymax = opts.ymax is not None and not no_data and opts.ymax or ymax+.1*(ymax-ymin)

            pylab.ylim( ymin, ymax )

            if y_is_string:
                pylab.gca().set_yticks( [ymin]+sorted(tagdicty.values())+[ymax] )
                pylab.gca().set_yticklabels([n in rev_tagdicty and rev_tagdicty[n].replace('_', '\_') or '' for n in pylab.gca().get_yticks()],
                    size = 'xx-small')

            pylab.grid()

            if opts.enable_output:
                param_tag = (param_name is not None) and param_label or ''
                if param_name is not None:
                    param_tag = '_'.join([ re.sub(r'\W', '', param_tag),
                        str(param_parser.param_ranges[param_group][0][1]),
                        str(param_parser.param_ranges[param_group][1][1]), '' ])
                plot_description = '%sF%i' % ( param_tag, fignum )
                #plot_description = '_'.join([ param_tag, re.sub( r'\W', '', ystat ), 'v', re.sub( r'\W', '', xstat ) ])
                name = InspiralUtils.set_figure_tag( plot_description,
                    datatype_plotted = opts.sim_tag, open_box = False)
                fname = InspiralUtils.set_figure_name(InspiralUtilsOpts, name)
                fname_thumb = InspiralUtils.savefig_pylal( filename=fname, dpi=opts.dpi )
                fnameList.append(fname)
                tagList.append(name)

            fignum = fignum + 1

    #
    #   Create the html page for this instrument time
    #

    if opts.enable_output:
        if opts.verbose:
            print >> sys.stdout, "\twriting html file and cache."

        # create html of closed box plots
        # FIXME: *Another* undefined variable, how did this code ever work?
        #        Setting this to "FAR" as this is the case it is catching.
        cbstat = 'FAR'
        comment_vals = (rank_by == "MAX" and "infinite" or "zero", cbstat.replace('_',' '))
        comment = "<b>Plot Key:</b> Stars are injections found with %s %s. " % comment_vals + \
            "Circles are injections found with non-%s %s. " % comment_vals + \
            "Red crosses (if plotted) are missed injections (vetoed injections are excluded)."
        plothtml = InspiralUtils.write_html_output( InspiralUtilsOpts, args, fnameList,
            tagList, comment = comment, add_box_flag = False )
        InspiralUtils.write_cache_output( InspiralUtilsOpts, plothtml, fnameList )

    if opts.show_plot:
        pylab.show()

    #
    # Close the figures and clear memory for the next instrument time
    #

    # Don't care if any of this fails
    for number in range(fignum):
        try:
            pylab.close(number)
        except:
            pass

    try:
        del found_table
    except:
        pass
    # How can I delete a table that doesn't exist?
    #del missed_table

    #
    #   Finished cycling over experiments; exit
    #
    if len(filenames) == 1:
        connection.close()
        dbtables.discard_connection_filename( filename, working_filename, verbose = opts.verbose)

if opts.verbose:
    print >> sys.stdout, "Finished!"
sys.exit(0)
