#!/usr/bin/python
#
# Copyright (C) 2006  Kipp Cannon, Sarah Caudill
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.


#
# =============================================================================
#
#                                   Preamble
#
# =============================================================================
#


"""
Translate rinca-style sngl_ringdown coincs to coinc-tables-style coincs.
"""


import itertools
import math
from optparse import OptionParser
import re
import sys
import os

from glue import iterutils
from glue import lal
from glue import segments
from glue.ligolw import ligolw
from glue.ligolw import table
from glue.ligolw import lsctables
from glue.ligolw import utils
from glue.ligolw.utils import ligolw_add
from glue.ligolw.utils import search_summary as ligolw_search_summary
from glue.ligolw.utils import segments as ligolw_segments
from glue.ligolw.utils import process as ligolw_process
from pylal import git_version
from pylal import llwapp
from pylal import ligolw_rinca
from pylal import ligolw_tisi
from pylal import SnglInspiralUtils


__author__ = "Kipp Cannon <kipp.cannon@ligo.org>"
__version__ = "git id %s" % git_version.id
__date__ = git_version.date


#
# =============================================================================
#
#                              Behavioural Tweaks
#
# =============================================================================
#


#
# Make sngl_ringdown table rows hashable so they can be used as dictionary
# keys and uniquified with set()
#


def __sngl_ringdown_hash__(self):
	# The things in this tuple must be the same things used in the
	# __cmp__() method for comparison (if two objects have different
	# hashes they must compare as not equal or stuff breaks), so make
	# sure to keep this updated if the choice of how to compare to
	# triggers changes.
	return hash((self.ifo, self.start_time, self.start_time_ns, self.frequency, self.quality))


lsctables.SnglRingdown.__hash__ = __sngl_ringdown_hash__


#
# Disable the custom ID remapping attached to the SnglRingdownTable class
# (use the generic ID mapping provided by the parent class instead)
#


# del lsctables.SnglRingdownTable.updateKeyMapping


#
# =============================================================================
#
#                                 Command Line
#
# =============================================================================
#


def parse_command_line():
	parser = OptionParser(
		version = "Name: %%prog\n%s" % git_version.verbose_msg,
		usage = "%prog [options] --output-prefix filename",
		description = "Converts rinca output files to coinc tables format.  If an ihope cache is provided then all first-stage rinca files found in the cache will be processed, grouped by run tag.  Otherwise, one or both of a zero-lag or time-slide rinca output file must be provided."
	)
	parser.add_option("--zero-lag-file", metavar = "filename", help = "Rinca output file containing zero lag triggers.")
	parser.add_option("--time-slide-file", metavar = "filename", help = "Rinca output file containing time slide triggers.")
	parser.add_option("--ihope-cache", metavar = "filename", help = "Get rinca files from this ihope cache.")
	parser.add_option("--instruments", metavar = "name[,name,...]", help = "Set the list of instruments.  Example H1,H2,L1.  If not provided, attempts will be made to deduce the instrument list either from --??-slide command line options in the process_params table or from the contents of the ifos column in the search_summary table.")
	parser.add_option("--output-prefix", metavar = "string", help = "Set the prefix string for output filenames (required). Output filenames are constructed automatically, and are of the form \"PREFIX_blah_blah_blah.xml.gz\". Hint: can include a path.")
	parser.add_option("--veto-segments", metavar = "filename", help = "Load veto segments from this XML document.  See ligolw_segments for information on constructing such a document.")
	parser.add_option("--veto-segments-name", metavar = "string", help = "Set the name of the veto segments to use from the XML document.")
	parser.add_option("--statistic", metavar = "string", help = "Choose which detection statistic to use, effective_snr, new_snr, or snr.") 
	parser.add_option("--effective-snr-factor", metavar = "float", type = "float", default = 250.0, help = "Set the effective SNR factor (default = 250.0).")
	parser.add_option("--chisq-index", metavar = "float", type = "float", default = 6.0, help = "Set the new snr chisq index (default = 6.0).")
	parser.add_option("--lars-id", metavar = "string", type = "string", default = None, help = "Set the lars_id for this experiment in the experiment table (optional).")
	parser.add_option("--search", metavar = "string", type = "string", default = None, help = "Set the search, e.g., 'low_mass', for this experiment in the experiment table (optional).")
	parser.add_option("--experiment-start-time", type="int", help = "Start time, in gps seconds, of the experiment being performed. This is NOT the start time of the rinca-files being used. Required.")
	parser.add_option("--experiment-end-time", type="int", help = "End time, in gps seconds, of the experiment being performed. This is NOT the end time of the rinca-files being used. Required.")
	parser.add_option("--simulation", action = "store_true", default = False, help = "Turn on if thinca file contains injections. Doing so will causes only zero-lag entries to be written to the experiment_summary table whose datatype is 'simulation'.")
	parser.add_option("-v", "--verbose", action = "store_true", help = "Be verbose.")
	options, args = parser.parse_args()

	if options.ihope_cache:
		if options.zero_lag_file or options.time_slide_file:
			raise ValueError, "cannot specify --zero-lag-file or --time-slide-file with --ihope-cache"
		if options.verbose:
			print >>sys.stderr, "reading %s ..." % options.ihope_cache
		# find first rinca cache entries
		pattern = re.compile(r"RINCA(?P<slide>(_SLIDE)?)_FIRST_+(((?P<taga>.+)_CAT_(?P<cat>[\d]+)_VETO)|(?P<tagb>.+))\Z")
		metadata_cache = {}
		zero_lag_urls = {}
		slide_urls = {}
		for n, c in enumerate(lal.CacheEntry(line) for line in file(options.ihope_cache)):
			if options.verbose and not n % 1597:
				print >>sys.stderr, "\t%d lines\r" % n,
			if "RING_FIRST" in c.url:
				metadata_cache[os.path.basename(c.path)] = c.url
			category = pattern.match(c.description)
			if category is None:
				continue
			category = category.groupdict()
			if category["slide"] == "":
				# zero-lag URL
				zero_lag_urls.setdefault((category["taga"] or category["tagb"], category["cat"] or "1"), []).append(c)
			else:
				# slide URL
				slide_urls.setdefault((category["taga"] or category["tagb"], category["cat"] or "1"), []).append(c)
		if options.verbose:
			print >>sys.stderr
		# extract URLs.  sorting cache entries puts them in time
		# order
		zero_lag_urls = dict((key, [c.url for c in sorted(cache)]) for key, cache in zero_lag_urls.items())
		slide_urls = dict((key, [c.url for c in sorted(cache)]) for key, cache in slide_urls.items())
		# pad lists with None's so zero-lag and slide lists exist
		# for all the same categories and have the same number of
		# entries
		for key in set(zero_lag_urls) | set(slide_urls):
			if key not in slide_urls:
				# must be in zero-lag
				slide_urls[key] = [None,] * len(zero_lag_urls[key])
			elif key not in zero_lag_urls:
				# must be in slides
				zero_lag_urls[key] = [None,] * len(slide_urls[key])
			# must be in both, make sure lengths are the same
			elif len(zero_lag_urls[key]) > len(slide_urls[key]):
				slide_urls[key] += [None,] * (len(zero_lag_urls[key]) - len(slide_urls[key]))
			elif len(zero_lag_urls[key]) < len(slide_urls[key]):
				zero_lag_urls[key] += [None,] * (len(slide_urls[key]) - len(zero_lag_urls[key]))
		# match up each zero-lag url with corresponding slides urls
		# in each category
		categories = dict((key, zip(zero_lag_urls[key], slide_urls[key])) for key in zero_lag_urls)
	elif options.zero_lag_file or options.time_slide_file:
		metadata_cache = None
		categories = {(None, None): [(options.zero_lag_file, options.time_slide_file)]}
	else:
		raise ValueError, "must specify --ihope-cache or at least one of --zero-lag-file and --time-slide-file"
	if options.veto_segments and not options.veto_segments_name:
		raise ValueError, "must specify --veto-segments-name if --veto-sements is specified"
	if options.instruments:
		options.instruments = set(options.instruments.split(","))
	if not options.experiment_start_time or not options.experiment_end_time:
		raise ValueError, "must specify --experiment-start-time and --experiment-end-time"

	if args:
		raise ValueError, "extraneous command line arguments specified"

	return options, categories, metadata_cache


#
# =============================================================================
#
#                              Metadata Retrieval
#
# =============================================================================
#


#
# content handler used to remove the sngl_ringdown table from input
# documents
#


class MetadataContentHandler(ligolw.FilteringLIGOLWContentHandler):
	def __init__(self, xmldoc):
		ligolw.FilteringLIGOLWContentHandler.__init__(self, xmldoc, lambda name, attrs: not lsctables.IsTableProperties(lsctables.SnglRingdownTable, name, attrs))


try:
	lsctables.use_in(MetadataContentHandler)
except AttributeError:
	# old glue did not allow .use_in().
	# FIXME:  remove when we can require the latest version of glue
	pass


#
# uses a dictionary mapping filename to URL to retrieve the process
# metadata for the lalapps_ring jobs whose triggers are found in
# xmldoc.  re-assigns each trigger's process_id to indicate the
# lalapps_ring job that generated it.
#


def add_metadata(xmldoc, metadata_cache, verbose = False):
	if verbose:
		print >>sys.stderr, "retrieving lalapps_ring metadata ..."

	#
	# choose the metadata URLs for this document.  because the input
	# includes search_summ_vars rows from both the slides and
	# non-slides rinca jobs, each url gets listed twice so we use
	# set() to uniquify the list.
	#

	process_ids = table.get_table(xmldoc, lsctables.ProcessTable.tableName).get_ids_by_program("rinca")
	urls = sorted(set(metadata_cache[row.string] for row in table.get_table(xmldoc, lsctables.SearchSummVarsTable.tableName) if row.process_id in process_ids and row.name == "input_file"))

	#
	# insert the metadata from the URLs into xmldoc, remapping IDs to
	# prevent collision with existing information
	#

	for tbl in xmldoc.getElementsByTagName(ligolw.Table.tagName):
		tbl.sync_next_id()
	xmldoc = ligolw_add.ligolw_add(xmldoc, urls, verbose = verbose, contenthandler = MetadataContentHandler)

	#
	# check for no-op and unusable documents
	#

	try:
		table.get_table(xmldoc, lsctables.SnglRingdownTable.tableName)
	except ValueError:
		# no ringdown table --> no-op
		if verbose:
			print >>sys.stderr, "cannot find sngl_ringdown table"
		return xmldoc

	#
	# index lalapps_ring entries in the search_summary table
	#

	if verbose:
		print >>sys.stderr, "re-linking sngl_ringdown rows to process rows ..."
	process_ids = table.get_table(xmldoc, lsctables.ProcessTable.tableName).get_ids_by_program("lalapps_ring")
	segs = {}
	ids = {}
	for row in table.get_table(xmldoc, lsctables.SearchSummaryTable.tableName):
		if row.process_id in process_ids:
			segs.setdefault(row.ifos, segments.segmentlist()).append(row.get_out())
			ids.setdefault(row.ifos, []).append(row.process_id)

	#
	# assign each sngl_ringdown to its (presumably) original
	# lalapps_ring job
	#

	for row in table.get_table(xmldoc, lsctables.SnglRingdownTable.tableName):
		row.process_id = ids[row.ifo][segs[row.ifo].find(row.get_start())]

	#
	# done
	#

	return xmldoc


#
# =============================================================================
#
#                             Process Information
#
# =============================================================================
#


#
# create and initialize this job's row in the process table
#


def initialize_process(xmldoc, comment = u""):
	return ligolw_process.append_process(xmldoc, program = u"ligolw_rinca_to_coinc", version = __version__, cvs_repository = u"lscsoft", cvs_entry_time = __date__, comment = comment)


#
# record command line arguments
#


def set_process_params(xmldoc, process, options):
	params = []
	if options.zero_lag_file:
		params.append((u"--zero-lag-file", u"lstring", options.zero_lag_file))
	if options.time_slide_file:
		params.append((u"--time-slide-file", u"lstring", options.time_slide_file))
	if options.ihope_cache:
		params.append((u"--ihope-cache", u"lstring", options.ihope_cache))
	if options.instruments:
		params.append((u"--instruments", u"lstring", ",".join(options.instruments)))
	if options.veto_segments:
		params.append((u"--veto-segments", u"lstring", options.veto_segments))
	if options.veto_segments_name:
		params.append((u"--veto-segments-name", u"lstring", options.veto_segments_name))
	if options.statistic:
		params.append((u"--statistic", u"lstring", options.statistic))
	if options.effective_snr_factor:
		params.append((u"--effective-snr-factor", u"lstring", options.effective_snr_factor))
	if options.chisq_index:
		params.append((u"--chisq-index", u"lstring", options.chisq_index))
	if options.output_prefix:
		params.append((u"--output-prefix", u"lstring", options.output_prefix))

	ligolw_process.append_process_params(xmldoc, process, params)

	return xmldoc


#
# =============================================================================
#
#                               Document Fix-Up
#
# =============================================================================
#


#
# add missing ID columns.  this is needed because the cbc 2yr tags are on
# versions of LAL that don't have these columns in the tables.  current
# versions of LAL do.
#


def add_missing_id_columns(xmldoc, verbose = False):
	if verbose:
		print >>sys.stderr, "adding any missing ID columns ...",

	#
	# fix summ_value and search_summ_vars tables
	#

	for name in (lsctables.SummValueTable.tableName, lsctables.SearchSummVarsTable.tableName):
		try:
			tbl = table.get_table(xmldoc, name)
			tbl.appendColumn(tbl.next_id.column_name)
		except ValueError:
			# document doesn't have this table or already has
			# ID column
			continue
		if verbose:
			print >>sys.stderr, name,
		for row in tbl:
			setattr(row, tbl.next_id.column_name, tbl.get_next_id())

	#
	# done
	#

	if verbose:
		print >>sys.stderr


#
# Fix ifos columns by turning strings like "H1H2L1" into strings like
# "H1,H2,L1".
#


def fix_ifos_columns(xmldoc, verbose = False):
	if verbose:
		print >>sys.stderr, "fixing ifos columns ...",

	#
	# fix process and search_summary tables
	#

	for table_name in (lsctables.ProcessTable.tableName, lsctables.SearchSummaryTable.tableName):
		try:
			tbl = table.get_table(xmldoc, table_name)
		except ValueError:
			# document doesn't have this table
			continue
		if options.verbose:
			print >>sys.stderr, table_name,
		for row in tbl:
			row.set_ifos(row.get_ifos())

	#
	# done
	#

	if verbose:
		print >>sys.stderr


#
# =============================================================================
#
#                          Populate time_slide Table
#
# =============================================================================
#


def get_rinca_process_ids(xmldoc):
	return table.get_table(xmldoc, lsctables.ProcessTable.tableName).get_ids_by_program("rinca")


def get_search_summary_instruments(xmldoc, rinca_process_ids):
	"""
	Extract the list of analyzed instruments from the rinca entries in
	the search summary table.
	"""
	search_summary_instruments = set(frozenset(row.get_ifos()) for row in table.get_table(xmldoc, lsctables.SearchSummaryTable.tableName) if row.process_id in rinca_process_ids)
	if len(search_summary_instruments) < 1:
		raise ValueError, "cannot find entries for rinca jobs in search_summary table"
	if len(search_summary_instruments) > 1:
		raise ValueError, "search_summary table contains entries for rinca jobs from more than 1 unique set of instruments:  found %s" % ", ".join(search_summary_instruments)
	return search_summary_instruments.pop()


def populate_rinca_time_slide_table(xmldoc, process, instruments = None, verbose = False):
	"""
	Reconstruct the list of time slides from lalapps_rinca's command
	line arguments.
	"""
	if verbose:
		print >>sys.stderr, "populating rinca time_slide table ...",

	#
	# find the time_slide table or add one if needed
	#

	try:
		time_slide_table = table.get_table(xmldoc, lsctables.TimeSlideTable.tableName)
	except ValueError:
		time_slide_table = lsctables.New(lsctables.TimeSlideTable)
		xmldoc.childNodes[0].appendChild(time_slide_table)

	#
	# move existing time_slide IDs out of the way
	#

	# find the lowest unused ID not less than 10000 and set next_id to
	# that value
	time_slide_table.sync_next_id()
	if time_slide_table.next_id < type(time_slide_table.next_id)(10000):
		time_slide_table.set_next_id(type(time_slide_table.next_id)(10000))
	# use the updateKeyMapping method to re-label all existing
	# time_slide IDs and record the old-->new mapping
	mapping = {}
	time_slide_table.updateKeyMapping(mapping)
	# apply the mapping to all other tables in the document to update
	# any references to existing time_slide IDs
	for tbl in xmldoc.getElementsByTagName(time_slide_table.tagName):
		tbl.applyKeyMapping(mapping)

	#
	# get process_ids for all rinca jobs, and for rinca jobs that
	# were run with a --num-slides command line option
	#

	rinca_process_ids = get_rinca_process_ids(xmldoc)
	slides_process_ids = rinca_process_ids & set(row.process_id for row in table.get_table(xmldoc, lsctables.ProcessParamsTable.tableName) if row.param == u"--num-slides")

	if len(rinca_process_ids - slides_process_ids) > 1:
		raise ValueError, "document contains more than 1 zero-lag rinca job"
	if len(slides_process_ids) > 1:
		raise ValueError, "document contains more than 1 non-zero-lag rinca job"

	#
	# get the list of analyzed instruments from the search_summary
	# table
	#

	search_summary_instruments = get_search_summary_instruments(xmldoc, rinca_process_ids)

	#
	# if a set of instruments has been provided by the user require it
	# to be a superset of the instruments named in the search_summary
	# table, either way the union of the two is the required instrument
	# list
	#

	if instruments is None:
		required_instruments = search_summary_instruments
	elif search_summary_instruments.issubset(instruments):
		required_instruments = instruments
	else:
		raise ValueError, "search_summary table named instrument(s) %s that were not named on the command line" % ", ".join(search_summary_instruments - instruments)

	#
	# identify lalapps_rinca's time slides
	#

	if not slides_process_ids:
		#
		# just zero-lag jobs.  synthesize an all-zero vector from
		# the instruments we've retrieved from the search_summary
		# table, and set num_slides to 0
		#

		offset_vector = dict((instrument, 0.0) for instrument in required_instruments)
		num_slides = 0

	else:
		#
		# construct the offset vector from the --??-slide command
		# line options, and extact num_slides from the --num-slides
		# command line option
		#

		num_slides = None
		offset_vector = {}
		slide_option_pattern = re.compile("--(?P<ifo>[a-zA-Z][0-9])-slide")

		for row in table.get_table(xmldoc, lsctables.ProcessParamsTable.tableName):
			if row.process_id in slides_process_ids:
				if row.param == u"--num-slides":
					num_slides = row.pyvalue
					if num_slides < 0:
						raise ValueError, "invalid --num-slides '%s' for process '%s'" % (row.value, row.process_id)
				else:
					match = re.search(slide_option_pattern, row.param)
					if match is not None:
						offset_vector[match.groupdict()["ifo"].upper()] = row.pyvalue

		#
		# confirm that the offset vector contains offsets for all
		# required instruments
		#

		if not required_instruments.issubset(set(offset_vector)):
			missing_instruments = set(offset_vector) - required_instruments
			raise ValueError, "no option(s) %s in process_params table for instrument(s) %s in search_summary table" % (", ".join(["--%s-slide" % instrument.lower() for instrument in missing_instruments]), ", ".join(missing_instruments))

		#
		# if the user has provided an explicit list of instruments,
		# remove instruments from the offset vector that were not
		# named by the user
		#

		if instruments is not None:
			for instrument in set(offset_vector) - instruments:
				del offset_vector[instrument]

	#
	# build the time slide vectors for the cases when all instruments
	# are on.  these must be given time_slide IDs that match the slide
	# number component of the old-style event_id
	#

	def ids(id_class, num_slides):
		for n in range(-num_slides, +num_slides + 1):
			if n < 0:
				yield id_class(5000 - n)
			else:
				yield id_class(n)

	n = 0
	for id, offset_vector in zip(ids(type(time_slide_table.next_id), num_slides), ligolw_tisi.Inspiral_Num_Slides_Iter(num_slides, offset_vector)):
		n += 1
		for instrument, offset in offset_vector.items():
			row = time_slide_table.RowType()
			row.process_id = process.process_id
			row.time_slide_id = id
			row.instrument = instrument
			row.offset = offset
			time_slide_table.append(row)

	#
	# done
	#

	if verbose:
		print >>sys.stderr, "added %d time slide vectors" % n


def depopulate_time_slide_table(xmldoc, verbose = False):
	"""
	Search for and remove duplicate time slide definitions from the
	time_slide table.
	"""
	if verbose:
		print >>sys.stderr, "depopulating time_slides ...",

	#
	# find the time_slide table
	#

	time_slide_table = table.get_table(xmldoc, lsctables.TimeSlideTable.tableName)

	length_before = len(set(time_slide_table.getColumnByName("time_slide_id")))

	#
	# translate time_slide table into a dictionary, and identify
	# redundant IDs
	#
	# NOTE:  the time slide vector comparison code treats {"H1": 0,
	# "L1": 5} and {"H1": 10, "L1": 15} as identical vectors because
	# the relative offsets are identical.  in the inspiral pipeline,
	# these are potentially different time slides because one of the
	# two can result in the pair of triggers straddling a ring boundary
	# while the other does not, so a trigger pair can be coincident for
	# one of these vectors but not the other.  this should never be an
	# issue because the pipeline also has the limitation of only being
	# able to apply vectors that are all multiples of a fixed basis
	# vector --- if two instruments have the same relative offsets in
	# two vectors then they must also have the same absolute offsets in
	# those vectors, so comparing by relative offsets yields the same
	# set of redundant vectors that a comparison by absolute offsets
	# would yield.  the use of the time_slides_vacuum() function here
	# is correct, but one should keep the reason why in mind if one is
	# tempted to copy-and-paste this code elsewhere.
	#

	mapping = ligolw_tisi.time_slides_vacuum(time_slide_table.as_dict())

	#
	# remove rows corresponding to redundant IDs
	#

	for i in xrange(len(time_slide_table) - 1, -1, -1):
		if time_slide_table[i].time_slide_id in mapping:
			del time_slide_table[i]

	#
	# reassign time_slide IDs in the rest of the document
	#

	for tbl in xmldoc.getElementsByTagName(time_slide_table.tagName):
		tbl.applyKeyMapping(mapping)

	#
	# done
	#

	if verbose:
		length_after = len(set(time_slide_table.getColumnByName("time_slide_id")))
		print >>sys.stderr, "removed %d redundant time slide vectors, %d remaining" % (length_before - length_after, length_after)


#
# =============================================================================
#
#          Initialize experiment and experiment_summ tables 
#
# =============================================================================
#


def populate_experiment_table(xmldoc, search_group, search, lars_id, instruments, expr_start_time, expr_end_time, comments = None, add_inst_subsets = False, verbose = False):
	"""
	Populate the experiment table using the given entries. If
	add_inst_subsets is set to True, will write additional
	entries for every possible sub-combination of the given instrument
	set. Returns a dictionary of experiment_ids keyed by the instrument
	set.

	@xmldoc: xmldoc to get/write table to
	@lars_id: lars_id of the experiment
	@search_group: lsc group that performed the experiment (e.g., cbc)
	@search: type of search performed (e.g., inspiral, grb, etc.)
	@expr_start_time: start time of the experiment (not of the file)
	@expr_end_time: end time of the experiment (not of the file)
	@comments: any desired comments
	@add_inst_subsets: will write an entry for every possible subset
		of @instruments
	@verbose: be verbose
	"""

	if verbose:
		print >> sys.stderr, "populating the Experiment table..."

	# find the experiment table or create one if needed
	try:
		expr_table = table.get_table(xmldoc, lsctables.ExperimentTable.tableName)
	except ValueError:
		expr_table = xmldoc.childNodes[0].appendChild(lsctables.New(lsctables.ExperimentTable))
	
	experiment_ids = {}

	# write entry to the experiment table for the given instruments if it doesn't already exist
	experiment_ids[frozenset(instruments)] =  expr_table.write_new_expr_id( search_group, search, lars_id, instruments, expr_start_time, expr_end_time, comments = comments )

	#  add every possible sub-combination of the instrument set if 
	# they're not already in the table 
	if add_inst_subsets:
		for nn in range(2, len(instruments) ):
			for sub_combo in iterutils.choices( list(instruments), nn ):
				if frozenset(sub_combo) not in experiment_ids:
					experiment_ids[frozenset(sub_combo)] = expr_table.write_new_expr_id(search_group, search, lars_id, sub_combo, expr_start_time, expr_end_time, comments = comments)

	return experiment_ids


def populate_experiment_summ_table( xmldoc, experiment_id, time_slide_dict, veto_def_name, simulation = False, return_dict = False, verbose = False ):
	"""
	Populate the experiment_summ_table using an experiment_id, a
	veto_def_name, and a list of time_slide ids. 

	@xmldoc: xmldoc to get/write table to
	@experiment_id: experiment_id to be added to the table.
	@veto_def_name: veto_def_name to be added to the table.
	@time_slide_dict: time_slide table as dictionary; used to set time_slide_id
	column and figure out whether or not is zero-lag. Can either be the result
	of lsctables.time_slide_table.as_dict or any dictionary having same format. 
	@simulation: if set to True, will only write a single, zero-lag row with datatype
		set to simulation. If false, will write entries for all slides, as well as
		zero-lag all_data, playground, and exclude_play.
	@return_dict: will return the experiment_summary table as an id_dict
	"""

	if verbose:
		print >> sys.stderr, "populating the Experiment Summary table..."

	# find the experiment_summary table or create one if needed
	try:
		expr_summ_table = table.get_table(xmldoc, lsctables.ExperimentSummaryTable.tableName)
	except ValueError:
		expr_summ_table = xmldoc.childNodes[0].appendChild(lsctables.New(lsctables.ExperimentSummaryTable))
	
	# populate the experiment_summary table
	if simulation:
		# for now, set sim_proc_id to None; this is updated by later programs once the injection
		# file is added to the database
		for slide_id in time_slide_dict:
			if not any(time_slide_dict[slide_id].values()):
				expr_summ_table.write_experiment_summ( experiment_id, slide_id, veto_def_name, 'simulation', sim_proc_id = None)
				break
	else:
		# write zero-lag entries for all_data, playground, and exclude_play
		expr_summ_table.write_non_injection_summary( experiment_id, time_slide_dict, veto_def_name, write_all_data = True, write_playground = True, write_exclude_play = True, return_dict = return_dict )


def generate_experiment_tables(xmldoc, search_group, search, lars_id, veto_def_name, expr_start_time, expr_end_time, comments = None, simulation = False, verbose = False):
	"""
	Create or adds entries to the experiment table and experiment_summ
	table using instruments pulled from the search summary table and
	offsets pulled from the time_slide table.
	"""

	if verbose:
		print >> sys.stderr, "Populating the experiment and experiment_summary tables using search_summary and time_slide tables..."

	# Get the instruments that were on
	rinca_process_ids = get_rinca_process_ids(xmldoc)
	instruments = set( get_search_summary_instruments(xmldoc, rinca_process_ids) )

	# Populate the experiment table
	experiment_ids = populate_experiment_table(xmldoc, search_group, search, lars_id, instruments, expr_start_time, expr_end_time, comments = comments, add_inst_subsets = True, verbose = verbose)
	
	# Get the time_slide table as dict
	time_slide_dict = table.get_table(xmldoc, lsctables.TimeSlideTable.tableName).as_dict()

	# Populate the experiment_summary table
	for instruments in experiment_ids:
		populate_experiment_summ_table( xmldoc, experiment_ids[instruments], time_slide_dict, veto_def_name, simulation = simulation, return_dict = False, verbose = False )


#
# =============================================================================
#
#          Clean the experiment and experiment_summ tables 
#
# =============================================================================
#


def depopulate_experiment_tables(xmldoc, verbose = False):
	"""
	Removes entries from the experiment tables that do not have events
	or durations in them. In other words, if none of the rows in the
	experiment_summary table that are assoicated with a single experiment_id
	have neither an event in them nor any duration then all of the rows in the
	experiment_summary table associated	with that experiment_id are deleted,
	as well as the corresponding row in	the experiment table. If, however, just
	one of the rows in the experiment_summary table associated with an experiment_id
	have at least a 1 in their nevents column or at least 1 second in the
	durations column then nothing associated with that experiment_id are deleted.
	(In other words, if just one time slide in an experiment has just 1 event or 
	is just 1 second long, then none of the time slides in that experiment are deleted,
	even if all of the other time slides in that experiment have nothing in them.)	
	"""

	if verbose:
		print >>sys.stderr, "Depopulating the experiment tables...",

	#
	# find the experiment and experiment summary table
	#

	try:
		experiment_table = table.get_table(xmldoc, lsctables.ExperimentTable.tableName)
	except ValueError:
		# no table --> no-op
		if verbose:
			print >>sys.stderr, "Cannot find the experiment table"
		return

	try:
		experiment_summ_table = table.get_table(xmldoc, lsctables.ExperimentSummaryTable.tableName)
	except ValueError:
		# no table --> no-op
		if verbose:
			print >>sys.stderr, "Cannot find the experiment_summary table"
		return
	
	del_eid_indices = []
	del_esid_indices = []

	for mm, erow in enumerate(experiment_table):
		this_eid = erow.experiment_id
		es_index_list = []
		for nn, esrow in enumerate(experiment_summ_table):
			if esrow.experiment_id == this_eid and (esrow.duration or esrow.nevents):
				# something in this experiment, go on to next experiment_id
				break
			if esrow.experiment_id == this_eid:
				es_index_list.append(nn)
			if nn == len(experiment_summ_table) - 1:
				# if get to here, nothing in that experiment, mark id and indices
				# for removal
				del_eid_indices.append(mm)
				del_esid_indices += es_index_list
	
	# delte all experiments who's eids fall in del_eid_indices
	del_eid_indices.sort(reverse = True)
	for eid_index in del_eid_indices:
		del experiment_table[eid_index]
	# delete all experiment_summaries whose esids fall in del_esid_indices
	del_esid_indices.sort(reverse = True)
	for esid_index in del_esid_indices:
		del experiment_summ_table[esid_index]

	if verbose:
		print >> sys.stderr, "\n\tremoved %i empty experiment(s) from the experiment table and %i associated time slides from the experiment_summary table." %( len(del_eid_indices), len(del_esid_indices) )


#
# =============================================================================
#
#       Populate coinc_event, coinc_event_map, and coinc_inspiral Tables
#
# =============================================================================
#


#
# retrieve the ring boundaries
#


def retrieve_ring_boundaries(xmldoc):
	#
	# grab the segment list for any instrument selected at random (they
	# are all the same)
	#

	rings = ligolw_search_summary.segmentlistdict_fromsearchsummary(xmldoc, program = "rinca").popitem()[1]

	#
	# because the input often contains two rinca jobs the rings might
	# be duplicated;  use set() to uniqueify them then sort them.
	#

	rings = segments.segmentlist(set(rings))
	rings.sort()

	#
	# check that the (sorted) rings are non-intersecting
	#
	# FIXME:  shouldn't there only be exactly 1 ring?
	#

	for i in range(len(rings) - 1):
		if rings[i].intersects(rings[i + 1]):
			raise ValueError, "non-disjoint rinca rings detected in search_summary table"

	#
	# done
	#

	return rings


def get_playground_segments(all_data_segments):
	"""
	Calculates playground segments using all_data segments.
	
	@all_data_segments: segmentlist of all_data segments.
	"""
	from glue import segmentsUtils	
	playground_segments = segments.segmentlist([])
	for this_segment in all_data_segments:
		playground_segments |= segmentsUtils.S2playground(this_segment)

	return playground_segments


#
# For sngl_ringdown <--> sngl_ringdown coincidences
#


def populate_coinc_event_sngls(xmldoc, process, effective_snr_factor = 250.0, chisq_index = 6.0, verbose = False):
	if verbose:
		print >>sys.stderr, "constructing coincs ...",

	#
	# retrieve the ring boundaries
	#

	rings = retrieve_ring_boundaries(xmldoc)

	#
	# get the list of analyzed instruments from the search_summary
	# table
	#

	on_instruments = get_search_summary_instruments(xmldoc, get_rinca_process_ids(xmldoc))

	#
	# find the coinc_definer_id for sngl_ringdown <--> sngl_ringdown
	# coincidences, or create it if needed
	#

	coinc_def_id = llwapp.get_coinc_def_id(xmldoc, ligolw_rinca.RingdownCoincDef.search, ligolw_rinca.RingdownCoincDef.search_coinc_type, create_new = True, description = ligolw_rinca.RingdownCoincDef.description)

	#
	# find the coinc_event table or create one if needed
	#

	try:
		coinc_event_table = table.get_table(xmldoc, lsctables.CoincTable.tableName)
	except ValueError:
		coinc_event_table = xmldoc.childNodes[0].appendChild(lsctables.New(lsctables.CoincTable))

	#
	# synchronize the coinc_event table's ID generator with any
	# pre-existing rows
	#

	coinc_event_table.sync_next_id()

	#
	# find the coinc_event_map table or create one if needed
	#

	try:
		coinc_event_map_table = table.get_table(xmldoc, lsctables.CoincMapTable.tableName)
	except ValueError:
		coinc_event_map_table = xmldoc.childNodes[0].appendChild(lsctables.New(lsctables.CoincMapTable))

	#
	# find the coinc_ringdown table or create one if needed
	#

	try:
		coinc_ringdown_table = table.get_table(xmldoc, lsctables.CoincRingdownTable.tableName)
	except ValueError:
		coinc_ringdown_table = xmldoc.childNodes[0].appendChild(lsctables.New(lsctables.CoincRingdownTable))

	#
	# find the sngl_ringdown table
	#

	try:
		sngl_ringdown_table = table.get_table(xmldoc, lsctables.SnglRingdownTable.tableName)
	except ValueError:
		# no sngl_ringdown table --> no-op
		if verbose:
			print >>sys.stderr, "cannot find sngl_ringdown table"
		return

	#
	# find the time_slide table, and convert it to a dictionary of
	# offset vectors
	#

	time_slide_table = table.get_table(xmldoc, lsctables.TimeSlideTable.tableName)
	time_slides = time_slide_table.as_dict()

	#
	# iterate over reconstructed coincs
	#

	sngl_ringdown_table.sort(lambda a, b: cmp(a.event_id, b.event_id))
	for event_id, events in itertools.groupby(sngl_ringdown_table, lambda row: row.event_id):
		#
		# alphabetize events by instrument
		#

		events = tuple(sorted(events, lambda a, b: cmp(a.ifo, b.ifo)))
		if len(events) < 2:
			# not a coincidence, just a single.  assign a new,
			# unique, event_id and continue.
			for event in events:
				event.event_id = sngl_ringdown_table.get_next_id()
			continue

		#
		# build a coinc_event
		#

		coinc = coinc_event_table.RowType()
		coinc.process_id = process.process_id
		coinc.coinc_event_id = coinc_event_table.get_next_id()
		coinc.coinc_def_id = coinc_def_id
		coinc.nevents = len(events)
		coinc.set_instruments(on_instruments)
		coinc.likelihood = None
		coinc_event_table.append(coinc)

		#
		# construct the time_slide_id from the "slide number" at
		# index 1 in the tuple returned by get_id_parts().  all
		# events in the coinc have the same ID at this stage so it
		# doesn't matter which event's ID we use.
		#

		coinc.time_slide_id = type(time_slide_table.next_id)(events[0].get_id_parts()[1])

		#
		# link events to coinc with coinc_event_map rows.  the
		# ringdown triggers are assigned new, unique, event IDs
		# here.
		#

		for event in events:
			coincmap = coinc_event_map_table.RowType()
			coincmap.coinc_event_id = coinc.coinc_event_id
			coincmap.event_id = event.event_id = sngl_ringdown_table.get_next_id()
			coincmap.table_name = coincmap.event_id.table_name
			coinc_event_map_table.append(coincmap)

		#
		# populate coinc_ringdown table with coinc summary:
		#
		# - end_time is the end time of the first trigger in
		#   alphabetical order by instrument (!?) time-shifted
		#   according to the coinc's offset vector
		# - frequency is blank, set in repop_coinc
		# - quality is blank, set in repop_coinc
		# - mass is blank, set in repop_coinc
		# - spin is blank, set in repop_coinc
		# - ds2 values are set in repop_coinc; they will be
		#   averaged values but should be same in both ifos so
		#   averaging shouldn't matter.
		# - snr is blank
		# - uncombined_far is blank
		# - false-alarm rate is blank
		#

		coinc_ringdown = lsctables.CoincRingdown()
		coinc_ringdown.coinc_event_id = coinc.coinc_event_id
		coinc_ringdown.set_ifos(str(event.ifo) for event in events)
		coinc_ringdown.frequency = None
		coinc_ringdown.quality = None
		coinc_ringdown.mass = None
		coinc_ringdown.spin = None
		coinc_ringdown.snr = None
		coinc_ringdown.snr_sq = None
		coinc_ringdown.choppedl_snr = None
		coinc_ringdown.eff_coh_snr = None
		coinc_ringdown.null_stat = None
		coinc_ringdown.kappa = None
		coinc_ringdown.snr_ratio = None
		coinc_ringdown.false_alarm_rate = None
		coinc_ringdown.combined_far = None

		#
		# the time of the coinc = the end time of the first trigger
		# in the coinc in alphabetical order by instrument
		#

		coinc_time = events[0].get_start()

		#
		# which ring is it in?
		#

		ring = rings[rings.find(coinc_time)]

		#
		# the amount by which to slide it = the offset recorded in
		# the time_slide table for the instrument from which the
		# coinc's time has been taken
		#

		offset = time_slides[coinc.time_slide_id][events[0].ifo]

		#
		# slide the coinc's time on the ring.
		#

		coinc_ringdown.set_start(SnglInspiralUtils.slideTimeOnRing(coinc_time, offset, ring))

		coinc_ringdown_table.append(coinc_ringdown)

	#
	# done
	#

	if verbose:
		print >>sys.stderr, "constructed %d coincs" % len(coinc_event_table)


#
# =============================================================================
#
#                        Depopulate sngl_ringdown Table
#
# =============================================================================
#


def depopulate_sngl_ringdown(xmldoc, verbose = False):
	if verbose:
		print >>sys.stderr, "depopulating sngl_ringdown ...",

	#
	# find the sngl_ringdown table
	#

	try:
		sngl_ringdown_table = table.get_table(xmldoc, lsctables.SnglRingdownTable.tableName)
	except ValueError:
		# no sngl_ringdown table --> no-op
		if verbose:
			print >>sys.stderr, "cannot find sngl_ringdown table"
		return

	length_before = len(sngl_ringdown_table)

	#
	# delete duplicates, recording replacement event_ids.  this relies
	# on the SnglRingdown class' __eq__() and __hash__() methods to
	# define when two triggers are the same
	#

	trigger_to_id_index = {}
	mapping = {}
	for i in xrange(len(sngl_ringdown_table) - 1, -1, -1):
		trigger = sngl_ringdown_table[i]
		if trigger in trigger_to_id_index:
			mapping[trigger.event_id] = trigger_to_id_index[trigger]
			del sngl_ringdown_table[i]
		else:
			trigger_to_id_index[trigger] = trigger.event_id

	#
	# update IDs in other tables
	#

	for tbl in xmldoc.getElementsByTagName(sngl_ringdown_table.tagName):
		tbl.applyKeyMapping(mapping)

	#
	# done
	#

	if verbose:
		print >>sys.stderr, "removed %d redundant sngl_ringdown events, %d remaining" % (length_before - len(sngl_ringdown_table), len(sngl_ringdown_table))


#
# =============================================================================
#
#                          Populate experiment_map_table
#
# =============================================================================
#


def populate_experiment_map(xmldoc, search_group, search, lars_id, veto_def_name, expr_start_time, expr_end_time, simulation = False, verbose = False):
	from glue.pipeline import s2play as is_in_playground

	#
	# find the experiment_map table or create one if needed
	#
	if verbose:
		print >> sys.stderr, "Mapping coinc events to experiment_summary table..."

	try:
		expr_map_table = table.get_table(xmldoc, lsctables.ExperimentMapTable.tableName)
	except ValueError:
		expr_map_table = xmldoc.childNodes[0].appendChild(lsctables.New(lsctables.ExperimentMapTable))

	#
	# find the coinc_event table
	#

	coinc_event_table = table.get_table(xmldoc, lsctables.CoincTable.tableName)

	#
	# Index the coinc_ringdown table as a dictionary
	#

	coinc_index = dict((row.coinc_event_id, row) for row in table.get_table(xmldoc, lsctables.CoincRingdownTable.tableName))

	#
	# Get the time_slide_table as dict
	#

	time_slide_dict = table.get_table(xmldoc, lsctables.TimeSlideTable.tableName).as_dict()

	#
	# find the experiment table
	#

	expr_table = table.get_table(xmldoc, lsctables.ExperimentTable.tableName)

	#
	# find the experiment_summary table
	#

	expr_summ_table = table.get_table(xmldoc, lsctables.ExperimentSummaryTable.tableName)

	#
	# Get the on_instruments from the search summary table
	# for now will assign all coincs to the same experiment ifo time
	# (this may change later when vetoes are applied)
	#

	rinca_process_ids = get_rinca_process_ids(xmldoc)
	instruments = get_search_summary_instruments(xmldoc, rinca_process_ids)

	#
	# cycle through the coincs in the coinc_ringdown table
	# assigning initial experiment_summ_ids
	#
	for coinc in coinc_event_table:

		#
		#  map the coinc to an experiment
		#

		expr_map = lsctables.ExperimentMap()

		expr_map.coinc_event_id = coinc.coinc_event_id

		# get the (temporary) experiment and experiment_summ_id for this coinc 
		expr_id = expr_table.get_expr_id( search_group, search, lars_id, instruments, expr_start_time, expr_end_time )

		if simulation:
			# map to the simulation entry in the experiment_summary table
			expr_map.experiment_summ_id = expr_summ_table.get_expr_summ_id( expr_id, coinc.time_slide_id,
				veto_def_name, 'simulation', sim_proc_id = None )
			if not expr_map.experiment_summ_id:
				raise ValueError, "simulation experiment_summ_id could not be found with %s" \
				%(','.join([ str(expr_id), str(coinc.time_slide_id), veto_def_name ]))

			# map the experiment
			expr_map_table.append(expr_map)
			# Increment number of events in nevents column by 1
			expr_summ_table.add_nevents( expr_map.experiment_summ_id, 1 )

		# if not simulation, check if the coinc is time-slide or zero-lag
		elif not any( time_slide_dict[coinc.time_slide_id].values() ):
			# map it to the all_data zero-lag entry for this experiment in the experiment_summary table
			expr_map.experiment_summ_id = expr_summ_table.get_expr_summ_id( expr_id, coinc.time_slide_id,
				veto_def_name, 'all_data', sim_proc_id = None )
			if not expr_map.experiment_summ_id:
				raise ValueError, "all_data experiment_summ_id could not be found with %s" \
				%(','.join([ str(expr_id), str(coinc.time_slide_id), veto_def_name ]))

			expr_map_table.append(expr_map)
			expr_summ_table.add_nevents( expr_map.experiment_summ_id, 1 )
			
			#
			# determine if the coinc is in playground or not; depending on the result, also map it
			# to either the playground or exclude_play entry for this experiment
			#

			# reset expr_map
			expr_map = lsctables.ExperimentMap()
			expr_map.coinc_event_id = coinc.coinc_event_id

			if is_in_playground( coinc_index[coinc.coinc_event_id].start_time ) == 1:
				expr_map.experiment_summ_id = expr_summ_table.get_expr_summ_id( expr_id, coinc.time_slide_id,
					veto_def_name, 'playground', sim_proc_id = None )
			else:
				expr_map.experiment_summ_id = expr_summ_table.get_expr_summ_id( expr_id, coinc.time_slide_id,
					veto_def_name, 'exclude_play', sim_proc_id = None )

			if not expr_map.experiment_summ_id:
				raise ValueError, "playground/exclude_play experiment_summ_id could not be found with %s" \
				%(','.join([ str(expr_id), str(coinc.time_slide_id), veto_def_name ]))

			expr_map_table.append(expr_map)
			expr_summ_table.add_nevents( expr_map.experiment_summ_id, 1 )

		# otherwise, if not zero_lag (and not simulation), just map to the appropiate slide entry
		else:
			expr_map.experiment_summ_id = expr_summ_table.get_expr_summ_id( expr_id, coinc.time_slide_id,
				veto_def_name, 'slide', sim_proc_id = None )

			if expr_map.experiment_summ_id is None:
				raise ValueError, "experiment_summ_id could not be found with ids %s" \
				%(','.join([ str(expr_id), str(coinc.time_slide_id), veto_def_name ]))

			expr_map_table.append(expr_map)
			expr_summ_table.add_nevents( expr_map.experiment_summ_id, 1 )



#
# =============================================================================
#
#                                 Apply Vetoes
#
# =============================================================================
#


def apply_vetoes(xmldoc, veto_segments, process, verbose = False):
	if verbose:
		print >>sys.stderr, "applying vetoes ..."

	#
	# find the tables we'll need
	#
	
	try:
		sngl_ringdown_table = table.get_table(xmldoc, lsctables.SnglRingdownTable.tableName)
	except ValueError:
		# no sngl_ringdown table --> no-op
		if verbose:
			print >>sys.stderr, "\tcannot find sngl_ringdown table"
		return
	coinc_table = table.get_table(xmldoc, lsctables.CoincTable.tableName)
	coinc_map_table = table.get_table(xmldoc, lsctables.CoincMapTable.tableName)
	coinc_ringdown_table = table.get_table(xmldoc, lsctables.CoincRingdownTable.tableName)
	time_slide_table = table.get_table(xmldoc, lsctables.TimeSlideTable.tableName)
	expr_table = table.get_table(xmldoc, lsctables.ExperimentTable.tableName)
	expr_summ_table = table.get_table(xmldoc, lsctables.ExperimentSummaryTable.tableName)
	expr_map_table = table.get_table(xmldoc, lsctables.ExperimentMapTable.tableName)

	#
	# turn the time slide table into a dictionary
	#

	if verbose:
		print >>sys.stderr, "\tindexing time_slide table and computing segment lists ..."
	time_slides = time_slide_table.as_dict()

	#
	# retrieve the ring boundaries
	#

	rings = retrieve_ring_boundaries(xmldoc)

	#
	# performance assist:  remove veto segments that don't fall in any
	# of the rings, then remove veto segment lists that are empty
	#

	coalesced_rings = segments.segmentlist(rings).coalesce()
	veto_segments = segments.segmentlistdict((instrument, seglist & coalesced_rings) for (instrument, seglist) in veto_segments.items() if seglist.intersects(coalesced_rings))
	if not veto_segments:
		if verbose:
			print >>sys.stderr, "\tno vetos were on during the times spanned by this document"
		return

	#
	# create the coinc_event_id --> sngl_ringdown index, and
	# coinc_event_id --> coinc_ringdown index
	#

	if verbose:
		print >>sys.stderr, "\tindexing coinc tables ..."
	index = dict((row.event_id, row) for row in sngl_ringdown_table)

	coinc_map_table.sort(lambda a, b: cmp(a.coinc_event_id, b.coinc_event_id))
	index = dict(
		(coinc_event_id, tuple(index[row.event_id] for row in rows))
		for coinc_event_id, rows in itertools.groupby(
			(row for row in coinc_map_table if row.table_name == "sngl_ringdown"),
			lambda row: row.coinc_event_id
		)
	)

	coinc_ringdowns = dict((row.coinc_event_id, row) for row in coinc_ringdown_table)

	#
	# create an experiment_id --> experiment index, 
	# an experiment_summ_id --> experiment_summ index,
	# and a coinc_event_id --> experiment_map index
	#
	if verbose:
		print >> sys.stderr, "\tindexing experiment tables..."
		# set-up a dictionary to keep track of number of coincs moved to different times
		moved_coincs = {}
	experiment_index = dict((row.experiment_id, row) for row in expr_table)
	experiment_summ_index = dict((row.experiment_summ_id, row) for row in expr_summ_table)
	experiment_map_index = dict((row.coinc_event_id, row) for row in expr_map_table)
	# note: if a coinc is mapped to multiple experiment_summary rows (e.g., if zero-lag, a
	# coinc will be mapped to both all_data and either playground or exclude_play), the
	# experiment_map_index will only store one of those mappings; this is ok, since we
	# only need to know what row in the experiment table the coinc maps to, which will
	# be the same for both the all_data and playground/exclude_play rows

	#
	# iterate over coincs
	#

	if verbose:
		print >>sys.stderr, "\tchecking for coincs during vetoed times ..."

	cached_vetoes = {}
	N = len(coinc_table) / 1000 or 1

	for n, coinc_event in enumerate(coinc_table):
		if verbose and not (n % N):
			print >>sys.stderr, "\t\t%.1f%%\r" % (100.0 * n / len(coinc_table)),

		if coinc_event.coinc_event_id not in index:
			#
			# not a coinc we care about
			#

			continue

		#
		# retrieve the time slide vector
		#

		offset_vector = time_slides[coinc_event.time_slide_id]

		#
		# retrieve the instruments named in the search summary
		# table for the ring in which the coinc was found (recorded
		# earlier)
		#

		on_instruments = coinc_event.get_instruments()

		#
		# compare the instruments participating in the coinc to the
		# instruments for which offsets are available
		#

		found_instruments = set(event.ifo for event in index[coinc_event.coinc_event_id])

		if not found_instruments.issubset(set(offset_vector)):
			raise ValueError, "coinc '%s' has instrument(s) %s that are not in time slide vector '%s' (%s)" % (coinc_event.coinc_event_id, ", ".join(found_instruments - set(offset_vector)), coinc_event.time_slide_id, ", ".join(["%s=%g" % item for item in offset_vector.items()]))

		#
		# from the vetoes, check to see which instruments were
		# *really* on at the time of the coinc
		#

		#
		# the slid time of the coinc
		#

		coinc_time = coinc_ringdowns[coinc_event.coinc_event_id].get_start()

		#
		# which ring is it in?  note:  although the time recorded
		# for the coinc is its time after offsets are applied to
		# triggers, that time is always in the same ring as the
		# original time of the trigger from which the coinc's time
		# has been taken
		#

		ring = rings[rings.find(coinc_time)]

		#
		# slide the veto segments on that ring, cache the results
		# to avoid recalculation
		#

		try:
			vetoes = cached_vetoes[(ring, coinc_event.time_slide_id)]
		except KeyError:
			vetoes = cached_vetoes[(ring, coinc_event.time_slide_id)] = SnglInspiralUtils.slideSegListDictOnRing(ring, veto_segments, offset_vector)

		#
		# set the coinc's instruments to just those that were on at
		# the time of the coinc.  note that because the assignment
		# of a single time to a coinc is a poorly-defined
		# procedure, it can come to pass that a coinc is said to
		# have occured at a time when the instruments that
		# participate in it are vetoed.  we sweep this under the
		# rug by adding to the list of instruments that are on at
		# least those that participated in the coinc.
		#

		on_instruments -= set(instrument for instrument, vetosegs in vetoes.items() if coinc_time in vetosegs)
		coinc_event.set_instruments(on_instruments | found_instruments)

		# the following re-sets the detectors column in the experiment summary table,
		# if needed

		if not found_instruments.issubset(on_instruments):
			on_instruments = on_instruments | found_instruments 
		
		# check if on_instruments matches instruments listed in experiment table
		# for this coinc
		current_esid = experiment_map_index[coinc_event.coinc_event_id].experiment_summ_id
		current_eid = experiment_summ_index[current_esid].experiment_id
		current_inst = lsctables.instrument_set_from_ifos(experiment_index[current_eid].instruments)
		if current_inst != on_instruments:
			# get the experiment_id corresponding to on_instruments
			use_same = experiment_index[current_eid]
			new_eid = expr_table.get_expr_id(use_same.search_group, use_same.search, use_same.lars_id, on_instruments, use_same.gps_start_time, use_same.gps_end_time, comments = use_same.comments)
			# update the the experiment_map table by mapping the coinc to the new experiment
			for row in expr_map_table:
				if row.coinc_event_id == coinc_event.coinc_event_id:
					# get the new esid
					old_esid = row.experiment_summ_id
					use_same = experiment_summ_index[ old_esid ]
					new_esid = expr_summ_table.get_expr_summ_id( new_eid, use_same.time_slide_id, use_same.veto_def_name, use_same.datatype, sim_proc_id = use_same.sim_proc_id )
					# change the current esid to the new one
					row.experiment_summ_id = new_esid
					# update number of events in experiment_summary table
					expr_summ_table.add_nevents( old_esid, -1 )
					expr_summ_table.add_nevents( new_esid, 1 )

			# if verbose update moved_coincs
			if verbose:
				on_instr = lsctables.ifos_from_instrument_set(on_instruments)
			if verbose and experiment_index[current_eid].instruments not in moved_coincs:
				moved_coincs[experiment_index[current_eid].instruments] = {}
			if verbose and on_instr not in moved_coincs[experiment_index[current_eid].instruments]:
				moved_coincs[experiment_index[current_eid].instruments][on_instr] = 0
			if verbose:
				moved_coincs[experiment_index[current_eid].instruments][on_instr] += 1 

	if verbose:
		print >>sys.stderr, "\t\t100.0%"
		for old_instruments in moved_coincs:
			for new_instruments in moved_coincs[old_instruments]:
				print >> sys.stderr, "\tMoved %i coinc(s) from %s time to %s time." %(moved_coincs[old_instruments][new_instruments], old_instruments, new_instruments)


#
# =============================================================================
#
#                                     Main
#
# =============================================================================
#


options, url_categories, metadata_cache = parse_command_line()


if options.veto_segments:
	try:
		veto_segments = ligolw_segments.segmenttable_get_by_name(utils.load_filename(options.veto_segments, verbose = options.verbose), options.veto_segments_name).coalesce()
	except AttributeError:
		# will get an AttributeError if using new-format veto segment file because
		# new format does not include _ns; if so, remove the _ns columns from the
		# segment table and reset the definitions of lsctables.Segment.get and lsctables.Segment.set
		del lsctables.SegmentTable.validcolumns['start_time_ns']
		del lsctables.SegmentTable.validcolumns['end_time_ns']

		def get_segment(self):
			"""
			Return the segment described by this row.
			"""
			return segments.segment(lsctables.LIGOTimeGPS(self.start_time, 0), lsctables.LIGOTimeGPS(self.end_time, 0))

		def set_segment(self, segment):
			"""
			Set the segment described by this row.
			"""
			self.start_time = segment[0].seconds
			self.end_time = segment[1].seconds

		lsctables.Segment.get = get_segment
		lsctables.Segment.set = set_segment

		veto_segments = ligolw_segments.segmenttable_get_by_name(utils.load_filename(options.veto_segments, verbose = options.verbose), options.veto_segments_name).coalesce()


for (tag, veto_category), url_pairs in url_categories.items():
	if options.output_prefix:
		output_template = "%s%s%s_%%0%dd.xml.gz" % (options.output_prefix, tag and ("_%s" % tag) or "", veto_category and ("_CAT_%s" % veto_category) or "", int(math.floor(math.log10(len(url_pairs) or 1) + 1)))
	else:
		output_template = None
	for n, url_pair in enumerate(url_pairs):
		lsctables.SnglRingdownTable.next_id = None
		xmldoc = ligolw_add.ligolw_add(ligolw.Document(), [url for url in url_pair if url], verbose = options.verbose)
		lsctables.SnglRingdownTable.next_id = lsctables.SnglRingdownID(0)

		process = initialize_process(xmldoc)
		set_process_params(xmldoc, process, options)

		add_missing_id_columns(xmldoc, verbose = options.verbose)
		# FIXME:  uncomment this after Kipp talks people into it.
		#fix_ifos_columns(xmldoc, verbose = options.verbose)

		populate_rinca_time_slide_table(xmldoc, process, instruments = options.instruments, verbose = options.verbose)

		# set variables for experiment tables
		#FIXME: should search_group be command line set? comments?
		search_group = "cbc"

		# write the experiment_tables 
		generate_experiment_tables(xmldoc, search_group, options.search, options.lars_id, options.veto_segments_name, options.experiment_start_time, options.experiment_end_time, comments = None, simulation = options.simulation, verbose = options.verbose)

		populate_coinc_event_sngls(xmldoc, process, effective_snr_factor = options.effective_snr_factor, chisq_index = options.chisq_index, verbose = options.verbose)

		depopulate_sngl_ringdown(xmldoc, verbose = options.verbose)

		populate_experiment_map(xmldoc, search_group, options.search, options.lars_id, options.veto_segments_name, options.experiment_start_time, options.experiment_end_time, simulation = options.simulation, verbose = options.verbose)

		if options.veto_segments:
			apply_vetoes(xmldoc, veto_segments, process, verbose = options.verbose)
		elif options.verbose:
			print >>sys.stderr, "no vetoes applied"

		depopulate_time_slide_table(xmldoc, verbose = options.verbose)

		depopulate_experiment_tables(xmldoc, verbose = options.verbose)

		if metadata_cache is not None:
			add_metadata(xmldoc, metadata_cache, verbose = options.verbose)

		ligolw_process.set_process_end_time(process)

		# construct the output filename from the zero lag file being processed
		if output_template:
			output = output_template % n
		else:
			zero_lag_filename = os.path.basename(url_pair[0])
			output = re.sub('RINCA','RINCA_TO_COINC',zero_lag_filename)
		utils.write_filename(xmldoc, output, verbose = options.verbose, gz = (output or "stdout").endswith(".gz"))
		xmldoc.unlink()
		lsctables.table.reset_next_ids(lsctables.TableByName.values())
