#!/usr/bin/env python
#
# Copyright (C) 2011 Chris Pankow
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
"""Stream-based burst analysis tool"""

#
# =============================================================================
#
#                                   Preamble
#
# =============================================================================
#


import os
import sys
import time
import signal
import glob
import tempfile
import threading

from optparse import OptionParser
import ConfigParser
from ConfigParser import SafeConfigParser

import numpy

import pygtk
pygtk.require("2.0")
import gobject
gobject.threads_init()
import pygst
pygst.require("0.10")

# This mess is to make gstreamer stop eating our help messages.
if "--help" in sys.argv or "-h" in sys.argv:
	try:
		del sys.argv[ sys.argv.index( "--help" ) ]
	except ValueError:	
		pass
	try:
		del sys.argv[ sys.argv.index( "-h" ) ]
	except ValueError:	
		pass
	
	import gst
	sys.argv.append( "--help" )
else:
	import gst

from gstlal import pipeparts
from gstlal.reference_psd import write_psd, read_psd_xmldoc

import gstlal.excesspower as ep
from gstlal.epparts import EPHandler
from gstlal import datasource

from glue import GWDataFindClient

from glue.ligolw import ligolw, array, param, lsctables, table
array.use_in(ligolw.LIGOLWContentHandler)
param.use_in(ligolw.LIGOLWContentHandler)
lsctables.use_in(ligolw.LIGOLWContentHandler)
from glue.ligolw import utils

from glue.segments import segment, segmentlist, segmentlistdict, PosInfinity
from glue import segmentsUtils
from glue import gpstime
from glue.lal import LIGOTimeGPS, Cache, CacheEntry

from pylal.xlal.datatypes.snglburst import from_buffer as sngl_bursts_from_buffer
from pylal.xlal.datatypes.real8frequencyseries import REAL8FrequencySeries
from pylal.xlal.lalburst import XLALlnOneMinusChisqCdf

__author__ = "Chris Pankow <chris.pankow@ligo.org>"
__version__ = "Defiant" # until we get proper versioning tags

#
# =============================================================================
#
#                        Message Handler Methods
#
# =============================================================================
#

# These are linked later in the pipeline to do the appropriate actions when signals are sent up.

def on_psd_change( elem, pspec, hand ):
	"""
	Get the PSD object and signal the handler to rebuild everything if the spectrum has changed appreciably from the PSD which was used to rebuild the filters originally.
	"""
	if options.verbose:
		print >> sys.stderr, "Intercepted spectrum signal."

	# Get the new one
	new_psd = REAL8FrequencySeries(
		name = "PSD",
		#epoch = laltypes.LIGOTimeGPS(0, message.structure["timestamp"]),
		f0 = 0.0,
		deltaF = elem.get_property( "delta-f" ),
		#sampleUnits = laltypes.LALUnit(message.structure["sample-units"].strip()),
		data = numpy.array( elem.get_property( "mean-psd" ) )
	)
	hand.cur_psd = new_psd

	# Determine if the PSD has changed enough to warrant rebuilding the 
	# filter bank.
	hand.psd_power = sum(new_psd.data)*hand.psd.deltaF
	#hand.psd_change = (hand.psd_power - psd_power) / (hand.psd_power + psd_power) 
	
	whitener_pos = elem.query_position(gst.FORMAT_TIME)[0]*1e-9

	# This will get triggered in two cases: the rate (and thus bin length)
	# has changed, or we had the default PSD in place previously.
	if len(new_psd.data) != len(hand.psd.data):
		if options.verbose:

			print >>sys.stderr, "Different PSD lengths detected, automatically regenerating filters."
		hand.psd = new_psd
		hand.rebuild_everything()
		return
	else:
		# Poor man's coherence
		hand.psd_change = 2.0/len(hand.psd.data) * sum(abs(hand.psd.data-new_psd.data)/(new_psd.data+hand.psd.data))
	#psd_change = abs(hand.psd.data-new_psd.data)/(new_psd.data+hand.psd.data)
	#print >>sys.stderr , "PSD estimate: %g / %g (min: %g, max: %g)" % (sum(hand.psd.data),sum(new_psd.data), min(psd_change), max(psd_change))
	#print >>sys.stderr , "change estimate: %f" % hand.psd_change

	if abs(hand.psd_change) > hand.psd_change_thresh and whitener_pos - handler.start > 0.75*options.drop_time:
		if options.verbose:
			print >> sys.stderr, "Processed signal, change estimate: %f, regenerating filters" % hand.psd_change
		hand.psd = new_psd
		hand.rebuild_everything()


def on_spec_corr_change( elem, pspec, hand ):
	"""
	Get the 2-point spectral correlation object and signal the handler to rebuild everything.
	"""
	if options.verbose:
		print >> sys.stderr, "Intercepted correlation signal."
	hand.spec_corr = elem.get_property( "spectral-correlation" )

	# If the spectrum correlation changes, rebuild everything
	if hand.psd is not None:
		hand.rebuild_everything()

def get_triggers(elem, handler):

	buffer = elem.emit("pull-buffer")

	if not handler.output:
		return # We don't want event information

	# TODO: Can I set units here on the buffer fields, avoid changing the 
	# triggers themslves and *not* screw up the underlying framework?

	# NOTE: This is here and not in the handler, since self.start may not be 
	# well defined before the pipeline starts, and thus shouldn't be checked 
	# until after we start running. We must know handler.start by the time we 
	# start drawing triggers, though
	# FIXME: This probably isn't needed anymore because of the drop time
	whiten_seg = segment( 
		LIGOTimeGPS(handler.start), 
		LIGOTimeGPS(handler.start + handler.whitener_offset)
	)

	trigs = lsctables.New(lsctables.SnglBurstTable,
			["ifo", "peak_time", "peak_time_ns", "start_time", "start_time_ns",
			"duration",  "search", "event_id", "process_id",
			"central_freq", "channel", "amplitude", "snr", "confidence",
			"chisq", "chisq_dof", "bandwidth"])
	for row in sngl_bursts_from_buffer(buffer):
		if row.peak_time in whiten_seg:
			continue
		# FIXME: Determine "magic number" or remove it
		row.confidence = -XLALlnOneMinusChisqCdf(row.snr * 0.62, row.chisq_dof * 0.62)
		#if options.compat:
		row.snr = row.snr / row.chisq_dof - 1

		# This is done here so that the current PSD is used rather than
		# what might be there when the triggers are actually output
		ep.compute_amplitude( row, handler.psd )

		trigs.append( row )
	
	#buf_ts = (buffer.timestamp*1e-9 - handler.start) / handler.units + handler.start
	# TODO: Why does the buf_dur need unit conversion, but not the timestamp
	buf_ts = buffer.timestamp*1e-9 
	buf_dur = buffer.duration*1e-9 / handler.units
	handler.stop = (buf_ts + buf_dur)

	# Check if clustering reduces the amount of events
	if len(handler.triggers) >= handler.max_events and handler.clustering:
		handler.process_triggers( trigs, cluster_passes=1 )
	else:
		handler.process_triggers( trigs )

	# We use the buffer timestamp here, since it's always guaranteed to be the
	# latest available buffer, so we guarantee that the span of triggers is
	# always greater than file stride duration
	print >>sys.stderr, buf_ts, handler.time_since_dump
	if buf_ts - handler.time_since_dump > handler.dump_frequency or len(handler.triggers) >= handler.max_events:
		outseg = segment( LIGOTimeGPS(handler.time_since_dump), LIGOTimeGPS(handler.time_since_dump + handler.dump_frequency) )

		subdir = ep.append_formatted_output_path( handler.outdirfmt, handler, mkdir=False )
		subdir = handler.outdir + "/" + subdir
		if not os.path.exists( subdir ):
			handler.lock.acquire()
			os.makedirs( subdir )
			handler.lock.release()
		fname = ep.make_cache_parseable_name(
			inst = handler.inst,
			tag = handler.channel,
			start = float(outseg[0]),
			stop = float(outseg[1]),
			ext = "xml.gz",
			dir = subdir
		)

		if handler.clustering:
			handler.process_triggers( [], cluster_passes = True )
		handler.write_triggers( filename = fname, seg = outseg )
		#handler.time_since_dump = handler.stop 
		handler.time_since_dump = float(outseg[1])

#
# =============================================================================
#
#                             Options Handling
#
# =============================================================================
#

parser = OptionParser()

datasource.append_options( parser )

parser.add_option("-f", "--initialization-file", dest="infile", help="Options to be pased to the pipeline handler. Required.", default=None)
#parser.add_option("-d", "--diagnostics", dest="diagnostics", action="store_true", help="Turn on multiple diagnostic dumps. Use with caution, as it will dump gigabytes of data (potentially) in a matter of minutes. Useful in nongraphical environemnts to monitor data throughput.", default=False)
parser.add_option("-v", "--verbose", dest="verbose", action="store_true", help="Be verbose.", default=False)
parser.add_option("-r", "--sample-rate", dest="sample_rate", action="store", type="int", help="Sample rate of the incoming data.")
parser.add_option("-S", "--stream-tfmap", dest="stream_tfmap", action="store", help="Encode the time frequency map to video as it is analyzed. If the argument to this option is \"video\" then the pipeline will attempt to stream to a video source. If the option is instead a filename, the video will be sent to that name. Prepending \"keyframe=\" to the filename will start a new video every time a keyframe is hit.")
parser.add_option("-t", "--disable-triggers", dest="disable_triggers", action="store_true", help="Don't record triggers.", default=False)
parser.add_option("-T", "--file-cache", dest="file_cache", action="store_true", help="Create file caches of output files. If no corresponding --file-cache-name option is provided, an approrpriate name is constructed by default, and will only be created at the successful conclusion of the pipeline run.", default=False)
parser.add_option("-F", "--file-cache-name", dest="file_cache_name", action="store", help="Name of the trigger file cache to be written to. If this option is specified, then when each trigger file is written, this file will be written to with the corresponding cache entry. See --file-cache for other details.", default=None)
#parser.add_option("-c", "--lalapps-power-compatibility", dest="compat", action="store_true", default=False, help="Output trigger information which conforms to the lalapps_power conventions.")
parser.add_option("-C", "--clustering", dest="clustering", action="store_true", default=False, help="Employ trigger tile clustering before output stage. Default or if not specificed is off." )
parser.add_option("-u", "--enable-db-uploads", dest="db_uploads", action="store_true", default=False, help="Upload uploads to trigger archiving services (e.g. gracedb). The threshold for upload should be in the initialization file. Default or if not specificed is off." )
parser.add_option("-m", "--enable-channel-monitoring", dest="channel_monitoring", action="store_true", default=False, help="Emable monitoring of channel statistics like even rate/signifiance and PSD power" )
parser.add_option("-p", "--peak-over-sample-fraction", type=float, default=None, dest="peak_fraction", help="Take the peak over samples corresponding to this fraction of the DOF for a given tile. Default is no peak." )
parser.add_option("-d", "--drop-start-time", type=float, default=120.0, dest="drop_time", help="Drop this amount of time (in seconds) in the beginning of a run. This is to allow time for the whitener to settle to the mean PSD. Default is 120 s.")

(options, args) = parser.parse_args()

# FIXME: This needs to be moved to after the option parsing
if options.data_source == "frames" and options.frame_cache is None:
	if options.gps_start_time is None or options.gps_end_time is None:
		sys.exit( "No frame cache present, and no GPS times set. Cannot query for data without an interval to query in." )

	# Shamelessly stolen from gw_data_find
	print "Querying LDR server for data location." 
	try:
		server, port = os.environ["LIGO_DATAFIND_SERVER"].split(":")
	except ValueError:
		sys.exit( "Invalid LIGO_DATAFIND_SERVER environment variable set" )
	print "Server is %s:%s" % (server, port)

	try:
		frame_type = cfg.get( "instrument", "frame_type" )
	except ConfigParser.NoOptionError:
		sys.exit( "Invalid cache location, and no frame type set, so I can't query LDR for the file locations." )
	if frame_type == "":
		sys.exit( "No frame type set, aborting." )

	print "Frame type is %s" % frame_type
	connection =  \
		GWDataFindClient.GWDataFindHTTPConnection(host=server, port=port)
	print "Equivalent command line is "
	print "gw_data_find -o %s -s %d -e %d -u file -t %s" %(handler.inst[0], options.gps_start_time, options.gps_end_time, frame_type)
	cache = connection.find_frame_urls( handler.inst[0], 
		frame_type,
		options.gps_start_time,
		options.gps_end_time,
		urltype="file",
		on_gaps="error"
	)

	tmpfile, tmpname = tempfile.mkstemp()
	print "Writing cache of %d files to %s" % (len(cache), tmpname)
	cache.tofile( open(tmpname, "w") )
	connection.close()
	options.frame_cache = tmpname

gw_data_source_opts = datasource.GWDataSourceInfo( options )

# Verbosity and diagnostics
verbose = options.verbose

#
# =============================================================================
#
#                           Handler / Pipeline options
#
# =============================================================================
#

# We need a pipeline and pipeline handler instance to configure
pipeline = gst.Pipeline( "gstlal_excesspower" )
mainloop = gobject.MainLoop()
handler = EPHandler(mainloop, pipeline)

# Enable the periodic output of trigger statistics
if options.channel_monitoring:
	handler.channel_monitoring = True

# If a sample rate other than the native rate is requested, we'll need to keep
# track of it
# TODO: Link this to the datasource options
if options.sample_rate is not None:
	handler.rate = options.sample_rate

# Does the user want a cache file to track the trigger files we spit out?
if not options.file_cache:
	handler.output_cache = None

# And if so, if you give us a name, we'll update it every time we output, else
# only at the end of the run
if options.file_cache and options.file_cache_name is not None:
	handler.output_cache_name = options.file_cache_name

# Clustering on/off
handler.clustering = options.clustering
# Be verbose?
handler.verbose = options.verbose

# Locate and load the initialization file
if not options.infile:
	print >>sys.stderr, "Initialization file required."
elif not os.path.exists( options.infile ):
	print >>sys.stderr, "Initialization file path is invalid."
	sys.exit(-1)

cfg = SafeConfigParser()
cfg.read( options.infile )

# Instruments and channels
# FIXME: Remove the manual options, datasource should take care of this
if len(gw_data_source_opts.channel_dict.keys()) == 1:
	handler.inst = gw_data_source_opts.channel_dict.keys()[0]
elif cfg.has_option( "instrument", "detector" ):
	handler.inst = cfg.get( "instrument", "detector" )
else:
	sys.exit( "Unable to determine instrument." )

# FIXME: Remove the manual options, datasource should take care of this
if gw_data_source_opts.channel_dict[handler.inst] is not None:
	handler.channel = gw_data_source_opts.channel_dict[handler.inst]
elif cfg.has_option( "instrument", "channel" ):
	handler.channel = cfg.get( "instrument", "channel" )
else:
	# TODO: In the future, we may request multiple channels for the same 
	# instrument -- e.g. from a single raw frame
	sys.exit( "Unable to determine channel." )
print "Channel name(s): " + handler.channel 

# FFT and time-frequency parameters
# Low frequency cut off -- filter bank begins here
handler.flow = cfg.getfloat( "tf_parameters", "min-frequency" )
# High frequency cut off -- filter bank ends here
handler.fhigh = cfg.getfloat( "tf_parameters", "max-frequency" )
# Frequency resolution of the finest filters
handler.base_band = cfg.getfloat( "tf_parameters", "base-resolution" )
# Tile duration should not exceed this value
handler.max_duration = cfg.getfloat( "tf_parameters", "max-time-resolution" )
# Number of resolutions levels. Can't be less than 1, and can't be greater than log_2((fhigh-flow)/base_band)
handler.max_level = cfg.getint( "tf_parameters", "max-resolution-level" )

# DOF options -- this affects which tile types will be calculated
max_dof, fix_dof = None, None
if cfg.has_option( "tf_parameters", "max-dof" ):
	max_dof = cfg.getint( "tf_parameters", "max-dof" )
if cfg.has_option( "tf_parameters", "fix-dof" ):
	fix_dof = cfg.getint( "tf_parameters", "fix-dof" )

if cfg.has_option( "tf_parameters", "fft-length" ):
	handler.fft_length = cfg.getfloat( "tf_parameters", "fft-length" )

if cfg.has_option( "cache", "cache-psd-every" ):
	handler.cache_psd = cfg.getint( "cache", "cache-psd-every" )
	print "PSD caching enabled. PSD will be recorded every %d seconds" % handler.cache_psd
else:
	handler.cache_psd = None

if cfg.has_option( "cache", "cache-psd-dir" ):
	handler.cache_psd_dir = cfg.get( "cache", "cache-psd-dir" )
	print "Caching PSD to %s" % handler.cache_psd_dir
	
# Used to keep track if we need to lock the PSD into the whitener
# FIXME: This is now failing because of improper units
psdfile = None
if cfg.has_option( "cache", "reference-psd" ):
	psdfile = cfg.get( "cache", "reference-psd" )
	try:
		handler.psd = read_psd_xmldoc( utils.load_filename( psdfile, contenthandler = ligolw.LIGOLWContentHandler ) )[ handler.inst ]
		print "Reference PSD for instrument %s from file %s loaded" % ( handler.inst, psdfile )
		# Reference PSD disables caching (since we already have it)
		handler.cache_psd = None
	except KeyError: # Make sure we have a PSD for this instrument
		sys.exit( "PSD for instrument %s requested, but not found in file %s. Available instruments are %s" % (handler.inst, psdfile, str(handler.psd.keys())) )

# Triggering options
# FIXME: Make this option go away, it's more trouble than its worth
if cfg.has_option( "triggering", "output-file" ):
	handler.outfile = cfg.get( "triggering", "output-file" )
if cfg.has_option( "triggering", "output-directory" ):
	handler.outdir = cfg.get( "triggering", "output-directory" )
if cfg.has_option( "triggering", "output-dir-format" ):
	handler.outdirfmt = cfg.get( "triggering", "output-dir-format" )

handler.output = not options.disable_triggers

# FAP thresh overrides SNR thresh, because multiple resolutions will have 
# different SNR thresholds, nominally.
if cfg.has_option( "triggering", "snr-thresh" ):
	handler.snr_thresh = cfg.getfloat( "triggering", "snr-thresh" )
if cfg.has_option( "triggering", "fap-thresh" ):
	handler.fap = cfg.getfloat( "triggering", "fap-thresh" )

if handler.snr_thresh is not None and handler.fap is not None:
	print >>sys.stderr, "WARNING: Both an SNR threshold and a FAP threshold have been specified. The FAP threshold will take precedence."
elif handler.fap is not None:
	print "False alarm probability threshold (in Gaussian noise) is %f" % handler.fap
else:
	print "SNR threshold (sqrt of tile energy) is %f" % handler.snr_thresh

# If one wishes to try and automatically upload events to an event database 
# (read: gracedb and friends) these options will be sufficient to the pipeline 
# to do such a thing
# FIXME: Rework or delete... this probably isn't going to work in realtime
if options.db_uploads:

	handler.db_thresh = cfg.getfloat( "triggering", "db-thresh" )
	if handler.db_thresh is None:
		print >>sys.stderr, "Warning, DB upload requested, but no threshold provided. Disabling."

	handler.db_client = cfg.get( "triggering", "db-client" )
	if handler.db_thresh is None:
		print >>sys.stderr, "Warning, DB upload requested, but no DB path provided. Disablinhg"
		handler.db_thresh = None

# Maximum number of events (+/- a few in the buffer) before which we drop an 
# output file
if cfg.has_option( "triggering", "events_per_file" ):
	handler.max_events = cfg.get_int( "triggering", "events_per_file" )

# If a specific (trigger) time is of interest, specify its GPS here
# TODO: Read from sngl_inspirals and sngl_bursts
# TODO: Make this useful -- incorporate into scan
trigger_begin, trigger_end = None, None
if cfg.has_option( "triggering", "trig_time_start" ):
	trigger_begin = cfg.getfloat( "triggering", "trig_time_start" )
if cfg.has_option( "triggering", "trig_time_end" ):
	trigger_end = cfg.getfloat( "triggering", "trig_time_end" )
if trigger_begin and trigger_end:
	handler.set_trigger_time_and_action( segment( trigger_begin, trigger_end ) )

# This is invoked here, or else the default rate is used, which will cause
# funny behavior for the defaults with some cases
# FIXME: Can we move this back now? This probably isn't even needed with the 
# drop samples
df = 1.0 / handler.fft_length
handler.filter_len = 2*int(2*handler.rate / handler.base_band )
if handler.psd is None:
	handler.build_default_psd( handler.rate, df, handler.fhigh )
handler.rebuild_filter()
handler.rebuild_matrix_mixers()

# Set process params in handler for use with the output xmldocs
handler.make_process_tables( options, None )

#
# =============================================================================
#
#                                     Main
#
# =============================================================================
#

if options.verbose:
	print "Assembling pipeline... this is gstlal_excesspower, version code name %s\n" % __version__,

head = datasource.mkbasicsrc( pipeline, 
	gw_data_source_opts, 
	handler.inst, 
	verbose 
)

# If we're running online, we need to set up a few things
if gw_data_source_opts.data_source == "lvshm":
	# FIXME: This is a guess. Not a terrible one, but still inaccurate
	handler.start = gpstime.GpsSecondsFromPyUTC( time.time() )

	# This enables EP to get the on and off signals to make a segment list
	gate = pipeline.get_by_name( "%s_state_vector_gate" % handler.inst )
	gate.set_property( "emit-signals", True )
	gate.connect("start", handler.handle_segment, "on" )
	gate.connect("stop", handler.handle_segment, "off" )

elif gw_data_source_opts.seg is not None:
	handler.start = float(gw_data_source_opts.seg[0])

handler.time_since_dump = handler.stop = handler.start

# Resample down to the requested rate
head = pipeparts.mkcapsfilter( pipeline, pipeparts.mkresample( pipeline, head ), "audio/x-raw-float,rate=%d" % handler.rate )

head = whitener = pipeparts.mkwhiten( pipeline, head )
# Amount of time to do a spectrum estimate over
whitener.set_property( "fft-length", handler.fft_length ) 
# Ignore gaps in the whitener. Turning this off will induce wild triggers because
# the whitener will see sudden drops in the data (due to the gaps)
whitener.set_property( "expand-gaps", True ) 

if psdfile is not None: # In other words, we have a reference PSD
	whitener.set_property( "mean-psd", handler.psd.data )
	whitener.set_property( "psd-mode", 1 ) # GSTLAL_PSDMODE_FIXED

# Drop first PSD estimates until it settles
if psdfile is None:
	head = pipeparts.mkdrop( pipeline, head, int(options.drop_time*handler.rate) )

head = pipeparts.mkqueue( pipeline, head )

if verbose:
	head = pipeparts.mkprogressreport( pipeline, head, "whitened stream" )

# excess power channel firbank
head = pipeparts.mkfirbank( pipeline, head, 
	time_domain=False, 
	block_stride=handler.rate 
)

# This function stores a reference to the FIR bank, creates the appropriate 
# filters and sets the other options appropriately
handler.add_firbank( head )
nchannels = handler.filter_bank.shape[0]
print "FIR bank constructed with %d %f Hz channels" % (nchannels, handler.base_band)

if verbose:
	head = pipeparts.mkprogressreport( pipeline, head, "FIR bank stream" )

postfirtee = pipeparts.mktee( pipeline, head )

# object to handle the synchronization of the appsinks
def get_triggers_with_handler( elem ):
	return get_triggers( elem, handler )
appsync = pipeparts.AppSync(appsink_new_buffer = get_triggers_with_handler)

# First branch -- send fully sampled data to wider channels for processing
nlevels = int(numpy.ceil( numpy.log2( nchannels ) )) 
for res_level in range(0, min(handler.max_level, nlevels)):
	head = postfirtee
	# queue up data
	head = pipeparts.mkqueue( pipeline, head )

	# New level bandwidth
	band = handler.base_band * 2**res_level

	# The undersample_rate for band = R/2 is => sample_rate (passthrough)
	undersamp_rate = 2 * band

	# If the rate which would be set by the undersampler falls below one, we 
	# have to take steps to prevent this, as gstreamer can't handle this. The 
	# solution is to change the "units" of the rate. Ideally, this should be 
	# done much earlier in the pipeline (e.g. as the data comes out of the 
	# source), however, to avoid things like figuring out what that means for 
	# the FIR bank we change units here, and readjust appropriately in the 
	# trigger output.
	if undersamp_rate < 1:
		print "Automatically adjusting units to compensate for undersample rate falling below unity."
		# No, it's not factors of ten, but rates which aren't factors
		# of two are often tricky, thus if the rate is a factor of two, the 
		# units conversion won't change that.
		if undersamp_rate > ep.EXCESSPOWER_UNIT_SCALE['mHz']:
			unit = 'mHz'
		elif undersamp_rate > ep.EXCESSPOWER_UNIT_SCALE['uHz']:
			unit = 'uHz'
		elif undersamp_rate > ep.EXCESSPOWER_UNIT_SCALE['nHz']:
			unit = 'nHz'
		else:
			sys.exit( "Requested undersampling rate would fall below 1 nHz." )
		# FIXME: No love for positive power of 10 units?

		handler.units = ep.EXCESSPOWER_UNIT_SCALE[unit]
		undersamp_rate /= handler.units
		print "Undersampling rate for level %d: %f %s" % (res_level, undersamp_rate, unit)
		head = pipeparts.mkcapssetter( pipeline, head, "audio/x-raw-float,rate=%d" % (handler.rate/handler.units), replace=False )
		head = pipeparts.mkgeneric( pipeline, head, "lal_audioundersample" )
		head = pipeparts.mkcapssetter( pipeline, head, "audio/x-raw-float,rate=%d" % undersamp_rate, replace=False )
	else:
		print "Undersampling rate for level %d: %f Hz" % (res_level, undersamp_rate)
		head = pipeparts.mkgeneric( pipeline, head, "lal_audioundersample" )
		head = pipeparts.mkcapsfilter( pipeline, head, "audio/x-raw-float,rate=%d" % undersamp_rate )

	if verbose:
		head = pipeparts.mkprogressreport( pipeline, head, 
			"Undersampled stream level %d" % res_level
		)

	# This converts N base band channels into M wider channels via the use of 
	# a NxM matrix with M block diagonal elements containing the proper 
	# renormalization per row
	head = matmixer = pipeparts.mkmatrixmixer( pipeline, head )
	handler.add_matmixer( matmixer, res_level )

	if verbose:
		head = pipeparts.mkprogressreport( pipeline, head,
			"post matrix mixer %d" % res_level 
		)

	# Square the samples to get energies to sum over
	head = pipeparts.mkgeneric( pipeline, head, "pow", exponent=2 )

	if verbose:
		head = pipeparts.mkprogressreport( pipeline, head, 
			"Energy stream level %d" % res_level
		)

	# Second branch -- duration
	# FIXME: Units on max_duration need to be adjusted to match undersampling
	max_samp = int(handler.max_duration*undersamp_rate)
	# If the user requests a maximum DOF, we use that instead
	if max_dof is not None:
		max_samp = max_dof

	if max_samp < 2:
		sys.exit( "The duration for the largest tile is smaller than a two degree of freedom tile. Try increasing the requested maximum tile duration or maximum DOF requirement." )
	print "Can sum up to %s degress of freedom in powers of two for this resolution level." % max_samp

	# samples to sum -- two is min number
	ndof = 2

	#head = pipeparts.mktee( pipeline, pipeparts.mkqueue( pipeline, head ) )
	head = pipeparts.mktee( pipeline, head )
	while ndof <= max_samp:
		if fix_dof is not None and ndof != fix_dof:
			ndof <<= 1
			continue

		if ndof/undersamp_rate > handler.max_duration:
			break

		print "Resolution level %d, DOFs: %d" % (res_level, ndof)

		# Multi channel FIR filter -- used to add together frequency bands into 
		# tiles
		durtee = pipeparts.mkqueue( pipeline, head )
		durtee = pipeparts.mkmean( pipeline, durtee )
		durtee.set_property( "n", ndof )
		durtee.set_property( "type", 2 )
		durtee.set_property( "moment", 1 )

		if verbose:
			durtee = pipeparts.mkprogressreport( pipeline, durtee, 
				"After energy summation resolution level %d, %d DOF" % 
					(res_level, ndof) 
			)

		# TODO: Audio
		if options.stream_tfmap and res_level == 0 and ndof == 2:
			if len(options.stream_tfmap.split("=")) == 2:
				split_opt, filename = options.stream_tfmap.split("=")
			else:
				filename = options.stream_tfmap
				# TODO: Make this more elegant
				if filename == "video": 
					filename = None
				split_opt = None

			durtee = ep.stream_tfmap_video( pipeline, durtee, 
				handler, 
				filename,
				split_opt
			)

		if options.disable_triggers:
			pipeparts.mkfakesink( pipeline, durtee )
			ndof = ndof << 1
			continue

		# FIXME: This never seems to work, but it could be very useful as a 
		# diagnostic
		histogram_triggers = False
		if histogram_triggers:
			durtee = pipeparts.mkqueue( pipeline, durtee )
			durtee = tmptee = pipeparts.mktee( pipeline, durtee )
			tmptee = pipeparts.mkhistogram( pipeline, tmptee )
			#tmptee = pipeparts.mkcolorspace( pipeline, tmptee )
			#pipeparts.mkgeneric( pipeline, tmptee, "autovideosink", filter_caps=gst.caps_from_string("video/x-raw-rgb") )
			pipeparts.mkogmvideosink( pipeline, tmptee, "test_%d_%d.ogm" % (res_level, ndof) )

		# Trigger generator
		# Number of tiles to peak over, if necessary
		peak_samples = max(1, int((options.peak_fraction or 0) * ndof))
		durtee = pipeparts.mkbursttriggergen( pipeline, durtee, 
			peak_samples, 
			bank = handler.build_filter_xml( res_level, ndof, verbose=verbose )
		)

		# Determine the SNR threshold for this trigger generator
		if handler.fap is not None:
			# FIXME: Still needs magic number... or use the EP version
			# ndof_eff = ndof * 0.62
			snr_thresh = ep.determine_thresh_from_fap(handler.fap, ndof)**2
		else:
			# TODO: Make clear in the ini file that the thresh is power SNR, 
			# not amplitude and then remove the square here
			snr_thresh = handler.snr_thresh**2
		if verbose:
			print "SNR threshold for level %d, ndof %d: %f, will take peak over %d samples for this branch." % (res_level, ndof, snr_thresh, peak_samples)
		durtee.set_property( "snr-thresh", snr_thresh )

		if verbose:
			durtee = pipeparts.mkprogressreport( pipeline, durtee, 
				"Trigger generator resolution level %d, %d DOF" % (res_level, ndof) 
			)

		# Funnel the triggers for this subbranch to the appsink
		# FIXME: Why doesn't this negotiate the caps properly?
		#appsync.add_sink( pipeline, pipeparts.mkqueue(pipeline, durtee), caps = gst.Caps("application/x-lal-snglburst") )
		appsync.add_sink( pipeline, pipeparts.mkqueue(pipeline, durtee) )

		ndof <<= 1
		if ndof > max_samp:
			break

# Check for empty frame segments list if handed frames 
for ifo, seglist in gw_data_source_opts.frame_segments.iteritems():
	if seglist is not None and abs( seglist ) == 0:
		fname = ep.make_cache_parseable_name( inst = handler.inst, tag = handler.channel, start = float(gw_data_source_opts.seg[0]), stop = float(gw_data_source_opts.seg[1]), ext = "xml.gz", dir = handler.outdir )
		handler.write_triggers( filename = fname )
		if verbose:
			print "No analyzable livetime in requested segment."
		sys.exit( 0 )

### END OF PIPELINE

# Spectrum notification processing
whitener.connect_after( "notify::mean-psd", on_psd_change, handler )
# Handle spectral correlation changes
# TODO: Make sure this doesn't have to be in the mm loop
whitener.connect_after( "notify::spectral-correlation", on_spec_corr_change, handler )

# Handle shutdowns
signal.signal( signal.SIGINT, handler.shutdown )
signal.signal( signal.SIGTERM, handler.shutdown )

print >>sys.stderr, "Startin' up."
if pipeline.set_state( gst.STATE_PLAYING ) == gst.STATE_CHANGE_FAILURE:
	raise RuntimeError( "pipeline failed to enter PLAYING state" )
def write_pipeline_graph():
	pipeparts.write_dump_dot(pipeline, "gstlal_excesspower", verbose = True)
	gst.DEBUG_BIN_TO_DOT_FILE( pipeline,
		gst.DEBUG_GRAPH_SHOW_ALL,
		"gstlal_excesspower"
	)
#write_pipeline_graph()
mainloop.run()
