diff --git a/README.md b/README.md index 946d8c6..154811b 100644 --- a/README.md +++ b/README.md @@ -3,10 +3,35 @@ Multi-source traceroute with geolocation information. Demo: [IP Address Lookup]( ![Using output from traceroute.py to plot hops on Google Map](https://raw.github.com/ayeowch/traceroute/master/screenshot.png) +## Prerequisites + +1. python2.7 +2. pip +3. virtualenv +4. traceroute (commandline version) +5. Might need to ensure you have gcc and python dev modules for your distribution + ## Installation -1. Install dependencies listed in requirements.txt file. Pip is recommended. You could also use virtualenv to keep things isolated. -2. Save traceroute.py into a directory with its path stored in your PYTHONPATH environment variable. (if using virtualenv, copy it here) +1. Create a project root directory (proj_root herein) for the traceroute scripts to live. The init-script assumes /var/lib/python/traceroute. + The source can be cloned here. This directory can be changed by editing init-script/traceroute. +2. Inside the proj_root directory , initialize a virtual environment to house python and the projects dependencies. Call the environment directory 'env'. +3. Install dependencies listed in requirements.txt file. This can be done by activating the virtualenv in step 2 and running pip install -r requirements.txt +4. Copy 'traceroute.sh' from init-script into the project root dir (/var/lib/python/traceroute). Or create a symbolic link. At the end it should look like: + bmartin@crappy-laptop:/var/lib/python/traceroute$ ls + env init-script LICENSE persistence.json README.md requirements.txt screenshot.png sources.json traceroute.py traceroute.sh + +5. Copy 'traceroute' from init-script into the /etc/init.d folder. Ensure to make the traceroute script executable. +6. If Debian - Run the command: (tbd) + + update-rc-d traceroute defaults + (might see some complaints) + +7. If Centos, run this: + chkconfig --level 35 traceroute on + +8. For a quick test, run + /etc/init.d/traceroute start ## Usage diff --git a/init-script/traceroute b/init-script/traceroute new file mode 100644 index 0000000..29b71dd --- /dev/null +++ b/init-script/traceroute @@ -0,0 +1,64 @@ +#!/bin/bash +# traceroute.py daemon +# chkconfig: 345 20 80 +# description: traceroute.py daemon +# processname: traceroute.py + +DAEMON_PATH="/var/lib/python/traceroute" + +DAEMON=$DAEMON_PATH/traceroute.sh +DAEMONOPTS="" + +NAME=traceroute.sh +DESC="Traceroute Python Tool" +PIDFILE=/var/run/$NAME.pid +SCRIPTNAME=/etc/init.d/$NAME + +case "$1" in +start) + printf "%-50s" "Starting $NAME..." + cd $DAEMON_PATH + PID=`$DAEMON $DAEMONOPTS > /dev/null 2>&1 & echo $!` + #echo "Saving PID" $PID " to " $PIDFILE + if [ -z $PID ]; then + printf "%s\n" "Fail" + else + echo $PID > $PIDFILE + printf "%s\n" "Ok" + fi +;; +status) + printf "%-50s" "Checking $NAME..." + if [ -f $PIDFILE ]; then + PID=`cat $PIDFILE` + if [ -z "`ps axf | grep ${PID} | grep -v grep`" ]; then + printf "%s\n" "Process dead but pidfile exists" + else + echo "Running" + fi + else + printf "%s\n" "Service not running" + fi +;; +stop) + printf "%-50s" "Stopping $NAME" + PID=`cat $PIDFILE` + cd $DAEMON_PATH + if [ -f $PIDFILE ]; then + kill -HUP $PID + printf "%s\n" "Ok" + rm -f $PIDFILE + else + printf "%s\n" "pidfile not found" + fi +;; + +restart) + $0 stop + $0 start +;; + +*) + echo "Usage: $0 {status|start|stop|restart}" + exit 1 +esac diff --git a/init-script/traceroute.sh b/init-script/traceroute.sh new file mode 100755 index 0000000..1db088e --- /dev/null +++ b/init-script/traceroute.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +EXEC_DIR=/var/lib/python/traceroute +VIRTUALENV_DIR=/var/lib/python/traceroute/env + +# Repeat every 5 seconds +INTERVAL=5 # + +source $VIRTUALENV_DIR/bin/activate + +cd $EXEC_DIR + +# TODO add some logging later + +while true; +do + python ./traceroute.py --ip_address=8.8.8.8 -c LO --webhook=http://localhost:8081/test; + sleep $INTERVAL; +done diff --git a/requirements.txt b/requirements.txt index 7e3031d..23de536 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ netifaces==0.10.4 requests==2.7.0 +tinydb==2.3.2 diff --git a/traceroute.py b/traceroute.py index ed2e735..f83256f 100755 --- a/traceroute.py +++ b/traceroute.py @@ -18,16 +18,19 @@ from subprocess import Popen, PIPE import requests import netifaces import time +from tinydb import TinyDB, where USER_AGENT = "traceroute/1.0 (+https://github.com/ayeowch/traceroute)" +DB_FILE = "./persistence.json" +WEBHOOK_OFFLINE = "webhook_offline" class Traceroute(object): """ Multi-source traceroute instance. """ def __init__(self, ip_address, source=None, country="US", tmp_dir="/tmp", - no_geo=False, timeout=120, debug=False): + no_geo=False, timeout=120, debug=False, max_latency=5): super(Traceroute, self).__init__() self.ip_address = ip_address self.source = source @@ -37,6 +40,9 @@ class Traceroute(object): self.source = sources[country] self.tmp_dir = tmp_dir + self.LATENCY_THRESHOLD = float(max_latency) + + self.no_geo = no_geo self.timeout = timeout self.debug = debug @@ -44,6 +50,9 @@ class Traceroute(object): self.hops = {} self.country = country + # flag to determine if webhook alert is warranted + self.latency_exceeded = False + # Localhost Specific operations happen here if self.country == 'LO': self.local_mode = True @@ -59,6 +68,11 @@ class Traceroute(object): self.__run_traceroute() self.probe_end = time.time() * 1000 + def pingLatencyThresholdExceeded(self): + """public method to query state of Traceroute calls""" + + return self.latency_exceeded + def __run_traceroute(self): """ Instead of running the actual traceroute command, we will fetch @@ -66,9 +80,9 @@ class Traceroute(object): that are listed at traceroute.org. For each hop, we will then attach geolocation information to it. """ - self.print_debug("ip_address={}".format(self.ip_address)) + self.print_debug("ip_address={0}".format(self.ip_address)) - filename = "{}.{}.txt".format(self.ip_address, self.country) + filename = "{0}.{1}.txt".format(self.ip_address, self.country) filepath = os.path.join(self.tmp_dir, filename) if not os.path.exists(filepath): @@ -108,7 +122,7 @@ class Traceroute(object): traceroute = re.findall(pattern, content)[0].strip() except IndexError: # Manually append closing for partially downloaded page - content = "{}".format(content) + content = "{0}".format(content) traceroute = re.findall(pattern, content)[0].strip() return (status_code, traceroute) @@ -134,6 +148,7 @@ class Traceroute(object): hop_element_pattern = '([\d\w.-]+)\s+\((\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\)\s+(\d+\.\d+ ms)' hp = re.compile(hop_element_pattern) + alertTriggered = False for entry in traceroute.split('\n'): entry = entry.strip() result = re.match(hop_pattern,entry) @@ -146,9 +161,17 @@ class Traceroute(object): hop_hosts = re.findall(host_pattern, hop['hosts']) self.hops[hop_num] = [] + for host in hop_hosts: m = hp.search(host) (hostname, ip, ping_time) = m.groups() + + # Check ping time to see if it exceeds threshold. Once one is found, don't need any more info from other hops + if alertTriggered is False: + if self._exceeds_hop_latency(ping_time): + self.latency_exceeded = True + alertTriggered = True + if self.no_geo: self.hops[hop_num].append( { @@ -200,13 +223,24 @@ class Traceroute(object): Returns geolocation information for the given IP address. """ location = None - url = "http://dazzlepod.com/ip/{}.json".format(ip_address) + url = "http://dazzlepod.com/ip/{0}.json".format(ip_address) status_code, json_data = self.urlopen(url) if status_code == 200 and json_data: tmp_location = json.loads(json_data) if 'latitude' in tmp_location and 'longitude' in tmp_location: location = tmp_location return location + def _exceeds_hop_latency(self,ping_time): + """return true if hop time exceeds specified latency threshold""" + # remote ' ms' from ping time + ping_as_float = float(ping_time.replace(" ms","")) + print "Compare {0} to {1}".format(ping_as_float, self.LATENCY_THRESHOLD) + + return ping_as_float >= self.LATENCY_THRESHOLD + + + + def execute_cmd(self, cmd): """ @@ -220,9 +254,9 @@ class Traceroute(object): signal.alarm(self.timeout) stdout, stderr = process.communicate() returncode = process.returncode - self.print_debug("cmd={}, returncode={}".format(cmd, returncode)) + self.print_debug("cmd={0}, returncode={1}".format(cmd, returncode)) if returncode != 0: - self.print_debug("stderr={}".format(stderr)) + self.print_debug("stderr={0}".format(stderr)) signal.alarm(0) except Exception as err: self.print_debug(str(err)) @@ -261,23 +295,37 @@ class Traceroute(object): 'mac' : addr[netifaces.AF_LINK][0]['addr'] }}) except KeyError,e: - self.print_debug("Key not found - _get_network_interface_info - {}".format(addr)) + pass + self.print_debug("Key not found - _get_network_interface_info - {0}".format(addr)) return iface_list def __get_network_routes(self): """ - Gather network routes on localhost. Only grabs default gateway. Need to play around on different hosts to see what output - should be + Gather routes from netifaces module """ routes = [] - gw = netifaces.gateways() - if 'default' in gw.keys(): - routes.append( { - 'default' : gw['default'][netifaces.AF_INET] - }) + gws = netifaces.gateways() + for k in gws.keys(): + if k == 'default': + continue + + for r in gws[k]: + (ip,interface,is_gateway) = r + + gw_name = "{0}".format(netifaces.address_families[k]) + + routes.append({ + gw_name : { + 'ip_address' : ip, + 'interface' : interface, + 'default' : is_gateway + } + + } + ) return routes @@ -294,7 +342,7 @@ class Traceroute(object): content = "" try: response = urllib2.urlopen(request) - self.print_debug("url={}".format(response.geturl())) + self.print_debug("url={0}".format(response.geturl())) content = self.chunked_read(response) except urllib2.HTTPError as err: status_code = err.code @@ -320,7 +368,7 @@ class Traceroute(object): break content += data read_bytes += bytes_per_read - self.print_debug("read_bytes={}, {}".format(read_bytes, data)) + self.print_debug("read_bytes={0}, {1}".format(read_bytes, data)) signal.alarm(0) except Exception as err: self.print_debug(str(err)) @@ -330,14 +378,14 @@ class Traceroute(object): """ Raises exception when signal is caught. """ - raise Exception("Caught signal {}".format(signum)) + raise Exception("Caught signal {0}".format(signum)) def print_debug(self, msg): """ Prints debug message to standard output. """ if self.debug: - print("[DEBUG {}] {}".format(datetime.datetime.now(), msg)) + print("[DEBUG {0}] {1}".format(datetime.datetime.now(), msg)) def get_report(self): report = {} @@ -354,12 +402,57 @@ class Traceroute(object): return report + +############################################################################################ +# +# Utility Functions For Reporting and Command-line Usage. +# +############################################################################################ + def post_result(webhook_url, report, timeout=120): """ POST traceroute report to specified website. Exceptions need to be caught in the caller """ return requests.post(webhook_url, data=json.dumps(report), timeout=timeout) +def webhook_available(webhook_url): + """ + Function to check if a webhook host is responding. + Not 100% sure this will work... + """ + try: + data = urllib.urlopen(webhook_url) + return True + except Exception,e: + return False + + +def cacheFull(webhook_cache): + """check if cache contains webhook records""" + + return webhook_cache.__len__() > 0 + +def purgeAndDeleteCache(webhook_cache, url): + """cycle through db, post results and delete db. + TODO - only delete successful posts. Figure out later. + """ + + totalRecords = webhook_cache.__len__() # not used currently. + print "Now posting offline cache" + for data in webhook_cache.all(): + try: + result = post_result(url, data) + except Exception,e: + print "Unable to post record from cache. Message was: {0}".format(e) + # clear cache + webhook_cache.purge() + print "Webhook cache cleared" + + + + + + def main(): cmdparser = optparse.OptionParser("%prog --ip_address=IP_ADDRESS") @@ -393,11 +486,18 @@ def main(): cmdparser.add_option( "-w", "--webhook", type="string", default="", help="Specify URL to POST report payload rather than stdout") + + cmdparser.add_option( + "--max_latency", type="int", default="5", + help="Maximum latency whereby the system will trigger the webhook ( if requested ). ") + options, _ = cmdparser.parse_args() json_file = open(options.json_file, "r").read() sources = json.loads(json_file.replace("_IP_ADDRESS_", options.ip_address)) + db = TinyDB(DB_FILE) + webhook_cache = db.table(WEBHOOK_OFFLINE) # Get Hope info using Traceroute Object traceroute = Traceroute(ip_address=options.ip_address, @@ -406,17 +506,37 @@ def main(): tmp_dir=options.tmp_dir, no_geo=options.no_geo, timeout=options.timeout, - debug=options.debug) + debug=options.debug, max_latency = options.max_latency) # pull complete report -> Hop data plus meta info about the network report = traceroute.get_report() + + + if options.webhook != "": - try: - result = post_result(options.webhook, report, options.timeout) - print "Webhook POST Result: {}".format(result) - except Exception,e: - print "Provided webhook {0} is invalid. Message was: {1}".format(options.webhook, e) + # check if remote host is available + if webhook_available(options.webhook): + + # if available, check if there are any outstanding reports that should be sent + # if traceroute::backlog == true => Purge results + + if cacheFull(webhook_cache): + purgeAndDeleteCache(webhook_cache, options.webhook) + + if traceroute.pingLatencyThresholdExceeded(): + try: + result = post_result(options.webhook, report, options.timeout) + print "Webhook POST Result: {0}".format(result) + except Exception,e: + print "Provided webhook {0} is invalid. Message was: {1}".format(options.webhook, e) + else: + print "Webhook unavailable, caching" + if traceroute.pingLatencyThresholdExceeded(): + #cache results until data is restored + webhook_cache.insert(report) + # Dump Result into Redis + # Set redis flag traceroute::backlog => true else: print(json.dumps(report, indent=4)) return 0