- alter format to be more flexible to values coming back

- no need for default gateway element

README.md

@@ -3,10 +3,35 @@ Multi-source traceroute with geolocation information. Demo: [IP Address Lookup](
 
 ![Using output from traceroute.py to plot hops on Google Map](https://raw.github.com/ayeowch/traceroute/master/screenshot.png)
 
+## Prerequisites
+
+1. python2.7
+2. pip
+3. virtualenv
+4. traceroute (commandline version)
+5. Might need to ensure you have gcc and python dev modules for your distribution
+
 ## Installation
 
-1. Install dependencies listed in requirements.txt file. Pip is recommended. You could also use virtualenv to keep things isolated.
-2. Save traceroute.py into a directory with its path stored in your PYTHONPATH environment variable. (if using virtualenv, copy it here)
+1. Create a project root directory (proj_root herein) for the traceroute scripts to live. The init-script assumes /var/lib/python/traceroute. 
+   The source can be cloned here.  This directory can be changed by editing init-script/traceroute.
+2. Inside the proj_root directory , initialize a virtual environment to house python and the projects dependencies. Call the environment directory 'env'. 
+3. Install dependencies listed in requirements.txt file. This can be done by activating the virtualenv in step 2 and running pip install -r requirements.txt
+4. Copy 'traceroute.sh' from init-script into the project root dir (/var/lib/python/traceroute). Or create a symbolic link. At the end it should look like:
+     bmartin@crappy-laptop:/var/lib/python/traceroute$ ls
+     env  init-script  LICENSE  persistence.json  README.md  requirements.txt  screenshot.png  sources.json  traceroute.py  traceroute.sh
+
+5. Copy 'traceroute' from init-script into the /etc/init.d folder. Ensure to make the traceroute script executable.
+6. If Debian - Run the command: (tbd)
+
+    update-rc-d traceroute defaults
+    (might see some complaints)
+
+7. If Centos, run this:
+    chkconfig --level 35 traceroute on
+
+8. For a quick test, run 
+    /etc/init.d/traceroute start
 
 
 ## Usage

init-script/traceroute

@@ -0,0 +1,64 @@
+#!/bin/bash
+# traceroute.py daemon
+# chkconfig: 345 20 80
+# description: traceroute.py daemon
+# processname: traceroute.py
+
+DAEMON_PATH="/var/lib/python/traceroute"
+
+DAEMON=$DAEMON_PATH/traceroute.sh
+DAEMONOPTS=""
+
+NAME=traceroute.sh
+DESC="Traceroute Python Tool"
+PIDFILE=/var/run/$NAME.pid
+SCRIPTNAME=/etc/init.d/$NAME
+
+case "$1" in
+start)
+	printf "%-50s" "Starting $NAME..."
+	cd $DAEMON_PATH
+	PID=`$DAEMON $DAEMONOPTS > /dev/null 2>&1 & echo $!`
+	#echo "Saving PID" $PID " to " $PIDFILE
+        if [ -z $PID ]; then
+            printf "%s\n" "Fail"
+        else
+            echo $PID > $PIDFILE
+            printf "%s\n" "Ok"
+        fi
+;;
+status)
+        printf "%-50s" "Checking $NAME..."
+        if [ -f $PIDFILE ]; then
+            PID=`cat $PIDFILE`
+            if [ -z "`ps axf | grep ${PID} | grep -v grep`" ]; then
+                printf "%s\n" "Process dead but pidfile exists"
+            else
+                echo "Running"
+            fi
+        else
+            printf "%s\n" "Service not running"
+        fi
+;;
+stop)
+        printf "%-50s" "Stopping $NAME"
+            PID=`cat $PIDFILE`
+            cd $DAEMON_PATH
+        if [ -f $PIDFILE ]; then
+            kill -HUP $PID
+            printf "%s\n" "Ok"
+            rm -f $PIDFILE
+        else
+            printf "%s\n" "pidfile not found"
+        fi
+;;
+
+restart)
+  	$0 stop
+  	$0 start
+;;
+
+*)
+        echo "Usage: $0 {status|start|stop|restart}"
+        exit 1
+esac

init-script/traceroute.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+EXEC_DIR=/var/lib/python/traceroute
+VIRTUALENV_DIR=/var/lib/python/traceroute/env
+
+# Repeat every 5 seconds
+INTERVAL=5 #
+
+source $VIRTUALENV_DIR/bin/activate
+
+cd $EXEC_DIR
+
+# TODO add some logging later
+
+while true;
+do
+    python ./traceroute.py --ip_address=8.8.8.8 -c LO --webhook=http://localhost:8081/test;
+    sleep $INTERVAL;
+done

requirements.txt

@@ -1,2 +1,3 @@
 netifaces==0.10.4
 requests==2.7.0
+tinydb==2.3.2

traceroute.py

@@ -18,16 +18,19 @@ from subprocess import Popen, PIPE
 import requests
 import netifaces
 import time
+from tinydb import TinyDB, where
 
 USER_AGENT = "traceroute/1.0 (+https://github.com/ayeowch/traceroute)"
 
+DB_FILE        = "./persistence.json"
+WEBHOOK_OFFLINE = "webhook_offline"
 
 class Traceroute(object):
     """
     Multi-source traceroute instance.
     """
     def __init__(self, ip_address, source=None, country="US", tmp_dir="/tmp",
-                 no_geo=False, timeout=120, debug=False):
+                 no_geo=False, timeout=120, debug=False, max_latency=5):
         super(Traceroute, self).__init__()
         self.ip_address = ip_address
         self.source = source
@@ -37,6 +40,9 @@ class Traceroute(object):
             self.source = sources[country]
         self.tmp_dir = tmp_dir
 
+        self.LATENCY_THRESHOLD = float(max_latency)
+    
+
         self.no_geo = no_geo
         self.timeout = timeout
         self.debug = debug
@@ -44,6 +50,9 @@ class Traceroute(object):
         self.hops = {}
         self.country = country
 
+        # flag to determine if webhook alert is warranted
+        self.latency_exceeded = False
+
         # Localhost Specific operations happen here
         if self.country == 'LO':
             self.local_mode = True
@@ -59,6 +68,11 @@ class Traceroute(object):
         self.__run_traceroute() 
         self.probe_end   = time.time() * 1000
 
+    def pingLatencyThresholdExceeded(self):
+        """public method to query state of Traceroute calls"""
+
+        return self.latency_exceeded
+
     def __run_traceroute(self):
         """
         Instead of running the actual traceroute command, we will fetch
@@ -66,9 +80,9 @@ class Traceroute(object):
         that are listed at traceroute.org. For each hop, we will then attach
         geolocation information to it.
         """
-        self.print_debug("ip_address={}".format(self.ip_address))
+        self.print_debug("ip_address={0}".format(self.ip_address))
 
-        filename = "{}.{}.txt".format(self.ip_address, self.country)
+        filename = "{0}.{1}.txt".format(self.ip_address, self.country)
         filepath = os.path.join(self.tmp_dir, filename)
 
         if not os.path.exists(filepath):
@@ -108,7 +122,7 @@ class Traceroute(object):
             traceroute = re.findall(pattern, content)[0].strip()
         except IndexError:
             # Manually append closing </pre> for partially downloaded page
-            content = "{}</pre>".format(content)
+            content = "{0}</pre>".format(content)
             traceroute = re.findall(pattern, content)[0].strip()
         return (status_code, traceroute)
 
@@ -134,6 +148,7 @@ class Traceroute(object):
         hop_element_pattern = '([\d\w.-]+)\s+\((\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\)\s+(\d+\.\d+ ms)'
         hp  = re.compile(hop_element_pattern)
 
+        alertTriggered = False
         for entry in traceroute.split('\n'):
             entry = entry.strip()
             result = re.match(hop_pattern,entry)
@@ -146,9 +161,17 @@ class Traceroute(object):
             hop_hosts   = re.findall(host_pattern, hop['hosts'])
 
             self.hops[hop_num] = []
+            
             for host in hop_hosts:
                 m = hp.search(host)
                 (hostname, ip, ping_time) = m.groups()
+                
+                # Check ping time to see if it exceeds threshold. Once one is found, don't need any more info from other hops
+                if alertTriggered is False:
+                    if self._exceeds_hop_latency(ping_time):
+                        self.latency_exceeded   = True
+                        alertTriggered          = True
+
                 if self.no_geo:
                     self.hops[hop_num].append(
                         { 
@@ -200,13 +223,24 @@ class Traceroute(object):
         Returns geolocation information for the given IP address.
         """
         location = None
-        url = "http://dazzlepod.com/ip/{}.json".format(ip_address)
+        url = "http://dazzlepod.com/ip/{0}.json".format(ip_address)
         status_code, json_data = self.urlopen(url)
         if status_code == 200 and json_data:
             tmp_location = json.loads(json_data)
             if 'latitude' in tmp_location and 'longitude' in tmp_location:
                 location = tmp_location
         return location
+    def _exceeds_hop_latency(self,ping_time):
+        """return true if hop time exceeds specified latency threshold"""
+        # remote ' ms' from ping time
+        ping_as_float = float(ping_time.replace(" ms",""))
+	print "Compare {0} to {1}".format(ping_as_float, self.LATENCY_THRESHOLD)
+
+        return ping_as_float >= self.LATENCY_THRESHOLD
+
+
+
+
 
     def execute_cmd(self, cmd):
         """
@@ -220,9 +254,9 @@ class Traceroute(object):
             signal.alarm(self.timeout)
             stdout, stderr = process.communicate()
             returncode = process.returncode
-            self.print_debug("cmd={}, returncode={}".format(cmd, returncode))
+            self.print_debug("cmd={0}, returncode={1}".format(cmd, returncode))
             if returncode != 0:
-                self.print_debug("stderr={}".format(stderr))
+                self.print_debug("stderr={0}".format(stderr))
             signal.alarm(0)
         except Exception as err:
             self.print_debug(str(err))
@@ -261,23 +295,37 @@ class Traceroute(object):
                     'mac'        : addr[netifaces.AF_LINK][0]['addr']
                }})
            except KeyError,e:
-               self.print_debug("Key not found - _get_network_interface_info - {}".format(addr))
+	       pass
+               self.print_debug("Key not found - _get_network_interface_info - {0}".format(addr))
 
         return iface_list
 
 
     def __get_network_routes(self):
         """
-        Gather network routes on localhost. Only grabs default gateway. Need to play around on different hosts to see what output
-        should be
+        Gather routes from netifaces module
         """
         routes = []
 
-        gw = netifaces.gateways()
-        if 'default' in gw.keys():
-            routes.append( {
-                'default' : gw['default'][netifaces.AF_INET]
-            })
+        gws = netifaces.gateways()
+        for k in gws.keys():
+            if k == 'default':
+                continue
+
+	    for r in gws[k]:
+                (ip,interface,is_gateway) = r
+
+                gw_name = "{0}".format(netifaces.address_families[k])
+
+                routes.append({
+                       gw_name : {
+                            'ip_address' : ip,
+                            'interface'  : interface,
+			    'default'    : is_gateway
+                       }
+                
+                    }
+                )
 
         return routes
 
@@ -294,7 +342,7 @@ class Traceroute(object):
         content = ""
         try:
             response = urllib2.urlopen(request)
-            self.print_debug("url={}".format(response.geturl()))
+            self.print_debug("url={0}".format(response.geturl()))
             content = self.chunked_read(response)
         except urllib2.HTTPError as err:
             status_code = err.code
@@ -320,7 +368,7 @@ class Traceroute(object):
                     break
                 content += data
                 read_bytes += bytes_per_read
-                self.print_debug("read_bytes={}, {}".format(read_bytes, data))
+                self.print_debug("read_bytes={0}, {1}".format(read_bytes, data))
             signal.alarm(0)
         except Exception as err:
             self.print_debug(str(err))
@@ -330,14 +378,14 @@ class Traceroute(object):
         """
         Raises exception when signal is caught.
         """
-        raise Exception("Caught signal {}".format(signum))
+        raise Exception("Caught signal {0}".format(signum))
 
     def print_debug(self, msg):
         """
         Prints debug message to standard output.
         """
         if self.debug:
-            print("[DEBUG {}] {}".format(datetime.datetime.now(), msg))
+            print("[DEBUG {0}] {1}".format(datetime.datetime.now(), msg))
 
     def get_report(self):
         report = {}
@@ -354,12 +402,57 @@ class Traceroute(object):
         
         return report
 
+
+############################################################################################
+#
+#  Utility Functions For Reporting and Command-line Usage.
+#
+############################################################################################
+
 def post_result(webhook_url, report, timeout=120):
     """
     POST  traceroute report to specified website. Exceptions need to be caught in the caller
     """
     return requests.post(webhook_url, data=json.dumps(report), timeout=timeout)
 
+def webhook_available(webhook_url):
+    """ 
+    Function to check if a webhook host is responding.
+    Not 100% sure this will work...
+    """
+    try:
+        data = urllib.urlopen(webhook_url)
+        return True
+    except Exception,e:
+        return False
+
+
+def cacheFull(webhook_cache):
+    """check if cache contains webhook records"""
+    
+    return webhook_cache.__len__() > 0
+
+def purgeAndDeleteCache(webhook_cache, url):
+    """cycle through db, post results and delete db.
+       TODO - only delete successful posts. Figure out later. 
+    """
+    
+    totalRecords = webhook_cache.__len__() # not used currently. 
+    print "Now posting offline cache"
+    for data in webhook_cache.all():
+        try:
+            result = post_result(url, data) 
+        except Exception,e:
+            print "Unable to post record from cache. Message was: {0}".format(e)
+    # clear cache 
+    webhook_cache.purge()
+    print "Webhook cache cleared"
+
+
+
+    
+
+
 
 def main():
     cmdparser = optparse.OptionParser("%prog --ip_address=IP_ADDRESS")
@@ -393,11 +486,18 @@ def main():
     cmdparser.add_option(
         "-w", "--webhook", type="string", default="",
         help="Specify URL to POST report payload rather than stdout")
+    
+    cmdparser.add_option(
+        "--max_latency", type="int", default="5",
+        help="Maximum latency whereby the system will trigger the webhook ( if requested ). ")
+    
 
     options, _  = cmdparser.parse_args()
     json_file   = open(options.json_file, "r").read()
     sources     = json.loads(json_file.replace("_IP_ADDRESS_", options.ip_address))
 
+    db            = TinyDB(DB_FILE)
+    webhook_cache = db.table(WEBHOOK_OFFLINE)
 
     # Get Hope info using Traceroute Object
     traceroute  = Traceroute(ip_address=options.ip_address,
@@ -406,17 +506,37 @@ def main():
                             tmp_dir=options.tmp_dir,
                             no_geo=options.no_geo,
                             timeout=options.timeout,
-                            debug=options.debug)
+                            debug=options.debug, max_latency = options.max_latency)
 
     # pull complete report -> Hop data plus meta info about the network
     report = traceroute.get_report()
 
+
+ 
+
     if options.webhook != "":
-        try:
-            result = post_result(options.webhook, report, options.timeout)
-            print "Webhook POST Result: {}".format(result)
-        except Exception,e:
-            print "Provided webhook {0} is invalid. Message was: {1}".format(options.webhook, e)
+        # check if remote host is available
+        if webhook_available(options.webhook):
+
+            # if available, check if there are any outstanding reports that should be sent
+            # if traceroute::backlog == true => Purge results
+
+            if cacheFull(webhook_cache):
+                purgeAndDeleteCache(webhook_cache, options.webhook)
+            
+            if traceroute.pingLatencyThresholdExceeded():
+                try:
+                    result = post_result(options.webhook, report, options.timeout)
+                    print "Webhook POST Result: {0}".format(result)
+                except Exception,e:
+                    print "Provided webhook {0} is invalid. Message was: {1}".format(options.webhook, e)
+        else:
+            print "Webhook unavailable, caching"
+            if traceroute.pingLatencyThresholdExceeded():
+                #cache results until data is restored
+                webhook_cache.insert(report)
+            # Dump Result into Redis
+            # Set redis flag traceroute::backlog => true
     else:
         print(json.dumps(report, indent=4))
     return 0