experiment: Add multiple failure experiment

Change-Id: Ia7e856606ec483f0bb63840a0e40399058e5b6ac
diff --git a/ndn/experiments/experiment.py b/ndn/experiments/experiment.py
index 9487e22..80fba81 100644
--- a/ndn/experiments/experiment.py
+++ b/ndn/experiments/experiment.py
@@ -58,13 +58,16 @@
                 host.nfd.stop()
             sys.exit(1)
 
+    def ping(self, source, dest, nPings):
+        # Use "&" to run in background and perform parallel pings
+        print "Scheduling ping(s) from %s to %s" % (source.name, dest.name)
+        source.cmd("ndnping -t -c "+ str(nPings) + " /ndn/edu/" + dest.name + " >> ping-data/" + dest.name + ".txt &")
+        time.sleep(0.2)
+
     def startPings(self):
         for host in self.net.hosts:
             for other in self.net.hosts:
                 # Do not ping self
                 if host.name != other.name:
-                    # Use "&" to run in background and perform parallel pings
-                    print "Scheduling ping(s) from %s to %s" % (host.name, other.name)
-                    host.cmd("ndnping -t -c "+ str(self.nPings) + " /ndn/edu/" + other.name + " > ping-data/" + other.name + ".txt &")
-                    time.sleep(0.2)
+                    self.ping(host, other, self.nPings)
 
diff --git a/ndn/experiments/multiple_failure_experiment.py b/ndn/experiments/multiple_failure_experiment.py
new file mode 100644
index 0000000..db7d536
--- /dev/null
+++ b/ndn/experiments/multiple_failure_experiment.py
@@ -0,0 +1,67 @@
+#!/usr/bin/python
+
+from ndn.experiments.experiment import Experiment
+from ndn.nlsr import Nlsr
+
+import time
+
+class MultipleFailureExperiment(Experiment):
+
+    def __init__(self, net, nodes, convergenceTime, strategy):
+
+        self.PING_COLLECTION_TIME_BEFORE_FAILURE = 60
+
+        self.FAILURE_INTERVAL = 60
+        self.RECOVERY_INTERVAL = 60
+
+        # This is the number of pings required to make it through the full experiment
+        nInitialPings = self.PING_COLLECTION_TIME_BEFORE_FAILURE + len(net.hosts)*(self.FAILURE_INTERVAL + self.RECOVERY_INTERVAL)
+        print("Scheduling with %s initial pings" % nInitialPings)
+
+        Experiment.__init__(self, net, nodes, convergenceTime, nInitialPings, strategy)
+
+    def failNode(self, host):
+        print("Bringing %s down" % host.name)
+        host.nfd.stop()
+
+    def recoverNode(self, host):
+        print("Bringing %s up" % host.name)
+        host.nfd.start()
+        host.nlsr.start()
+        host.nfd.setStrategy("/ndn/edu", self.strategy)
+        host.cmd("ndnpingserver /ndn/edu/" + str(host) + " > ping-server &")
+
+    def run(self):
+        self.startPings()
+
+        # After the pings are scheduled, collect pings for 1 minute
+        time.sleep(self.PING_COLLECTION_TIME_BEFORE_FAILURE)
+
+        nNodesRemainingToFail = len(self.net.hosts)
+
+        # Fail and recover each node
+        for host in self.net.hosts:
+            # Fail the node
+            self.failNode(host)
+
+            # Stay in failure state for FAILURE_INTERVAL
+            time.sleep(self.FAILURE_INTERVAL)
+
+            # Bring the node back up
+            self.recoverNode(host)
+
+            # Number of pings required to reach the end of the test
+            nPings = self.RECOVERY_INTERVAL + nNodesRemainingToFail*(self.FAILURE_INTERVAL + self.RECOVERY_INTERVAL)
+            nNodesRemainingToFail = nNodesRemainingToFail - 1
+
+            # Wait for NFD and NLSR to fully recover
+            time.sleep(1)
+            print("Scheduling with %s remaining pings" % nPings)
+
+            # Restart pings
+            for other in self.net.hosts:
+                # Do not ping self
+                if host.name != other.name:
+                    self.ping(host, other, nPings)
+
+            time.sleep(self.RECOVERY_INTERVAL)