Add a new class of chaos monkey errors after the inner code has run. Lower the overall error rate to reduce traffic.

config.py

@@ -6,4 +6,4 @@ NAMESPACE = 'cosmopolite'
 CHANNEL_DURATION_SECONDS = 60 * 60 * 2  # 2 hours
 
 # Probabilities
-CHAOS_PROBABILITY = 0.50
+CHAOS_PROBABILITY = 0.05

lib/utils.py

@@ -52,7 +52,15 @@ def chaos_monkey(handler):
       self.response.headers['Retry-After'] = '0'
       self.error(503)
       return
-    return handler(self)
+
+    ret = handler(self)
+
+    if random.random() < config.CHAOS_PROBABILITY:
+      self.response.headers['Retry-After'] = '0'
+      self.error(503)
+      return
+
+    return ret
 
   return IntroduceFailures