a little refactoring on the way to making a pluggable architecture.

.gitignore

@@ -1,6 +1,8 @@
 # project
 settings.py
 output.log
+.project
+.pydevproject
 
 # python specific
 *.pyc

.project

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>dumpmon</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.python.pydev.PyDevBuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.python.pydev.pythonNature</nature>
+	</natures>
+</projectDescription>

.pydevproject

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?eclipse-pydev version="1.0"?><pydev_project>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">Default</pydev_property>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.7</pydev_property>
+</pydev_project>

dumpmon.py

@@ -12,12 +12,17 @@
 from lib.Pastebin import Pastebin, PastebinPaste
 from lib.Slexy import Slexy, SlexyPaste
 from lib.Pastie import Pastie, PastiePaste
-from lib.helper import log
+from lib.HaveIBeen import HaveIBeen, HaveIBeenPaste
+
+from lib.helper import log, createThread
+from lib.TwitterBot import TwitterBot
+from lib.RegexMgr import RegexMgr
+from lib.Stats import Stats
 from time import sleep
-from twitter import Twitter, OAuth
-from settings import CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET, log_file
+from settings import log_file
 import threading
 import logging
+from logging.handlers import RotatingFileHandler
 
 
 def monitor():
@@ -30,36 +35,55 @@ def monitor():
     parser.add_argument(
         "-v", "--verbose", help="more verbose", action="store_true")
     args = parser.parse_args()
+
     level = logging.INFO
     if args.verbose:
         level = logging.DEBUG
+
     logging.basicConfig(
-        format='%(asctime)s [%(levelname)s] %(message)s', filename=log_file, level=level)
+        format='%(asctime)s [%(levelname)s][%(module)s][%(funcName)s] %(message)s', filename=log_file, level=level)
+
+    handler = RotatingFileHandler(log_file, maxBytes=20*1000,
+                                  backupCount=5)
+    #logging.addHandler(handler)
+
     logging.info('Monitoring...')
-    bot = Twitter(
-        auth=OAuth(ACCESS_TOKEN, ACCESS_TOKEN_SECRET,
-            CONSUMER_KEY, CONSUMER_SECRET)
-        )
-    # Create lock for both output log and tweet action
+
+    regexMgr = RegexMgr()
+    bot = TwitterBot(regexMgr)
+                        
+    # Create lock for output log
     log_lock = threading.Lock()
-    tweet_lock = threading.Lock()
-
-    pastebin_thread = threading.Thread(
-        target=Pastebin().monitor, args=[bot, tweet_lock])
-    slexy_thread = threading.Thread(
-        target=Slexy().monitor, args=[bot, tweet_lock])
-    pastie_thead = threading.Thread(
-        target=Pastie().monitor, args=[bot, tweet_lock])
-
-    for thread in (pastebin_thread, slexy_thread, pastie_thead):
-        thread.daemon = True
-        thread.start()
+
+    #create an event to tell threads to keep running
+    isRunning = threading.Event()
+    isRunning.set()
+
+    #array to keep a handle on threads    
+    workers = []         
+
+    #these next 2 workers don't need to be joined when termd
+    createThread(bot.monitor)
+    createThread(Stats().monitor,bot)
+
+    #these workers need to be shut down gracefully
+    workers.append(createThread(HaveIBeen().monitor,bot,isRunning))
+    workers.append(createThread(Pastebin().monitor,bot,isRunning))
+    workers.append(createThread(Slexy().monitor,bot,isRunning))
+    workers.append(createThread(Pastie().monitor,bot,isRunning))
 
     # Let threads run
     try:
         while(1):
             sleep(5)
     except KeyboardInterrupt:
+        #signal threads to shutdown
+        isRunning.clear()
+        print 'stopping'
+        #wait for threads to join
+        for t in workers:
+            t.join()
+        print 'stopped'    
         logging.warn('Stopped.')
 
 

lib/HaveIBeen.py

@@ -0,0 +1,60 @@
+"""
+Troy Hunt's RSS Feed for the last 50 pastes
+
+http://feeds.feedburner.com/HaveIBeenPwnedLatestPastes
+
+"""
+import feedparser
+
+from .Site import Site
+from .Paste import Paste
+from bs4 import BeautifulSoup
+from . import helper
+from time import sleep
+from settings import SLEEP_HAVEIBEEN
+from twitter import TwitterError
+import logging
+
+class HaveIBeenPaste(Paste):
+    def __init__(self, id):
+        super(HaveIBeenPaste, self).__init__(id)
+        self.headers = None
+        self.url = 'http://pastebin.com/raw.php?i=' + self.id
+
+    def get(self):
+        self.text =  helper.curl(self.url)
+
+class HaveIBeen(Site):
+    def __init__(self):
+        super(HaveIBeen, self).__init__()
+        self.sleep = SLEEP_HAVEIBEEN
+        logging.info('[+] Started HaveIBeen')
+        self.feedURL = 'http://feeds.feedburner.com/HaveIBeenPwnedLatestPastes'
+
+    def _parse(self):
+        try:
+            d = feedparser.parse(self.feedURL)
+            return d['entries']
+        except Exception as e:
+            logging.error('[!] Feed Parser Error: %s'%(str(e)))
+            return None
+
+    def update(self):
+        logging.debug('Retrieving HaveIBeenPwned ID\'s')
+        i=0 
+
+        for entry in self._parse():
+            l = entry['links'][0]['href']
+            link = l.split(r'/')
+            paste = HaveIBeenPaste(link[3])
+            if not self.hasSeen(paste):
+                i+=1
+                self.put(paste)
+        logging.debug('HaveIBeenPwned Added URLs: ' + str(i))
+
+
+
+
+if __name__ == '__main__':
+    c = HaveIBeen()
+    c.update()

lib/Paste.py

@@ -2,22 +2,47 @@
 import settings
 import logging
 import re
+import time
 
 class Paste(object):
-    def __init__(self):
+    def __init__(self,id):
         '''
         class Paste: Generic "Paste" object to contain attributes of a standard paste
-
         '''
-        self.emails = 0
-        self.hashes = 0
+        self.id = id
+        self.emails = []
+        self.emails2 = []
+        self.hashes = []
         self.num_emails = 0
         self.num_hashes = 0
         self.text = None
         self.type = None
         self.sites = None
         self.db_keywords = 0.0
-
+
+    def __eq__(self,comparePaste):
+        #logging.info('id %s compares to %s'%(self.id, comparePaste.id))
+        return self.id == comparePaste.id
+
+    def row(self):
+        return {
+                'pid' : self.id,
+                'text' : self.text,
+                'emails' : self.emails,
+                'hashes' : self.hashes,
+                'num_emails' : self.num_emails,
+                'num_hashes' : self.num_hashes,
+                'type' : self.type,
+                'db_keywords' : self.db_keywords,
+                'url' : self.url,
+                "added":time.strftime("%c")
+               }
+
+    def get(self):
+        #override this
+        logging.error('[@] Function Not Implemented in Subclass')
+        pass
+
     def match(self):
         '''
         Matches the paste against a series of regular expressions to determine if the paste is 'interesting'
@@ -32,33 +57,64 @@ def match(self):
 
         '''
         # Get the amount of emails
-        self.emails = list(set(regexes['email'].findall(self.text)))
-        self.hashes = regexes['hash32'].findall(self.text)
-        self.num_emails = len(self.emails)
-        self.num_hashes = len(self.hashes)
-        if self.num_emails > 0:
-            self.sites = list(set([re.search('@(.*)$', email).group(1).lower() for email in self.emails]))
-        for regex in regexes['db_keywords']:
-            if regex.search(self.text):
-                logging.debug('\t[+] ' + regex.search(self.text).group(1))
-                self.db_keywords += round(1/float(
-                    len(regexes['db_keywords'])), 2)
-        for regex in regexes['blacklist']:
-            if regex.search(self.text):
-                logging.debug('\t[-] ' + regex.search(self.text).group(1))
-                self.db_keywords -= round(1.25 * (
-                    1/float(len(regexes['db_keywords']))), 2)
-        if (self.num_emails >= settings.EMAIL_THRESHOLD) or (self.num_hashes >= settings.HASH_THRESHOLD) or (self.db_keywords >= settings.DB_KEYWORDS_THRESHOLD):
-            self.type = 'db_dump'
-        if regexes['cisco_hash'].search(self.text) or regexes['cisco_pass'].search(self.text):
-            self.type = 'cisco'
-        if regexes['honeypot'].search(self.text):
-            self.type = 'honeypot'
-        if regexes['google_api'].search(self.text):
-            self.type = 'google_api'
-        # if regexes['juniper'].search(self.text): self.type = 'Juniper'
-        for regex in regexes['banlist']:
-            if regex.search(self.text):
-                self.type = None
-                break
-        return self.type
+        try:
+            r = self.text.splitlines()
+            logging.debug("[*] Num Lines in text: %i"%(len(r)))           
+
+            if regexes['email'].search(self.text):
+                self.emails = regexes['email'].findall(self.text)
+
+            if regexes['email2'].search(self.text):
+                self.emails2 = regexes['email2'].findall(self.text)
+
+            self.hashes = regexes['hash32'].findall(self.text)
+
+            self.num_emails = len(self.emails)
+            logging.debug("[*] Num Emails: %i"%(self.num_emails))
+
+            self.num_emails = len(self.emails2)
+            logging.debug("[*] Num Emails2: %i"%(self.num_emails))
+
+            self.num_hashes = len(self.hashes)
+            logging.debug("[*] Num Hashes: %i"%(self.num_hashes))
+
+            if self.num_emails > 0:
+                self.sites = list(set([re.search('@(.*)$', email).group(1).lower() for email in self.emails]))
+                logging.debug("[*] Num Sites: %i"%(len(self.sites)))
+
+            for regex in regexes['db_keywords']:
+                if regex.search(self.text):
+                    logging.debug('\t[+] ' + regex.search(self.text).group(1))
+                    self.db_keywords += round(1/float(
+                        len(regexes['db_keywords'])), 2)
+
+            for regex in regexes['blacklist']:
+                if regex.search(self.text):
+                    logging.debug('\t[-] ' + regex.search(self.text).group(1))
+                    self.db_keywords -= round(1.25 * (
+                        1/float(len(regexes['db_keywords']))), 2)
+
+            if (self.num_emails >= settings.EMAIL_THRESHOLD) or (self.num_hashes >= settings.HASH_THRESHOLD) or (self.db_keywords >= settings.DB_KEYWORDS_THRESHOLD):
+                self.type = 'db_dump'
+
+            if regexes['cisco_hash'].search(self.text) or regexes['cisco_pass'].search(self.text):
+                self.type = 'cisco'
+
+            if regexes['honeypot'].search(self.text):
+                self.type = 'honeypot'
+
+            if regexes['google_api'].search(self.text):
+                self.type = 'google_api'
+
+            # if regexes['juniper'].search(self.text): self.type = 'Juniper'
+            for regex in regexes['banlist']:
+                if regex.search(self.text):
+                    self.type = None
+                    break
+
+            logging.debug("[*] Type: %s"%(self.type))    
+            return self.type
+
+        except Exception as e:
+            logging.error("[!] Error: %s"%(str(e)))
+            return None