import sqlite3 import urllib from urlparse import urljoin from urlparse import urlparse from BeautifulSoup import * conn = sqlite3.connect('content.db') cur = conn.cursor() cur.execute('''CREATE TABLE IF NOT EXISTS Webs (url TEXT UNIQUE)''') cur.execute('''CREATE TABLE IF NOT EXISTS Links (from_id INTEGER, to_id INTEGER)''') cur.execute('''CREATE TABLE IF NOT EXISTS Pages (id INTEGER PRIMARY KEY, url TEXT UNIQUE, html TEXT, old_rank REAL, new_rank REAL)''') cur.execute('''CREATE TABLE IF NOT EXISTS Webs (url TEXT UNIQUE)''') cur.execute('''CREATE TABLE IF NOT EXISTS Links (from_id INTEGER, to_id INTEGER)''') # Get the current webs cur.execute('''SELECT url FROM Webs''') webs = list() for row in cur: webs.append(str(row[0])) starturl = raw_input('Enter new web url or enter: ') if ( len(starturl) < 1 and len(webs) < 1 ) : starturl = 'http://www.dr-chuck.com/' if len(starturl) > 1 and starturl not in webs : webs.append(starturl) cur.execute('INSERT OR IGNORE INTO Webs (url) VALUES ( ? )', ( starturl, ) ) conn.commit() print 'Webs:', webs if ( len(starturl) > 1 ) : cur.execute('INSERT OR IGNORE INTO Pages (url, html, new_rank) VALUES ( ?, NULL, 1.0 )', ( starturl, ) ) conn.commit() many = 0 while True: if ( many < 1 ) : sval = raw_input('How many pages:') if ( len(sval) < 1 ) : break many = int(sval) many = many - 1 # If this is a brand new web, lets hit it right away cur.execute('SELECT id,url FROM Pages WHERE html is NULL AND url=?', (starturl,)) try: row = cur.fetchone() except: row = None if row is None : cur.execute('SELECT id,url FROM Pages WHERE html is NULL ORDER BY RANDOM() LIMIT 1') try: row = cur.fetchone() print row except: row = None if row is None: print 'No unretrieved HTML pages found' many = 0 break # print row fromid = row[0] url = row[1] print fromid, url, document = urllib.urlopen(url) try: html = document.read() if 'text/html' != document.info().gettype() : print "Ignore non text/html page" cur.execute('DELETE FROM Pages WHERE url=?', ( url, ) ) conn.commit() continue soup = BeautifulSoup(html) except: print "Unable to parse page" cur.execute('DELETE FROM Pages WHERE url=?', ( url, ) ) conn.commit() continue cur.execute('INSERT OR IGNORE INTO Pages (url, html, new_rank) VALUES ( ?, NULL, 1.0 )', ( url, ) ) cur.execute('UPDATE Pages SET html=? WHERE url=?', (buffer(html), url ) ) cur.execute('DELETE from Links WHERE from_id=?', (fromid, ) ) conn.commit() # Retrieve all of the anchor tags tags = soup('a') count = 0 for tag in tags: href = tag.get('href', None) if ( href is None ) : continue # Resolve relative references like href="/contact" up = urlparse(href) if ( len(up.scheme) < 1 ) : href = urljoin(url, href) found = False for web in webs: if ( href.startswith(web) ) : found = True break if not found : continue ipos = href.find('#') if ( ipos > 1 ) : href = href[:ipos] if ( href.endswith('.png') or href.endswith('.jpg') or href.endswith('.gif') ) : continue # print href if ( len(href) < 1 ) : continue cur.execute('INSERT OR IGNORE INTO Pages (url, html, new_rank) VALUES ( ?, NULL, 1.0 )', ( href, ) ) count = count + 1 conn.commit() cur.execute('SELECT id FROM Pages WHERE url=? LIMIT 1', ( href, )) try: row = cur.fetchone() toid = row[0] except: print 'Could not retrieve id' continue # print fromid, toid cur.execute('INSERT OR IGNORE INTO Links (from_id, to_id) VALUES ( ?, ? )', ( fromid, toid ) ) print count cur.close()