import sqlite3 conn = sqlite3.connect('content.db') cur = conn.cursor() # Find the ids that send out page rank cur.execute('''SELECT DISTINCT from_id FROM Links''') from_ids = list() for row in cur: from_ids.append(row[0]) # print from_ids # Find the ids that receive page rank to_ids = list() links = list() cur.execute('''SELECT DISTINCT from_id, to_id FROM Links''') for row in cur: from_id = row[0] to_id = row[1] if from_id == to_id : continue if from_id not in from_ids : continue if to_id not in from_ids : continue links.append(row) if to_id not in to_ids : to_ids.append(to_id) # print to_ids # print links # Get old page ranks for scc prev_ranks = dict() for node in from_ids: cur.execute('''SELECT new_rank FROM Pages WHERE id = ?''', (node, )) row = cur.fetchone() prev_ranks[node] = row[0] # print prev_ranks sval = raw_input('How many iterations:') many = 1 if ( len(sval) > 0 ) : many = int(sval) # Lets do Page Rank for i in range(many): # print prev_ranks.items()[:5] next_ranks = dict(); total = 0.0 for (node, old_rank) in prev_ranks.items(): total = total + old_rank next_ranks[node] = 0.0 # print total # Find the number of outbound links and sent the page rank down each for (node, old_rank) in prev_ranks.items(): # print node, old_rank give_ids = list() for (from_id, to_id) in links: if from_id != node : continue # print ' ',from_id,to_id if to_id not in to_ids: continue give_ids.append(to_id) if ( len(give_ids) < 1 ) : continue amount = old_rank / len(give_ids) # print node, old_rank,amount, give_ids for id in give_ids: next_ranks[id] = next_ranks[id] + amount newtot = 0 for (node, next_rank) in next_ranks.items(): newtot = newtot + next_rank evap = (total - newtot) / len(next_ranks) # print newtot, evap for node in next_ranks: next_ranks[node] = next_ranks[node] + evap newtot = 0 for (node, next_rank) in next_ranks.items(): newtot = newtot + next_rank print i+1, abs(total-newtot) # rotate prev_ranks = next_ranks print next_ranks.items()[:5] cur.execute('''UPDATE Pages SET old_rank=new_rank''') for (id, new_rank) in next_ranks.items() : cur.execute('''UPDATE Pages SET new_rank=? WHERE id=?''', (new_rank, id)) conn.commit() print "Creating JSON output on content.json..." cur.execute('''SELECT COUNT(from_id) AS inbound, old_rank, new_rank, id, url FROM Pages JOIN Links ON Pages.id = Links.to_id WHERE html IS NOT NULL GROUP BY id ORDER BY id,inbound''') fhand = open('content.json','w') nodes = list() maxrank = None minrank = None for row in cur : nodes.append(row) rank = row[2] if maxrank < rank or maxrank is None : maxrank = rank if minrank > rank or minrank is None : minrank = rank fhand.write('{"nodes":[\n') count = 0 map = dict() ranks = dict() for row in nodes : if count > 0 : fhand.write(',\n') # print row rank = row[2] rank = 19 * ( (rank - minrank) / (maxrank - minrank) ) fhand.write('{'+'"weight":'+str(row[0])+',"rank":'+str(rank)+',') fhand.write(' "id":'+str(row[3])+', "url":"'+row[4]+'"}') map[row[3]] = count ranks[row[3]] = rank count = count + 1 fhand.write('],\n') cur.execute('''SELECT DISTINCT from_id, to_id FROM Links''') fhand.write('"links":[\n') count = 0 for row in cur : # print row if row[0] not in map or row[1] not in map : continue if count > 0 : fhand.write(',\n') rank = ranks[row[0]] srank = 19 * ( (rank - minrank) / (maxrank - minrank) ) fhand.write('{"source":'+str(map[row[0]])+',"target":'+str(map[row[1]])+',"value":1}') count = count + 1 fhand.write(']}') fhand.close() cur.close()