Python作为一门功能强大的脚本语言来说，经常被用来写爬虫程序，下面是Python爬虫多线程抓取代理服务器。
现在改抓取 http://www.proxy.com.ru 的代理服务器，这里的代理服务器更多，更简单。
下面代码可以直接用，代码如下：
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 |
#!/usr/bin/env python #coding:utf-8 #BLOG: blog.linuxeye.com import urllib2 import re import threading import time import MySQLdb rawProxyList = [] checkedProxyList = [] #抓取代理网站 targets = [] for i in xrange(1,42): target = r"http://www.proxy.com.ru/list_%d.html" % i targets.append(target) #抓取代理服务器正则 p = re.compile(r'''<tr><b><td>(\d+)</td><td>(.+?)</td><td>(\d+)</td><td>(.+?)</td><td>(.+?)</td></b></tr>''') #获取代理的类 class ProxyGet(threading.Thread): def __init__(self,target): threading.Thread.__init__(self) self.target = target def getProxy(self): print "代理服务器目标网站： " + self.target req = urllib2.urlopen(self.target) result = req.read() #print chardet.detect(result) matchs = p.findall(result) # print matchs for row in matchs: ip=row[1] port =row[2] addr = row[4].decode("cp936").encode("utf-8") proxy = [ip,port,addr] print proxy rawProxyList.append(proxy) def run(self): self.getProxy() #检验代理的类 class ProxyCheck(threading.Thread): def __init__(self,proxyList): threading.Thread.__init__(self) self.proxyList = proxyList self.timeout = 5 self.testUrl = "http://www.baidu.com/" self.testStr = "030173" def checkProxy(self): cookies = urllib2.HTTPCookieProcessor() for proxy in self.proxyList: proxyHandler = urllib2.ProxyHandler({"http" : r'http://%s:%s' %(proxy[0],proxy[1])}) #print r'http://%s:%s' %(proxy[0],proxy[1]) opener = urllib2.build_opener(cookies,proxyHandler) opener.addheaders = [('User-agent', 'Mozilla/5.0 (Windows NT 6.2; WOW64; rv:22.0) Gecko/20100101 Firefox/22.0')] #urllib2.install_opener(opener) t1 = time.time() try: #req = urllib2.urlopen("http://www.baidu.com", timeout=self.timeout) req = opener.open(self.testUrl, timeout=self.timeout) #print "urlopen is ok...." result = req.read() #print "read html...." timeused = time.time() - t1 pos = result.find(self.testStr) #print "pos is %s" %pos if pos > 1: checkedProxyList.append((proxy[0],proxy[1],proxy[2],timeused)) #print "ok ip: %s %s %s %s" %(proxy[0],proxy[1],proxy[2],timeused) else: continue except Exception,e: #print e.message continue def run(self): self.checkProxy() if __name__ == "__main__": getThreads = [] checkThreads = [] #对每个目标网站开启一个线程负责抓取代理 for i in range(len(targets)): t = ProxyGet(targets[i]) getThreads.append(t) for i in range(len(getThreads)): getThreads[i].start() for i in range(len(getThreads)): getThreads[i].join() print '.'*10+"总共抓取了%s个代理" %len(rawProxyList) +'.'*10 #开启20个线程负责校验，将抓取到的代理分成20份，每个线程校验一份 for i in range(20): t = ProxyCheck(rawProxyList[((len(rawProxyList)+19)/20) * i:((len(rawProxyList)+19)/20) * (i+1)]) checkThreads.append(t) for i in range(len(checkThreads)): checkThreads[i].start() for i in range(len(checkThreads)): checkThreads[i].join() print '.'*10+"总共有%s个代理通过校验" %len(checkedProxyList) +'.'*10 #插入数据库，表结构自己创建，四个字段ip,port,speed,address def db_insert(insert_list): try: conn = MySQLdb.connect(host="localhost", user="root", passwd="admin",db="m_common",charset='utf8') cursor = conn.cursor() cursor.execute('delete from proxy') cursor.execute('alter table proxy AUTO_INCREMENT=1') cursor.executemany("INSERT INTO proxy(ip,port,speed,address) VALUES (%s,%s,%s,%s)",insert_list) conn.commit() cursor.close() conn.close() except MySQLdb.Error,e: print "Mysql Error %d: %s" % (e.args[0], e.args[1]) #代理排序持久化 proxy_ok = [] f= open("proxy_list.txt",'w+') for proxy in sorted(checkedProxyList,cmp=lambda x,y:cmp(x[3],y[3])): if proxy[3] < 8: #print "checked proxy is: %s:%s\t%s\t%s" %(proxy[0],proxy[1],proxy[2],proxy[3]) proxy_ok.append((proxy[0],proxy[1],proxy[3],proxy[2])) f.write("%s:%s\t%s\t%s\n"%(proxy[0],proxy[1],proxy[2],proxy[3])) f.close() db_insert(proxy_ok) 测试结果 # python -V Python 2.6.6 # python proxy.py ['61.58.94.179', '8088', '\xe5\x8f\xb0\xe6\xb9\xbe\xe7\x9c\x81 \xe5\x8f\xb0\xe6\xb9\xbe\xe5\xae\xbd\xe9\xa2\x91\xe9\x80\x9a\xe8\xae\xaf\xe9\xa1\xbe\xe9\x97\xae\xe8\x82\xa1\xe4\xbb\xbd\xe6\x9c\x89\xe9\x99\x90\xe5\x85\xac\xe5\x8f\xb8'] ['200.84.116.99', '9064', '\xe5\xa7\x94\xe5\x86\x85\xe7\x91\x9e\xe6\x8b\x89 '] ['183.223.204.8', '8123', '\xe5\x9b\x9b\xe5\xb7\x9d\xe7\x9c\x81\xe8\x87\xaa\xe8\xb4\xa1\xe5\xb8\x82 \xe7\xa7\xbb\xe5\x8a\xa8'] ..........总共抓取了1921个代理.......... ..........总共有524个代理通过校验.......... |
支付宝扫一扫
