当前位置：首页 > 资讯 > 技术文档

Python使用代理抓取网站图片（多线程）

时间：2021-08-24 17:18 编辑：来源：阅读：
扫一扫，手机访问

摘要：Python使用代理抓取网站图片（多线程）

一、功能说明： 1. 多线程方式抓取代理服务器，并多线程验证代理服务器 ps 代理服务器是从http://www.cnproxy.com/ （测试只选择了8个页面）抓取 2. 抓取一个网站的图片地址，多线程随机取一个代理服务器下载图片二、实现代码

[url=(.+?)] pic_re = re.compile(r'<img src="(.*?.w{3,4})"') url_list = re.findall(url_re,home_page.read()) for url in url_list: #print url_home+url url_page = urllib2.urlopen(url_home+url) for imgurlList in re.findall(pic_re,url_page.read()): imgurl_list.append(imgurlList) #下载图片的类 class getPic(threading.Thread): def __init__(self,imgurl_list): threading.Thread.__init__(self) self.imgurl_list = imgurl_list self.timeout = 5 def downloadimg(self): for imgurl in self.imgurl_list: pic_suffix = imgurl.split('.')[-1] #获取图片后缀 pic_name = str(random.randint(0,10000000000))+'.'+pic_suffix cookies = urllib2.HTTPCookieProcessor() randomCheckedProxy = random.choice(checkedProxyList) #随机取一组代理服务器 proxyHandler = urllib2.ProxyHandler({"http" : r'http://%s:%s' %(randomCheckedProxy[0],randomCheckedProxy[1])}) opener = urllib2.build_opener(cookies,proxyHandler) opener.addheaders = [('User-agent', 'Mozilla/5.0 (Windows NT 6.2; WOW64; rv:22.0) Gecko/20100101 Firefox/22.0')] urllib2.install_opener(opener) try: data_img = opener.open(imgurl,timeout=self.timeout) f = open (pic_name,'wb') f.write(data_img.read()) f.close() except: continue def run(self): self.downloadimg() if __name__ == "__main__": getThreads = [] checkThreads = [] imgurlList('http://www.ivsky.com') getPicThreads = [] #对每个目标网站开启一个线程负责抓取代理 for i in range(len(targets)): t = ProxyGet(targets[i]) getThreads.append(t) for i in range(len(getThreads)): getThreads[i].start() for i in range(len(getThreads)): getThreads[i].join() print '.'*10+"总共抓取了%s个代理" %len(rawProxyList) +'.'*10 #开启20个线程负责校验，将抓取到的代理分成20份，每个线程校验一份 for i in range(20): t = ProxyCheck(rawProxyList[((len(rawProxyList)+19)/20) * i:((len(rawProxyList)+19)/20) * (i+1)]) checkThreads.append(t) for i in range(len(checkThreads)): checkThreads[i].start() for i in range(len(checkThreads)): checkThreads[i].join() print '.'*10+"总共有%s个代理通过校验" %len(checkedProxyList) +'.'*10 #开启20个线程随机取一个代理下载图片 for i in range(20): t = getPic(imgurl_list[((len(imgurl_list)+19)/20) * i:((len(imgurl_list)+19)/20) * (i+1)]) getPicThreads.append(t) for i in range(len(getPicThreads)): getPicThreads[i].start() for i in range(len(getPicThreads)): getPicThreads[i].join() print '.'*10+"总共有%s个图片下载" %len(imgurl_list) +'.'*10 #代理排序持久化 f= open("proxy_list.txt",'w+') for proxy in sorted(checkedProxyList,cmp=lambda x,y:cmp(x[3],y[3])): #print "checked proxy is: %s:%st%st%s" %(proxy[0],proxy[1],proxy[2],proxy[3]) f.write("%s:%st%st%sn"%(proxy[0],proxy[1],proxy[2],proxy[3])) f.close()

二、测试结果：

[u]复制代码[/u][/url] 代码如下:

全部评论(0)

上一篇：python 读取excel文件生成sql文件实例详解
下一篇：python实现简单的socket server实例

资讯排行榜
更多>>