python簡單爬蟲

[Python]代碼    

import re
import urllib
import urllib.request
from collections import deque

queue = deque()#存放待爬取的網址
visited = set()#存放爬取過的網址。判斷是否爬取過

url = "http://news.dbanotes.net"#入口網站
queue.append(url)
count = 1

while queue:
    url = queue.popleft()#刪除已經爬取過的隊首的網址url
    visited |= {url}#把已經爬取過的頁面放入set中，方便下面的判斷
    urlop = urllib.request.urlopen(url)
    if 'html' not in urlop.getheader('Content-Type'):
        continue#如果是html再繼續爬取
    try:
        data = urlop.read().decode('utf-8')
    except:
        continue
    value = re.findall(r'href="(.+?)"',data)
    for x in value:
        if 'http' in x and x not in visited:
            print("加入隊列：" + x)