Java多線程Web爬蟲 Crawler4j

Crawler4j是一個開源的Java Web爬蟲，提供一個用于抓取Web頁面的簡單接口。您可以在5分鐘內建立一個多線程的網絡爬蟲！

示例代碼：

import java.util.ArrayList;
import java.util.regex.Pattern;

import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.url.WebURL;

public class MyCrawler extends WebCrawler {

        Pattern filters = Pattern.compile(".*(\\.(css|js|bmp|gif|jpe?g"
                + "|png|tiff?|mid|mp2|mp3|mp4"
                + "|wav|avi|mov|mpeg|ram|m4v|pdf"
                + "|rm|smil|wmv|swf|wma|zip|rar|gz))$");

        public MyCrawler() {
        }

        public boolean shouldVisit(WebURL url) {
                String href = url.getURL().toLowerCase();
                if (filters.matcher(href).matches()) {
                        return false;
                }
                if (href.startsWith("http://www.ics.uci.edu/")) {
                        return true;
                }
                return false;
        }

        public void visit(Page page) {
                int docid = page.getWebURL().getDocid();
                String url = page.getWebURL().getURL();        
                String text = page.getText();
                ArrayList<WebURL> links = page.getURLs();              
        }
}

項目主頁：http://www.baiduhome.net/lib/view/home/1350054122400