最近我們要對站點的指定 url 測試其頁面請求完整性，并分析每個請求參數是否正常。如果不使用自動化測試工具，你當然可以人肉點擊、查看是否頁面請求遺漏和每個請求參數是否正常。當然也有如 Fiddler 之類的抓包工具，但這個工具貌似沒開源代碼，也沒有提供接口供第三方調用和擴展。找來找去發現 Selenium 可以滿足當前的業務需求。

Selenium是一個用于Web應用程序測試的工具。Selenium測試直接運行在瀏覽器中，就像真正的用戶在操作一樣。支持的瀏覽器包括IE(7、8、9)、Mozilla Firefox、Mozilla Suite等。這個工具的主要功能包括：測試與瀏覽器的兼容性——測試你的應用程序看是否能夠很好得工作在不同瀏覽器和操作系統之上。測試系統功能——創建衰退測試檢驗軟件功能和用戶需求。支持自動錄制動作和自動生成 .Net、Java、Perl等不同語言的測試腳本。Selenium 是ThoughtWorks專門為Web應用程序編寫的一個驗收測試工具。

1、Selenium 抓取請求的原理

其實 Selenium 抓取請求的原理和 fiddler 之類的工具是類似的：

只是這里 Selenium 代理取代了 Fiddler 代理，而自動化測試腳本則通過 Selenium 去操縱瀏覽器或者其里面的頁面元素去模擬人的行為，進而完成預設的測試任務。

2、Selenium 自動化測試實例：獲取指定 url 的所有請求并簡單統計分析

廢話就不說了，直接上代碼了。需要提示下的是，如果你想測試下面的代碼，請按如下步驟來：

（1）Install Python
（2）Install Java
（3）Install Selenium bindings for Python
（4）Run Selenium server: 'java -jar selenium-server.jar'
（5）Run web_profiler.py

# -- coding: utf-8 --

!/usr/bin/env python
#
Selenium Web/HTTP Profiler
Copyright (c) 2009-2011 Corey Goldberg (corey@goldb.org)
License: GNU GPLv3
run：
python web_profiler.py http://baidu.com/
import json
import socket
import sys
import time
import urlparse
import xml.etree.ElementTree as etree
from datetime import datetime
from selenium import selenium
def main():
    if len(sys.argv) < 2:
        print 'usage:'
        print '  %s <url> [browser_launcher]' % file
        print 'examples:'
        print '  $ python %s www.google.com' % file
        print '  $ python %s http://www.google.com/ firefox\n' % file
        sys.exit(1)
    else:
        url = sys.argv[1]
        if not url.startswith('http'):
            url = 'http://' + url
        parsed_url = urlparse.urlparse(url)
        site = parsed_url.scheme + '://' + parsed_url.netloc
        path = parsed_url.path
        if path == '':
            path = '/'
        browser = 'firefox'

if len(sys.argv) == 3:
    browser = sys.argv[2]

run(site, path, browser)




def run(site, path, browser):
    sel = selenium('127.0.0.1', 4444, browser, site)

try:
    sel.start('captureNetworkTraffic=true')
except socket.error:
    print 'ERROR - can not start the selenium-rc driver. is your selenium server running?'
    sys.exit(1)

sel.open(path)
sel.wait_for_page_to_load(90000)
end_loading = datetime.now()

raw_xml = sel.captureNetworkTraffic('xml')

sel.stop()

traffic_xml = raw_xml.replace('&', '&amp;').replace('=""GET""', '="GET"').replace('=""POST""', '="POST"') # workaround selenium bugs

nc = NetworkCapture(traffic_xml)

#json_results = nc.get_json()

num_requests = nc.get_num_requests()
total_size = nc.get_content_size()
status_map = nc.get_http_status_codes()
file_extension_map = nc.get_file_extension_stats()
http_details = nc.get_http_details()
start_first_request, end_first_request, end_last_request = nc.get_network_times()

end_load_elapsed = get_elapsed_secs(start_first_request, end_loading)
end_last_request_elapsed = get_elapsed_secs(start_first_request, end_last_request)
end_first_request_elapsed = get_elapsed_secs(start_first_request, end_first_request)

print '--------------------------------'
print 'results for %s' % site

print '\ncontent size: %s kb' % total_size

print '\nhttp requests: %s' % num_requests
for k,v in sorted(status_map.items()):
    print 'status %s: %s' % (k, v)

print '\nprofiler timing:'
print '%.3f secs (page load)' % end_load_elapsed
print '%.3f secs (network: end last request)' % end_last_request_elapsed
print '%.3f secs (network: end first request)' % end_first_request_elapsed

print '\nfile extensions: (count, size)'
for k,v in sorted(file_extension_map.items()):
    print '%s: %i, %.3f kb' % (k, v[0], v[1])

print '\nhttp timing detail: (status, method, doc, size, time)'
for details in http_details:
    print '%i, %s, %s, %i, %i ms' % (details[0], details[1], details[2], details[3], details[4])




def get_elapsed_secs(dt_start, dt_end):
    return float('%.3f' % ((dt_end - dt_start).seconds +
        ((dt_end - dt_start).microseconds / 1000000.0)))
class NetworkCapture(object):
    def init(self, xml_blob):
        self.xml_blob = xml_blob
        self.dom = etree.ElementTree(etree.fromstring(xml_blob))

def get_json(self):
    results = []
    for child in self.dom.getiterator():
        if child.tag == 'entry':
            url = child.attrib.get('url')
            start_time = child.attrib.get('start')
            time_in_millis = child.attrib.get('timeInMillis')
            results.append((url, start_time, time_in_millis))
    return json.dumps(results)


def get_content_size(self):  # total kb passed through the proxy
    byte_sizes = []
    for child in self.dom.getiterator():
        if child.tag == 'entry':
            byte_sizes.append(child.attrib.get('bytes'))
    total_size = sum([int(bytes) for bytes in byte_sizes]) / 1000.0
    return total_size


def get_num_requests(self):
    num_requests = 0
    for child in self.dom.getiterator():
        if child.tag == 'entry':
            num_requests += 1
    return num_requests


def get_http_status_codes(self):
    status_map = {}
    for child in self.dom.getiterator():
        if child.tag == 'entry':
            try:
                status_map[child.attrib.get('statusCode')] += 1
            except KeyError:
                status_map[child.attrib.get('statusCode')] = 1
    return status_map


def get_http_details(self):
    http_details = []
    for child in self.dom.getiterator():
        if child.tag == 'entry':
            url = child.attrib.get('url') + '?'
            # url = child.attrib.get('url')
            print "---->> " + url
            url_stem = url.split('?')[0]
            doc = '/' + url_stem.split('/')[-1]
            status = int(child.attrib.get('statusCode'))
            method = child.attrib.get('method').replace("'", '')
            size = int(child.attrib.get('bytes'))
            time = int(child.attrib.get('timeInMillis'))
            http_details.append((status, method, doc, size, time))
    http_details.sort(cmp=lambda x,y: cmp(x[3], y[3])) # sort by size
    return http_details


def get_file_extension_stats(self):
    file_extension_map = {}  # k=extension v=(count,size)
    for child in self.dom.getiterator():
        if child.tag == 'entry':
            size = float(child.attrib.get('bytes')) / 1000.0
            url = child.attrib.get('url') + '?'
            url_stem = url.split('?')[0]
            doc = url_stem.split('/')[-1]
            if '.' in doc:
                file_extension = doc.split('.')[-1]
            else:
                file_extension = 'unknown'
            try:
                file_extension_map[file_extension][0] += 1
                file_extension_map[file_extension][1] += size
            except KeyError:
                file_extension_map[file_extension] = [1, size]
    return file_extension_map


def get_network_times(self):
    timings = []
    start_times = []
    end_times = []
    for child in self.dom.getiterator():
        if child.tag == 'entry':
            timings.append(child.attrib.get('timeInMillis'))
            start_times.append(child.attrib.get('start'))
            end_times.append(child.attrib.get('end'))
    start_times.sort()
    end_times.sort()
    start_first_request = self.convert_time(start_times[0])
    end_first_request = self.convert_time(end_times[0])
    end_last_request = self.convert_time(end_times[-1])
    return (start_first_request, end_first_request, end_last_request)


def convert_time(self, date_string):
    if '-' in date_string: split_char = '-'
    else: split_char = '+'
    # dt = datetime.strptime(''.join(date_string.split(split_char)[:-1]), '%Y%m%dT%H:%M:%S.%f')
    dt = datetime.strptime(''.join(date_string.split(split_char)[:-1]), '%Y%m')
    return dt




if name == 'main':
    main()</pre> 
最后的結果輸出了所有的請求，并 對這次的請求進行簡單的分析， 包括異步加載的 ajax、圖片啥的，和你在瀏覽器 F12→network 看到的效果類似。 




3、Refer： 

（1）http://docs.seleniumhq.org/ 

（2）Selenium私房菜系列--總章 

http://www.cnblogs.com/hyddd/archive/2009/05/30/1492536.html 
https://code.google.com/p/selenium-profiler/