案例：使用BeautifuSoup4的爬蟲

我們以騰訊社招頁面來做演示：http://hr.tencent.com/position.php?&start=10#a

使用BeautifuSoup4解析器，將招聘網頁上的職位名稱、職位類別、招聘人數、工作地點、發佈時間，以及每個職位詳情的點擊鏈接存儲出來。

# bs4_tencent.py


from bs4 import BeautifulSoup
import urllib
import json # 使用了json格式存儲

def tencent():
 url = 'http://hr.tencent.com/'
 request = urllib.request.Request(url + 'position.php?&start=10#a')
 response =urllib.request.urlopen(request)
 resHtml = response.read()

 output =open('tencent.json','w')

 html = BeautifulSoup(resHtml,'lxml')

# 創建CSS選擇器
 result = html.select('tr[class="even"]')
 result2 = html.select('tr[class="odd"]')
 result += result2

 items = []
 for site in result:
 item = {}

 name = site.select('td a')[0].get_text()
 detailLink = site.select('td a')[0].attrs['href']
 catalog = site.select('td')[1].get_text()
 recruitNumber = site.select('td')[2].get_text()
 workLocation = site.select('td')[3].get_text()
 publishTime = site.select('td')[4].get_text()

 item['name'] = name
 item['detailLink'] = url + detailLink
 item['catalog'] = catalog
 item['recruitNumber'] = recruitNumber
 item['publishTime'] = publishTime

 items.append(item)

 # 禁用ascii編碼，按utf-8編碼
 line = json.dumps(items,ensure_ascii=False)

 output.write(line.encode('utf-8'))
 output.close()

if __name__ == "__main__":
 tencent()

閱讀更多 IT技術資源共享 的文章

關鍵字: 網絡爬蟲 Line 社招