# 官網(wǎng)解釋
'''
This library intends to make parsing HTML (e.g. scraping the web) as simple and intuitive as possible.
If you're interested in financially supporting Kenneth Reitz open source, consider visiting this link. Your support helps tremendously with sustainability of motivation, as Open Source is no longer part of my day job.
When using this library you automatically get:

Full JavaScript support!
CSS Selectors (a.k.a jQuery-style, thanks to PyQuery).
XPath Selectors, for the faint at heart.
Mocked user-agent (like a real web browser).
Automatic following of redirects.
Connection–pooling and cookie persistence.
The Requests experience you know and love, with magical parsing abilities.
Async Support
'''

# pip3 install requests-html

import requests
from bs4 import BeautifulSoup
url = 'http://www.zuihaodaxue.cn/'
HEADERS = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.157 Safari/537.36'
}
response = requests.get(url, headers=HEADERS)
response.encoding = 'gbk'
# print(response.status_code)
 print(response.text)

soup = BeautifulSoup(response.text, 'lxml')
# 獲取最新的五則新聞
post_rankings = soup.find_all(name='article', attrs={"class": "post_ranking"})
# 循環(huán)打印新聞簡介內(nèi)容
for post_ranking in post_rankings:
new = post_ranking.find(name='div', attrs={"class": 'post_summary'})
print(new.text)

from requests_html import HTMLSession
# 獲取請(qǐng)求對(duì)象
session = HTMLSession()
# 往新浪新聞主頁發(fā)送get請(qǐng)求
sina = session.get('https://news.sina.com.cn/')
# print(sina.status_code)
sina.encoding = 'utf-8'
# 獲取響應(yīng)文本信息，與requests無區(qū)別
 print(sina.text)

from requests_html import HTMLSession
# 獲取請(qǐng)求對(duì)象
session = HTMLSession()
# 往京東主頁發(fā)送get請(qǐng)求
jd = session.get('https://jd.com/')
# 得到京東主頁所有的鏈接，返回的是一個(gè)set集合
print(jd.html.links)
print('*' * 1000)
# 若獲取的鏈接中有相對(duì)路徑，我們還可以通過absolute_links獲取所有絕對(duì)鏈接
print(jd.html.absolute_links)

'''
　　CSS選擇器 and XPATH
 　　1.通過css選擇器選取一個(gè)Element對(duì)象
 　　2.獲取一個(gè)Element對(duì)象內(nèi)的文本內(nèi)容
 　　3.獲取一個(gè)Element對(duì)象的所有attributes
 　　 4.渲染出一個(gè)Element對(duì)象的HTML內(nèi)容
 　　5.獲取Element對(duì)象內(nèi)的特定子Element對(duì)象，返回列表
 　　6.在獲取的頁面中通過search查找文本
 　　 7.支持XPath
 　　 8.獲取到只包含某些文本的Element對(duì)象
'''

from requests_html import HTMLSession
session = HTMLSession()
url = "https://www.qiushibaike.com/text/"
# 獲取響應(yīng)數(shù)據(jù)對(duì)象
obj = session.get(url)
# 1.通過css選擇器選取一個(gè)Element對(duì)象
 獲取id為content-left的div標(biāo)簽，并且返回一個(gè)對(duì)象
content = obj.html.find('div#content-left', first=True)
# 2.獲取一個(gè)Element對(duì)象內(nèi)的文本內(nèi)容
 獲取content內(nèi)所有文本
print(content.text)
# 3.獲取一個(gè)Element對(duì)象的所有attributes
 獲取content內(nèi)所有屬性
print(content.attrs)
# 4.渲染出一個(gè)Element對(duì)象的完整的HTML內(nèi)容
html = content.html
print(html)
# 5.獲取Element對(duì)象內(nèi)的指定的所有子Element對(duì)象，返回列表
a_s = content.find('a')
print(a_s)
print(len(a_s)) # 79
# 循環(huán)所有的a標(biāo)簽
for a in a_s:
# 獲取a標(biāo)簽內(nèi)所有屬性的href屬性 并拼接
href = a.attrs['href']
if href.startswith('/'):
url = 'https://www.qiushibaike.com' + href
print(url)
# 6.在獲取的頁面中通過search查找文本
 {}大括號(hào)相當(dāng)于正則的從頭到后開始匹配，獲取當(dāng)中想要獲取的數(shù)據(jù)
text = obj.html.search('把{}夾')[0] # 獲取從 "把" 到 "夾" 字的所有內(nèi)容
text = obj.html.search('把糗事{}夾')[0] # 獲取從把子到夾字的所有內(nèi)容
print(text)
print('*' * 1000)
# 7.支持XPath
a_s = obj.html.xpath('//a') # 獲取html內(nèi)所有的a標(biāo)簽
for a in a_s:
href = a.attrs['href']
#若是//開頭的url都扔掉
if href.startswith('continue#若是/開頭的都是相對(duì)路徑)
 elif href.startswith('/')
 print('https://www.qiushibaike.com'+href)

# 8.獲取到只包含某些文本的Element對(duì)象（containing）
 獲取所有文本內(nèi)容為幽默笑話大全_爆笑笑話_笑破你的肚子的搞笑段子 - 糗事百科 title標(biāo)簽
 注意: 文本內(nèi)有空格也必須把空格帶上
title = obj.html.find('title', containing='幽默笑話大全_爆笑笑話_笑破你的肚子的搞笑段子 - 糗事百科')
print(title)

from requests_html import HTMLSession
session = HTMLSession()
url = 'http://www.win4000.com/'
obj = session.get(url)
obj.encoding = 'utf-8'
obj.html.render()

# 找到自己的操作系統(tǒng)相應(yīng)的配置位置
'''
chromiumExecutable = {
'linux': DOWNLOADS_FOLDER / REVISION / 'chrome-linux' / 'chrome',
'mac': (DOWNLOADS_FOLDER / REVISION / 'chrome-mac' / 'Chromium.app' /
'Contents' / 'MacOS' / 'Chromium'),
'win32': DOWNLOADS_FOLDER / REVISION / 'chrome-win32' / 'chrome.exe',
'win64': DOWNLOADS_FOLDER / REVISION / 'chrome-win32' / 'chrome.exe',
}
'''
from pyppeteer import __chromium_revision__, __pyppeteer_home__
DOWNLOADS_FOLDER = Path(pyppeteer_home) / 'local-chromium'
REVISION = os.environ.get('PYPPETEER_CHROMIUM_REVISION', chromium_revision)
# 打印這兩個(gè)變量可以知道執(zhí)行的驅(qū)動(dòng)具體位置
print(DOWNLOADS_FOLDER)
print(REVISION)
'''
由上面可以知道：chromium路徑是：C:\Users\Ray\AppData\Local\pyppeteer\pyppeteer\local-chromium\575458\chrome-win32\chrome.exe
所以自己建文件夾，然后一直到chrome-win32文件夾，把上面下載的chromium文件，拷貝到此目錄下
'''

from requests_html import HTMLSession
# pprint可以把數(shù)據(jù)打印得更整齊
from pprint import pprint
import json
get_url = 'http://httpbin.org/get'
session = HTMLSession()
# 返回的是當(dāng)前系統(tǒng)的headers信息
res = session.get(get_url)
pprint(json.loads(res.html.html))
# 可以在發(fā)送請(qǐng)求的時(shí)候更換user-agent
ua = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:62.0) Gecko/20100101 Firefox/62.0'
post_url = 'http://httpbin.org/get'
res = session.get(post_url, headers={'user-agent': ua})
pprint(json.loads(res.html.html))# 如果你有需要可以在header中修改其他參數(shù)。

# 表單登錄
r = session.post('http://httpbin.org/post', data={'username': 'tank_jam', 'password': 'tank9527'})
pprint(json.loads(r.html.html))
''' # 打印結(jié)果
{'args': {},
 'data': '',
 'files': {},
 'form': {'password': 'tank9527', 'username': 'tank_jam'},
 'headers': {'Accept': '*/*',
    'Accept-Encoding': 'gzip, deflate',
    'Content-Length': '35',
    'Content-Type': 'application/x-www-form-urlencoded',
    'Host': 'httpbin.org',
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) '
       'AppleWebKit/603.3.8 (KHTML, like Gecko) '
       'Version/10.1.2 Safari/603.3.8'},
 'json': None,
 'origin': '112.65.61.109, 112.65.61.109',
 'url': 'https://httpbin.org/post'}
'''

from requests_html import AsyncHTMLSession
from requests_html import HTMLSession
import time
# 使用異步發(fā)送請(qǐng)求
async_session = AsyncHTMLSession()
async def get_baidu():
url = 'https://www.baidu.com/'
res = await async_session.get(url)
print(res.html.absolute_links)
async def get_sougou():
url = 'https://www.sogou.com/'
res = await async_session.get(url)
print(res.html.links)
start_time = time.time()
async_session.run(get_baidu, get_sougou)
print('耗時(shí)：', time.time() - start_time)
# 同步發(fā)送請(qǐng)求
session = HTMLSession()
start_time = time.time()
res = session.get('https://www.baidu.com/')
print(res.html.links)
res = session.get('https://www.sogou.com/')
print(res.html.absolute_links)
print('耗時(shí)：', time.time() - start_time)

 from requests_html import HTMLSession
 session = HTMLSession()
 ​
 r = session.get('https://www.python.org/jobs/')

class HTMLSession(requests.Session):
  # 重寫 request 方法，返回 HTMLResponse 構(gòu)造
  def request(self, *args, **kwargs) -> HTMLResponse:
   r = super(HTMLSession, self).request(*args, **kwargs)
   return HTMLResponse._from_response(r, self)

 class HTMLResponse(requests.Response):
 # 構(gòu)造器
  @classmethod
  def _from_response(cls, response, session: Union['HTMLSession', 'AsyncHTMLSession']):
   html_r = cls(session=session)
   html_r.__dict__.update(response.__dict__)
   return html_r

 # css 獲取有多少個(gè)職位
 jobs = r.html.find("h1.call-to-action")
 # xpath 獲取
 jobs = r.html.xpath("http://h1[@class='call-to-action']")

 jobs.text
 jobs.full_text

 attrs = jobs.attrs
 value = attrs.get("key")

 ## 找某些內(nèi)容匹配
 r.html.search("Python {}")
 r.html.search_all()

 r.html.absolute_links
 r.html.links

 print(r.html)
 # 比較一下
 for url in r.html:
  print(url)
 ​

# print(r.html)
 <HTML url='https://www.python.org/jobs/'>
 # for
 <HTML url='https://www.python.org/jobs/'>
 <HTML url='https://www.python.org/jobs/?page=2'>
 <HTML url='https://www.python.org/jobs/?page=3'>
 <HTML url='https://www.python.org/jobs/?page=4'>
 <HTML url='https://www.python.org/jobs/?page=5'>

 def get_next():
 candidates = self.find('a', containing=next_symbol)
 ​
 for candidate in candidates:
 if candidate.attrs.get('href'):
 # Support 'next' rel (e.g. reddit).
 if 'next' in candidate.attrs.get('rel', []):
 return candidate.attrs['href']

r.html.render()