pip3 install requests　　

#各種請求方式：常用的就是requests.get()和requests.post()
import requests

r = requests.get('https://api.github.com/events')
r = requests.post('http://httpbin.org/post', data = {'key':'value'})
r = requests.put('http://httpbin.org/put', data = {'key':'value'})
r = requests.delete('http://httpbin.org/delete')
r = requests.head('http://httpbin.org/get')
r = requests.options('http://httpbin.org/get')

#GET請求
HTTP默認的請求方法就是GET
   * 沒有請求體
   * 數(shù)據(jù)必須在1K之內(nèi)！
   * GET請求數(shù)據(jù)會暴露在瀏覽器的地址欄中

GET請求常用的操作：
    1. 在瀏覽器的地址欄中直接給出URL，那么就一定是GET請求
    2. 點擊頁面上的超鏈接也一定是GET請求
    3. 提交表單時，表單默認使用GET請求，但可以設(shè)置為POST


#POST請求
(1). 數(shù)據(jù)不會出現(xiàn)在地址欄中
(2). 數(shù)據(jù)的大小沒有上限
(3). 有請求體
(4). 請求體中如果存在中文，會使用URL編碼！


#！??！requests.post()用法與requests.get()完全一致，特殊的是requests.post()有一個data參數(shù)，用來存放請求體數(shù)據(jù)

import requests
response=requests.get('http://http://www.cnblogs.com/')
print(response.text)

#在請求頭內(nèi)將自己偽裝成瀏覽器，否則百度不會正常返回頁面內(nèi)容
import requests

response=requests.get('https://www.baidu.com/s?wd=python&pn=1',
           #請求頭信息
           headers={
            'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.75 Safari/537.36',
           })
print(response.text)


#如果查詢關(guān)鍵詞是中文或者有其他特殊符號，則不得不進行url編碼后再拼接
from urllib.parse import urlencode
wd='蒼老師'
encode_res=urlencode({'k':wd},encoding='utf-8')
keyword=encode_res.split('=')[1]#拿到編碼后的字符串
print(keyword)
# 然后拼接成url
url='https://www.baidu.com/s?wd=%s&pn=1' %keyword

response=requests.get(url,
           headers={
            'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.75 Safari/537.36',
           })
res1=response.text

#上述操作可以用requests模塊的一個params參數(shù)搞定，本質(zhì)還是調(diào)用urlencode
from urllib.parse import urlencode
wd='egon老師'
pn=1

response=requests.get('https://www.baidu.com/s',
           #參數(shù)進行傳參，幫我們省去了urlencode這一步
           params={
             'wd':wd,
             'pn':pn
           },
           headers={
            'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.75 Safari/537.36',
           })
res2=response.text

#驗證結(jié)果，打開a.html與b.html頁面內(nèi)容一樣
with open('a.html','w',encoding='utf-8') as f:
  f.write(res1) 
with open('b.html', 'w', encoding='utf-8') as f:
  f.write(res2)

params參數(shù)的使用

#通常我們在發(fā)送請求時都需要帶上請求頭，請求頭是將自身偽裝成瀏覽器的關(guān)鍵，常見的有用的請求頭如下
Host
Referer  #大型網(wǎng)站通常都會根據(jù)該參數(shù)判斷請求的來源
User-Agent #客戶端
Cookie   #Cookie信息包含在請求頭里，requests模塊有單獨的參數(shù)來處理他，處理后headers={}內(nèi)就可以不用放置

#添加headers(瀏覽器會識別請求頭,不加可能會被拒絕訪問,比如訪問https://www.zhihu.com/explore)
import requests
response=requests.get('https://www.zhihu.com/explore')
response.status_code #500


#自己定制headers
headers={
  'User-Agent':'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.76 Mobile Safari/537.36',

}
respone=requests.get('https://www.zhihu.com/explore',
           headers=headers)
print(respone.status_code) #200

設(shè)置請求頭的方式

#登錄github，然后從瀏覽器中獲取cookies，以后就可以直接拿著cookie登錄了，無需輸入用戶名密碼
#用戶名:egonlin 郵箱378533872@qq.com 密碼lhf@123
import requests

Cookies={  　　'user_session':'wGMHFJKgDcmRIVvcA14_Wrt_3xaUyJNsBnPbYzEL6L0bHcfc',
}

response=requests.get(　　　　　　　　'https://github.com/settings/emails',
       cookies=Cookies) #github對請求頭沒有什么限制，我們無需定制user-agent，對于其他網(wǎng)站可能還需要定制


print('378533872@qq.com' in response.text) #True

import requests
 
session = requests.session()

session.get(　　　　　　　　　 'https://passport.lagou.com/grantServiceTicket/grant.html',
         headers={
           'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36',
           'Referer': 'https://passport.lagou.com/login/login.html',
         }
         )

'''
一 目標站點分析
  瀏覽器輸入https://github.com/login
  然后輸入錯誤的賬號密碼，抓包
  發(fā)現(xiàn)登錄行為是post提交到：https://github.com/session
  而且請求頭包含cookie
  而且請求體包含：
    commit:Sign in
    utf8:✓
    authenticity_token:lbI8IJCwGslZS8qJPnof5e7ZkCoSoMn6jmDTsL1r/m06NLyIbw7vCrpwrFAPzHMep3Tmf/TSJVoXWrvDZaVwxQ==
    login:egonlin
    password:123


二 流程分析
  先GET：https://github.com/login拿到初始cookie與authenticity_token
  返回POST：https://github.com/session， 帶上初始cookie，帶上請求體（authenticity_token，用戶名，密碼等）
  最后拿到登錄cookie

  ps：如果密碼時密文形式，則可以先輸錯賬號，輸對密碼，然后到瀏覽器中拿到加密后的密碼，github的密碼是明文
'''

import requests
import re

#第一次請求
r1=requests.get('https://github.com/login')
r1_cookie=r1.cookies.get_dict() #拿到初始cookie(未被授權(quán))
authenticity_token=re.findall(r'name="authenticity_token".*?value="(.*?)"',r1.text)[0] #從頁面中拿到CSRF TOKEN

#第二次請求：帶著初始cookie和TOKEN發(fā)送POST請求給登錄頁面，帶上賬號密碼
data={
  'commit':'Sign in',
  'utf8':'✓',
  'authenticity_token':authenticity_token,
  'login':'317828332@qq.com',
  'password':'alex3714'
}
r2=requests.post('https://github.com/session',
       data=data,
       cookies=r1_cookie
       )


login_cookie=r2.cookies.get_dict()


#第三次請求：以后的登錄，拿著login_cookie就可以,比如訪問一些個人配置
r3=requests.get('https://github.com/settings/emails',
        cookies=login_cookie)

print('317828332@qq.com' in r3.text) #True

自動登錄github（自己處理cookie信息）

手動處理cookie后實現(xiàn)自動登錄github

import requests
import re

session=requests.session()
#第一次請求
r1=session.get('https://github.com/login')
authenticity_token=re.findall(r'name="authenticity_token".*?value="(.*?)"',r1.text)[0] #從頁面中拿到CSRF TOKEN

#第二次請求
data={
  'commit':'Sign in',
  'utf8':'✓',
  'authenticity_token':authenticity_token,
  'login':'317828332@qq.com',
  'password':'alex3714'
}
r2=session.post('https://github.com/session',
       data=data,
       )

#第三次請求
r3=session.get('https://github.com/settings/emails')

print('317828332@qq.com' in r3.text) #True
使用requests.session()自動幫我們處理cookie信息后自動登錄GitHub

requests.post(url='xxxxxxxx',
       data={'xxx':'yyy'}) #沒有指定請求頭,#默認的請求頭:application/x-www-form-urlencoed

#如果我們自定義請求頭是application/json,并且用data傳值, 則服務(wù)端取不到值
requests.post(url='',
       data={'':1,},
       headers={
         'content-type':'application/json'
       })


requests.post(url='',
       json={'':1,},
       ) #默認的請求頭:application/json

import requests
respone=requests.get('http://www.jianshu.com')
# respone屬性
print(respone.text)  #包含html內(nèi)容的字符串
print(respone.content)#bytes類型的字符串

print(respone.status_code)#狀態(tài)碼200
print(respone.headers)  #請求頭信息
print(respone.cookies)  #COOKIE對象
print(respone.cookies.get_dict())#封裝了cookie參數(shù)的字典{'locale': 'zh-CN'}

print(respone.cookies.items())#封裝了cookie參數(shù)的列表[('locale', 'zh-CN')]

print(respone.url)#url地址https://www.jianshu.com/
print(respone.history)

print(respone.encoding)#編碼utf-8

#關(guān)閉：response.close()
from contextlib import closing
with closing(requests.get('xxx',stream=True)) as response:
  for line in response.iter_content():
  pass#將內(nèi)容一行一行寫到文件中

#編碼問題
import requests
response=requests.get('http://www.autohome.com/news')
# response.encoding='gbk' #汽車之家網(wǎng)站返回的頁面內(nèi)容為gb2312編碼的，而requests的默認編碼為ISO-8859-1，如果不設(shè)置成gbk則中文亂碼
print(response.text)

import requests

response=requests.get('https://timgsa.baidu.com/timg?image&quality=80&size=b9999_10000&sec=1509868306530&di=712e4ef3ab258b36e9f4b48e85a81c9d&imgtype=0&src=http%3A%2F%2Fc.hiphotos.baidu.com%2Fimage%2Fpic%2Fitem%2F11385343fbf2b211e1fb58a1c08065380dd78e0c.jpg')

with open('a.jpg','wb') as f:
  f.write(response.content)

#stream參數(shù):一點一點的取,比如下載視頻時,如果視頻100G,用response.content然后一下子寫到文件中是不合理的

import requests

response=requests.get('https://gss3.baidu.com/6LZ0ej3k1Qd3ote6lo7D0j9wehsv/tieba-smallvideo-transcode/1767502_56ec685f9c7ec542eeaf6eac93a65dc7_6fe25cd1347c_3.mp4',
           stream=True)

with open('b.mp4','wb') as f:
  for line in response.iter_content():
    f.write(line)

#解析json
import requests
response=requests.get('http://httpbin.org/get')

import json
res1=json.loads(response.text) #太麻煩

res2=response.json() #直接獲取json數(shù)據(jù)

print(res1 == res2) #True

By default Requests will perform location redirection for all verbs except HEAD.

We can use the history property of the Response object to track redirection.

The Response.history list contains the Response objects that were created in order to complete the request. The list is sorted from the oldest to the most recent response.

For example, GitHub redirects all HTTP requests to HTTPS:

>>> r = requests.get('http://github.com')

>>> r.url
'https://github.com/'

>>> r.status_code

>>> r.history
[<Response [301]>]
If you're using GET, OPTIONS, POST, PUT, PATCH or DELETE, you can disable redirection handling with the allow_redirects parameter:

>>> r = requests.get('http://github.com', allow_redirects=False)

>>> r.status_code

>>> r.history
[]
If you're using HEAD, you can enable redirection as well:

>>> r = requests.head('http://github.com', allow_redirects=True)

>>> r.url
'https://github.com/'

>>> r.history
[<Response [301]>]

先看官網(wǎng)的解釋

官網(wǎng)解釋（強行裝作看得懂的樣子）

import requests
import re

#第一次請求
r1=requests.get('https://github.com/login')
r1_cookie=r1.cookies.get_dict() #拿到初始cookie(未被授權(quán))
authenticity_token=re.findall(r'name="authenticity_token".*?value="(.*?)"',r1.text)[0] #從頁面中拿到CSRF TOKEN

#第二次請求：帶著初始cookie和TOKEN發(fā)送POST請求給登錄頁面，帶上賬號密碼
data={
  'commit':'Sign in',
  'utf8':'✓',
  'authenticity_token':authenticity_token,
  'login':'317828332@qq.com',
  'password':'alex3714'
}






#測試一：沒有指定allow_redirects=False,則響應(yīng)頭中出現(xiàn)Location就跳轉(zhuǎn)到新頁面，r2代表新頁面的response
r2=requests.post('https://github.com/session',
       data=data,
       cookies=r1_cookie
       )

print(r2.status_code) #200
print(r2.url) #看到的是跳轉(zhuǎn)后的頁面
print(r2.history) #看到的是跳轉(zhuǎn)前的response
print(r2.history[0].text) #看到的是跳轉(zhuǎn)前的response.text


#測試二：指定allow_redirects=False,則響應(yīng)頭中即便出現(xiàn)Location也不會跳轉(zhuǎn)到新頁面，r2代表的仍然是老頁面的response
r2=requests.post('https://github.com/session',
       data=data,
       cookies=r1_cookie,
       allow_redirects=False
       )


print(r2.status_code) #302
print(r2.url) #看到的是跳轉(zhuǎn)前的頁面https://github.com/session
print(r2.history) #[]

利用github登錄后跳轉(zhuǎn)到主頁面的例子來驗證它

#證書驗證(大部分網(wǎng)站都是https)
import requests
respone=requests.get('https://www.12306.cn') #如果是ssl請求,首先檢查證書是否合法,不合法則報錯,不允許訪問


#改進1:去掉報錯,能訪問但是會報警告，因為畢竟我們沒有證書
import requests
respone=requests.get('https://www.12306.cn',verify=False) #verify參數(shù)改成false就不驗證證書,報警告,能訪問，返回200
print(respone.status_code)#200


#改進2:去掉報錯,并且去掉警報信息，與上步?jīng)]啥太大區(qū)別
import requests
from requests.packages import urllib3
urllib3.disable_warnings() #關(guān)閉警告
respone=requests.get('https://www.12306.cn',verify=False)
print(respone.status_code)

#改進3:加上證書
#很多網(wǎng)站都是https,但是不用證書也可以訪問,大多數(shù)情況都是可以攜帶也可以不攜帶證書
#知乎\百度等都是可帶可不帶
#有硬性要求的,則必須帶，比如對于定向的用戶,拿到證書后才有權(quán)限訪問某個類似12306的特定網(wǎng)站
import requests
respone=requests.get('https://www.12306.cn',
           cert=('/path/server.crt',
              '/path/key'))#證書存放在本地
print(respone.status_code)

#官網(wǎng)鏈接: http://docs.python-requests.org/en/master/user/advanced/#proxies

#代理設(shè)置:先發(fā)送請求給代理,然后由代理幫忙發(fā)送
import requests
#代理的參數(shù)
proxies={
  'http':'http://egon:123@localhost:9743',#帶用戶名密碼的代理,@符號前是用戶名與密碼，@符號后是IP和端口
  'http':'http://localhost:9743',#不帶用戶名密碼的代理
  'https':'https://localhost:9743',
}
respone=requests.get('https://www.12306.cn',
           proxies=proxies)

print(respone.status_code)

#支持socks代理,安裝:pip install requests[socks]
import requests

#用socks代替了http和https，無需特意做區(qū)分
proxies = {
  'http': 'socks5://user:pass@host:port',
  'https': 'socks5://user:pass@host:port'
}
respone=requests.get('https://www.12306.cn',
           proxies=proxies)

print(respone.status_code)

#兩種超時設(shè)置:float or tuple
#timeout=0.1 #代表接收數(shù)據(jù)的超時時間
#timeout=(0.1,0.2)#0.1代表鏈接的超時時間 0.2代表接收數(shù)據(jù)的超時時間

import requests
respone=requests.get('https://www.baidu.com',
           timeout=0.01)

r.headers['Authorization'] = _basic_auth_str(self.username, self.password)

# 一般的網(wǎng)站都不用默認的加密方式，都是自己寫
# 那么我們就需要按照網(wǎng)站的加密方式，自己寫一個類似于_basic_auth_str的方法得到加密字符串后添加到請求頭
#     r.headers['Authorization'] =func('.....')

#默認的加密方式（不過通常網(wǎng)站都不會用默認的加密設(shè)置）
import requests
from requests.auth import HTTPBasicAuth
r=requests.get('xxx',auth=HTTPBasicAuth('user','password'))
print(r.status_code)

#HTTPBasicAuth可以簡寫為auth
import requests
r=requests.get('xxx',auth=('user','password'))
print(r.status_code)

#異常處理
import requests
from requests.exceptions import * #可以查看requests.exceptions獲取異常類型

try:
  r=requests.get('http://www.baidu.com',timeout=0.00001)
except ReadTimeout:
  print('===:')
except ConnectionError: #網(wǎng)絡(luò)不通
  print('-----')
except Timeout:
  print('aaaaa')

except RequestException:
  print('Error')

import requests
files={'file':open('a.jpg','rb')}
respone=requests.post('http://httpbin.org/post',files=files)
print(respone.status_code)

import requests
from urllib3 import encode_multipart_formdata

path = 'test1.png'

# 上傳圖片
url = 'xxxxxx'
data = {'upload':(path, open(path, 'rb').read())}
header = {}
encode_data = encode_multipart_formdata(data)
file_data = encode_data[0]
header['Content-Type'] = encode_data[1]
r = requests.post(url, headers=header, data=file_data)

import requests
import re

session = requests.session()

# 第一步：訪問登陸頁,拿到X_Anti_Forge_Token，X_Anti_Forge_Code
# 1、請求url:https://passport.lagou.com/login/login.html
# 2、請求方法:GET
# 3、請求頭:
#  User-agent
r1 = session.get('https://passport.lagou.com/login/login.html',
         headers={
           'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36',
         },
         )

X_Anti_Forge_Token = re.findall("X_Anti_Forge_Token = '(.*?)'", r1.text, re.S)[0]
X_Anti_Forge_Code = re.findall("X_Anti_Forge_Code = '(.*?)'", r1.text, re.S)[0]
# print(X_Anti_Forge_Token,X_Anti_Forge_Code)

# 第二步：登陸
# 1、請求url:https://passport.lagou.com/login/login.json
# 2、請求方法:POST
# 3、請求頭:
#  cookie
#  User-agent
#  Referer:https://passport.lagou.com/login/login.html
#  X-Anit-Forge-Code:53165984
#  X-Anit-Forge-Token:3b6a2f62-80f0-428b-8efb-ef72fc100d78
#  X-Requested-With:XMLHttpRequest
# 4、請求體：
# isValidate:true
# username:18611453110
# password:70621c64832c4d4d66a47be6150b4a8e
# request_form_verifyCode:''
# submit:''
r2 = session.post('https://passport.lagou.com/login/login.json',
         headers={
           'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36',
           'Referer': 'https://passport.lagou.com/login/login.html',
           'X-Anit-Forge-Code': X_Anti_Forge_Code,
           'X-Anit-Forge-Token': X_Anti_Forge_Token,
           'X-Requested-With': 'XMLHttpRequest'
         },
         data={
           "isValidate": True,
           'username': '18611453110',
           'password': '70621c64832c4d4d66a47be6150b4a8e',
           'request_form_verifyCode': '',
           'submit': ''
         }
         )
# 第三步：授權(quán)
# 1、請求url:https://passport.lagou.com/grantServiceTicket/grant.html
# 2、請求方法:GET
# 3、請求頭:
#  User-agent
#  Referer:https://passport.lagou.com/login/login.html

r3 = session.get('https://passport.lagou.com/grantServiceTicket/grant.html',
         headers={
           'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36',
           'Referer': 'https://passport.lagou.com/login/login.html',
         }
         )

# 第四步：驗證
r4 = session.get('https://www.lagou.com/resume/myresume.html',
         headers={
           'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36',
         }
         )

# print('18611453110' in r4.text)


# 第五步：篩選職位信息
# 請求url：https://www.lagou.com/jobs/list_java%E9%AB%98%E7%BA%A7%E5%BC%80%E5%8F%91
# 請求方法：GET
# 請求頭：
# User-Agent
# 請求參數(shù)：
# gj:3年及以下
# px:default
# yx:25k-50k
# city:北京
from urllib.parse import urlencode

res = urlencode({'k': 'java高級開發(fā)'}, encoding='utf-8').split('=')[-1]
url = 'https://www.lagou.com/jobs/list_' + res
#
# r5 = session.get(url,
#         headers={
#           'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36',
#         },
#         params={
#           'gj': '3年及以下',
#           'px': 'default',
#           'yx': '25k-50k',
#           'city': '北京'
#         }
#         )
#
# print(r5.text)

#請求url：https://www.lagou.com/jobs/positionAjax.json
#請求方法：POST
#請求頭
#  Referer
#  User-Agent
#請求體：
  # first:true
  # pn:1
  # kd:java高級開發(fā)
#請求參數(shù)
# params={
#   'gj': '3年及以下',
#   'px': 'default',
#   'yx': '25k-50k',
#   'city': '北京',
#   'needAddtionalResult':False,
#   'isSchoolJob':0
# }
r6=session.post('https://www.lagou.com/jobs/positionAjax.json',
       headers={
          'Referer':url,
           'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36',

       },
       data={
         'first':True,
         'pn':2,
         'kd':'java高級開發(fā)'
       },
       params={
         'gj': '3年及以下',
         'px': 'default',
         'yx': '25k-50k',
         'city': '北京',
         'needAddtionalResult': False,
         'isSchoolJob': 0
       }
       )


from pprint import pprint
# print(r6.json())
comapines_list=r6.json()['content']['positionResult']['result']
for comapiny in comapines_list:
  positionId=comapiny['positionId']
  company_link='https://www.lagou.com/jobs/{pos_id}.html'.format(pos_id=positionId)
  companyShortName = comapiny['companyShortName']
  positionName = comapiny['positionName']
  salary = comapiny['salary']
  print('''
  詳情連接:%s
  公司名：%s
  職位名：%s
  薪資：%s
  ''' %(company_link,companyShortName,positionName,salary))


  #第七步：訪問詳情頁，拿到X_Anti_Forge_Token，X_Anti_Forge_Code
  # 請求url：詳情頁地址
  # 請求方式：GET
  # 請求頭：User-Agent
  r7=session.get(company_link,
        headers={
          'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36',
        }
        )
  X_Anti_Forge_Token = re.findall("X_Anti_Forge_Token = '(.*?)'", r7.text, re.S)[0]
  X_Anti_Forge_Code = re.findall("X_Anti_Forge_Code = '(.*?)'", r7.text, re.S)[0]
  # print(X_Anti_Forge_Token,X_Anti_Forge_Code)


  #第八步：投遞簡歷
  #請求url：https://www.lagou.com/mycenterDelay/deliverResumeBeforce.json
  #請求方式：POST
  #請求頭：
    #Referer:詳情頁地址
    #User-agent
    #X-Anit-Forge-Code:53165984
    #X-Anit-Forge-Token:3b6a2f62-80f0-428b-8efb-ef72fc100d78
    #X-Requested-With:XMLHttpRequest
  #請求體：
  # positionId:職位ID
  # type:1
  # force:true

  session.post('https://www.lagou.com/mycenterDelay/deliverResumeBeforce.json',
         headers={
           'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36',
           'Referer': company_link,
           'X-Anit-Forge-Code': X_Anti_Forge_Code,
           'X-Anit-Forge-Token': X_Anti_Forge_Token,
           'X-Requested-With': 'XMLHttpRequest'
         },
         data={
  'positionId':positionId,
  'type':1,
  'force':True
         }
         )
  print('%s 投遞成功' %(companyShortName))

import re
import requests

session=requests.session()#可以幫助我們自動處理cookie信息，我們不用再手動穿cookie的值


# 第一步：訪問登陸頁,這里的登錄頁就是首頁
# 1、請求url:https://www.liepin.com/
# 2、請求方法:GET
# 3、請求頭:
#  User-agent
r1_url='https://www.liepin.com/'
user_agent='Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.108 Safari/537.36'



# 第二步：登陸
# 1、請求url:https://passport.liepin.com/c/login.json?__mn__=user_login
# 2、請求方法:POST
# 3、請求頭:
#  cookie
#  User-agent
#  Referer:https://passport.liepin.com/ajaxproxy.html
#  X-Alt-Referer:https://www.liepin.com/
#  X-Requested-With:XMLHttpRequest
# 4、請求體：
# user_pwd:5d353bc2f5474201c8a2f1891735999a
# version:
# user_login:15011316546
# chk_remember_pwd:on
r2_url='https://passport.liepin.com/c/login.json?__mn__=user_login'
r2_header_dict={
  'User-agent':user_agent,
  'Referer':'https://passport.liepin.com/ajaxproxy.html',
  'X-Alt-Referer':'https://www.liepin.com/',
  'X-Requested-With':'XMLHttpRequest'
}
r2_form_data={
  'user_pwd':'5d353bc2f5474201c8a2f1891735999a',
  'user_login':'15011316546',
  'chk_remember_pwd':'on'
}




# 第三步：篩選職位信息
# 請求url：https://www.liepin.com/zhaopin/
# 請求方法：GET
# 請求頭：
# User-Agent
# 請求參數(shù)：
# salary:15$20
# dqs:010
# key:前臺
r3_url='https://www.liepin.com/zhaopin/'
r3_header_dict={
  'User-Agent':user_agent
}
# r3_params={
#   'salary':'10$20',
#   'dqs':'010',
#   'key':'前臺'
# }
r3_params={
  'salary':'10$20',
  'dqs':'070020',
  'key':'Python開發(fā)'
}



#第四步 正則匹配 拿到頁面中所有的職位數(shù)據(jù)，寫入文件中
re_rule = 'class="icon icon-yellow-triangle".*?class="job-info".*?title="(.*?)".*?href="(.*?)" rel="external nofollow" .*?title="(.*?)".*?class="company-name".*?title="(.*?)".*?target="_blank">(.*?)</a>'
txt = '獵聘網(wǎng)招聘（Python1）.txt'

#封裝，便于傳參
dict={
  'r1_url':r1_url,
  'user_agent':user_agent,
  'r2_url':r2_url,
  'r2_header_dict':r2_header_dict,
  'r2_form_data':r2_form_data,
  'r3_url':r3_url,
  'r3_header_dict':r3_header_dict,
  'r3_params':r3_params,
  're_rule':re_rule,
  'txt':txt
}

def spider(**kwargs):
  #第一步
  r1=session.get(r1_url,
          headers={
            'Referer':'https://www.liepin.com/',
            'User-Agent':user_agent
          }
          )
  #第二步
  r2=session.post(r2_url,
          headers=r2_header_dict,
          data=r2_form_data
  )
  # 第三步3
  r3=session.get(
    r3_url,
    headers=r3_header_dict,
    params=r3_params
  )
  print('國內(nèi)某知名投行' in r3.text)


  #第四步
  result=re.findall(re_rule,r3.text,re.S)
  for i in result:
    a="""
    招聘職位：%s
    招聘鏈接：%s
    薪資及要求：%s
    公司名稱：%s
    公司類型：%s
     
    """%i
    with open(txt, 'a',encoding='UTF-8') as f:
      f.write(a)




if __name__ == '__main__':
  spider(**dict)