百度贴码爬取 

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

import urllib.request
import urllib.parse

url = 'http://www.kfc.com.cn/kfccda/ashx/GetStoreList.ashx?op=keyword'
word = input('请输入你要查的内容：')
page = input('shuzi')
pagesz= input('sz')
formodata = {
	'cname':'' ,
	'pid':'' ,
	'keyword': word,
	'pageIndex': page,
	'pageSize': pagesz,
}

headers = {
	'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 Edg/116.0.1938.81',
}

formodata = urllib.parse.urlencode(formodata).encode()

request = urllib.request.Request(url=url,headers=headers)

response = urllib.request.urlopen(request,data=formodata)

print(response.read().decode())

进阶版本，生成网页保存到本地

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
import urllib.request
import urllib.parse
import os
url = 'https://tieba.baidu.com/f?ie=utf-8&'

bmname = input('请输入吧名：')
stnpage = int(input('请输入起始页码：'))
enpate = int(input('请输入结束页码：'))

if not os.path.exists(bmname):
	os.mkdir(bmname)
for page in range(stnpage,enpate+1):
	data={
	'kw':bmname,
	'pn':(page - 1)*50,
	}
	print('第%s页开始下载……'%page)
	data= urllib.parse.urlencode(data)
	urlpg = url+data
	# print(urlpg)
	headers = {
	'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 Edg/116.0.1938.81',
	}
	request = urllib.request.Request(url=urlpg,headers=headers)
	response = urllib.request.urlopen(request)
	filename = bmname + '_'+ str(page) + '.html'
	filepath = bmname + '/' + filename

	with open(filepath,'wb') as pf:
		pf.write(response.read())
	print('第%s页下载结束……'%page)

展开全文 >>

ajaxget请求测试 

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
import urllib.request
import urllib.parse

url = 'https://movie.douban.com/j/chart/top_list?type=5&interval_id=100%3A90&action=&'
headers = {
	'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 Edg/116.0.1938.81',
}

page = int(input('请输入覃查询的页数：'))

number = 20


data = {
	'start':(page - 1)*number,
	'limit':number,
}


url += urllib.parse.urlencode(data)

# url += query_str

request = urllib.request.Request(url,headers=headers)

response = urllib.request.urlopen(request)

print(response.read().decode())

展开全文 >>

postfanyi 

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
import urllib.request
import urllib.parse
word = 'wolf'
post_url = 'https://fanyi.baidu.com/v2transapi?from=en&to=zh'
hearders = {
	'Host': 'fanyi.baidu.com',
	'Connection': 'keep-alive',
	# 'Content-Length': '151',
	'sec-ch-ua': '"Chromium";v="116", "Not)A;Brand";v="24", "Microsoft Edge";v="116"',
	'sec-ch-ua-mobile': '?0',
	'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 Edg/116.0.1938.76',
	'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
	'Accept':'*/*',
	'Acs-Token': '1694697149072_1694697161652_grIp1xN+4+O/Mqn+6QlRIgL1Te8vSgV6C8hOOM7Mh0OgGOQH0s1M2znTlV3aMw7uHtlwq9VAfe/jAJeJyYiQko2u2IwKJ0m2pXpWVwUiwy8mGxihQqbRoSihR6C92JJoimguIQ9FIlV0FUSg3vXoD6434WnzN4kj3ewmDNk4Jcd9BZ8WQUjsOcvldlm8yvIQXessa0w/X6QAzQ813jzKaU7u2Ri8j8MLG2NYGGpJqUxSOd9xt/GiVsxfTVNj1u0K/EZatLKTqt+pjXR726eqRj7x6LuY56KI6JUS1ce9vng0OeFAU1vYxE7GoDTkoCnJxII4lDW+oZvWFau8JtUJeEYQ/kP4N3XgtjWcfeHdlY4gu6a5aFBga2AH1MhtcvPVnlckloNELES72ewsXORWENaoFLY2fwMwn9g6ii5oMgw5oFaI3GM0eFanwaimPrSkmW9oDxxGlZyTx3ul4kZRujCmbsTgYeUn5LC6ZORbDzDQnP9QKZmnvveKJgTTwI5kAbm1IYSCPhp+/qVsoDxDDmTOcmpNwEH6AiWlRSjPw3U=',
	'X-Requested-With': 'XMLHttpRequest',
	'sec-ch-ua-platform': '"Windows"',
	'Origin': 'https://fanyi.baidu.com',
	'Sec-Fetch-Site': 'same-origin',
	'Sec-Fetch-Mode': 'cors',
	'Sec-Fetch-Dest': 'empty',
	'Referer': 'https://fanyi.baidu.com/',
	# 'Accept-Encoding': 'gzip, deflate, br',
	'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
	'Cookie': 'BIDUPSID=5400077289767CBECB57585BA613A703; PSTM=1690891215; BAIDUID=5400077289767CBECB57585BA613A703:SL=0:NR=10:FG=1; REALTIME_TRANS_SWITCH=1; FANYI_WORD_SWITCH=1; HISTORY_SWITCH=1; SOUND_SPD_SWITCH=1; SOUND_PREFER_SWITCH=1; MCITY=-268%3A308%3A; BAIDUID_BFESS=5400077289767CBECB57585BA613A703:SL=0:NR=10:FG=1; ZFY=grTbJMiejhfvuT9DratFBB3dGiB:BbSrz5XFE3OniQew:C; BAIDU_WISE_UID=wapp_1694605716567_839; RT="z=1&dm=baidu.com&si=6d19c296-2398-4671-9638-f474519761d6&ss=lmhohlwc&sl=4&tt=3lf&bcn=https%3A%2F%2Ffclog.baidu.com%2Flog%2Fweirwood%3Ftype%3Dperf&ld=3xxf"; BA_HECTOR=05a08l0k210485a42l81ag0s1ig5ug01p; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; BDRCVFR[feWj1Vr5u3D]=I67x6TjHwwYf0; PSINO=1; delPer=0; H_PS_PSSID=39329_39362_39280_39349_39097_39198_39293_39261_39358_39377_39233_26350; Hm_lvt_64ecd82404c51e03dc91cb9e8c025574=1694093886,1694435195,1694694017; APPGUIDE_10_6_2=1; PPA_CI=afadc0f2cb4aff67ec9de4e1705c4b31; Hm_lpvt_64ecd82404c51e03dc91cb9e8c025574=1694697149; ab_sr=1.0.1_OTZkMGYyZWZmYjkzYjgyYWMzYWRmYWZkODJiYTQ5YWI3NjY4NDU5OGZkZmUyNzhlYzAxZmM4YjJkMTkxMjUzN2MzZDQzMmI3M2FmNDdkYTA1OWI4MmZmM2VjZDU3NmI2Y2ZmZDg3Zjk4YmYzODQ3MzIwMmQwZTUxMTJhMjRiOGJlMjZmNWZjNjZjNTgzNzcwNmNjMDMwZDZmOTkyYjNlMQ==',
}

form_data = {
'from':'en',
'to':'zh',
'query':'wolf',
'transtype':'realtime',
'simple_means_flag': '3',
'sign':'275695.55262',
'token':'0d0fc210f005dbe0c3c529c02b7d5447',
'domain' : 'common',
'ts' : '1694697161625',
}

request = urllib.request.Request(url=post_url,headers=hearders)

form_data = urllib.parse.urlencode(form_data).encode()

response = urllib.request.urlopen(request,form_data)

print(response.read().decode())

展开全文 >>

urlpost请求方式 

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
import urllib.request
import urllib.parse
word = input('请输入要翻译的单词:')
post_url = 'https://fanyi.baidu.com/sug'
form_data = {
	'kw':word
}

headers = {
	'User-Agen':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 Edg/116.0.1938.76'
}
# 注意form_data需要通过urllib.parse.urlencode转码转换成&拼接的方式，需要通过encode转换成字节形式
request = urllib.request.Request(url=post_url,headers=headers)
form_data = urllib.parse.urlencode(form_data).encode()
response = urllib.request.urlopen(request,data=form_data)
print(response.read().decode())

展开全文 >>

网页测试 

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
import urllib.request,urllib.parse

word = input('请输入关键词：')
url = 'http://www.baidu.com/s?'

data = {
	'ie':'utf-8',
	'wd':'word',
}

quest_str = urllib.parse.urlencode(data)

url += quest_str

response = urllib.request.urlopen(url)

filename = word + '.html'

print(filename)

with open(filename, 'wb') as fl:
	fl.write(response.read())

下载跟据输入的内容下载成网页测试

展开全文 >>

URLlibprase

urllib.parse
quote
url编码函数，将中文进行转化为%xxx
unquote url解码函数，将%xxx转化为指定字符
urlencode给一个字典，将字典拼接为query_.string

1
2
3
4
5
6
7
8
9
10
11
import urllib.parse


url = 'http://www.baidu.com/index.html?name=&pwd=123456'
# url编码
ret = urllib.parse.quote(url)

print(ret)
# url解码
re = urllib.parse.unquote(ret)
print(re)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
import urllib.parse
url = 'http://www.baidu.com/index.html'
name = 'goudan'
age = 18
sex = 'nv'
height = '180'

data = {
	'name':name,
	'age':age,
	'sex':sex,
	'height':height,
}

# 遍历字典

lt = []
for k,v in data.items():
	lt.append(k + '=' +str(v))
query_string ='&'.join(lt)

print(lt)
# url = url +'?'+query_string

# print(url)
1
2
3
4
5
6
7
query_string = urllib.parse.urlencode(data)

print(query_string)

url = url + '?' +query_string

print(url)

response
read()读取相应内容，内容是字节类型
geturl()获取请求的url
getheaders()获取头部信息，列表里面有元组
getcode()获取状态码
readlines()按行读取，返回列表，都是字节类型

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
import urllib.request

abc = "http://www.baidu.com"

reponse = urllib.request.urlopen(url=abc)
# print(reponse.read().decode())

# with open('baidu.html','w', encoding='utf8') as f:
# 	f.write(reponse.read().decode())
# print(reponse.geturl())
# print(dict(reponse.getheaders()))
# print(reponse.getcode())
# print(reponse.readlines())
# with open('baidu2.html', 'wb') as f:
# 	f.write(reponse.read())

img_url = 'https://img2.baidu.com/it/u=3354585195,1512541150&fm=253&app=138&size=w931&n=0&f=JPEG&fmt=auto?sec=1693674000&t=5b9d9a7c11c5627c7b7bcb28b56ce8f2'

reponse = urllib.request.urlopen(url=img_url)
# with open('qing.jpg', 'wb') as f:
# 	f.write(reponse.read())
urllib.request.urlretrieve(img_url, 'chun.jpg')

展开全文 >>

tag:

    缺失模块。
    1、请确保node版本大于6.2
    2、在博客根目录（注意不是yilia根目录）执行以下命令：
    npm i hexo-generator-json-content --save

    3、在根目录_config.yml里添加配置： 

      jsonContent:
    meta: false
    pages: false
    posts:
      title: true
      date: true
      path: true
      text: false
      raw: false
      content: false
      slug: false
      updated: false
      comments: false
      link: false
      permalink: false
      excerpt: false
      categories: false
      tags: true

很惭愧<br><br>只做了一点微小的工作<br>谢谢大家