python爬虫——站酷数据获取
1.站酷数据获取
其中需要注意，本文使用了IP代理，以及不同的作品类型，他们详情页url拼接方式不同
import random import time import datetime import requests import threading from lxml import etree import pymysql class ZhankunSpider(object): def __init__(self): self.url = 'https://www.zcool.com.cn/p1/discover/first?p={}&ps=20' self.mysql = pymysql.connect(host='localhost', database='tenders', port=3306, user='root', password='123456') self.cur = self.mysql.cursor() self.blog = 1 def proxy_get(self): procy = requests.get( r'放入IP代理链接').json()['data'][0] proxy = str(procy["ip"]) + ':' + str(procy["port"]) http = 'http://' + proxy https = 'https://' + proxy self.proxys = {'http': http, 'https': https} print(self.proxys) # result = requests.get('https://www.baidu.com/',verify=False) result = requests.get('https://www.baidu.com/') print(result.status_code) if result.status_code != 200: self.proxy_get() time.sleep(0.2) return # self.expire_datetime = datetime.datetime.now() + datetime.timedelta(seconds=60) def _check_expire(self): self.expire_datetime = datetime.datetime.now() + datetime.timedelta(seconds=60) if datetime.datetime.now() >= self.expire_datetime: self.proxy_get() # 发送请求 def get_html(self, url): if self.blog <= 3: try: datas = { 'p': 'i', 'column': 5 } headers = {'Cookie': '登陆后cookie', 'User-Agent':'',} json_ids = requests.get(url=url, headers=headers, data=datas).json() return json_ids except Exception as e: print(e) self.blog += 1 self.get_html(url) # 解析提取数据 def parse_html(self, url): json_ids = self.get_html(url) self._check_expire() if json_ids: time.sleep(1) for dic in json_ids['datas']: titles = dic['content']['title'] #题目 types = dic['content']['typeStr'] viewCountStrs = dic['content']['viewCountStr'] #浏览量 subCateStrs = dic['content']['subCateStr'] cateStrs = dic['content']['cateStr'] url13 = 'https://www.zcool.com.cn/p1/product/'+dic['content']['idStr'] urll = dic['content']['pageUrl'] headers1 = { 'Cookie': '', 'User-Agent': '', } # self._check_expire() if 'work' in urll: url2 = 'https://www.zcool.com.cn/p1/product/' + dic['content']['idStr'] try: json_idss = requests.get(url=url2, headers=headers1, proxies=self.proxys, timeout=3).json() except: self.proxy_get() json_idss = requests.get(url=url2, headers=headers1, proxies=self.proxys, timeout=3).json() time.sleep(1) for dici in json_idss['data']['productImages']: datass = dici['url'] else: url2 = 'https://www.zcool.com.cn/p1/article/' + dic['content']['idStr'] try: json_idss = requests.get(url=url2, headers=headers1, proxies=self.proxys, timeout=3).json() except: self.proxy_get() json_idss = requests.get(url=url2, headers=headers1, proxies=self.proxys, timeout=3).json() time.sleep(1) # datass = json_idss['data']['id'] for dici in json_idss['data']['creatorObj']['contentCards']: datass = dici['cover1x'] timeStamp = dic['content']['timeTitleStr'] # timeArray = time.localtime(timeStamp) # 转化成对应的时间 # otherStyleTime = time.strftime("%Y-%m-%d %H:%M:%S", timeArray) # 字符串 # date = otherStyleTime photo = dic['content']['cover1x'] data = { 'title': titles, 'urls': url13, 'address': timeStamp, 'configuration': types, 'grade': viewCountStrs, 'collections': subCateStrs, 'price': cateStrs, 'unit': photo, 'photoadress': datass } print(data) self.save_mysql(data) def save_mysql(self, data): # str_sql = "insert into ftx values(0, '{}', '{}');".format(data['first_category'],data['second_category']) str_sql = "insert into meituan values(0, '{}', '{}', '{}', '{}', '{}', '{}', '{}', '{}', '{}');".format( data['title'], data['urls'],data['address'], data['configuration'], data['grade'], data['collections'], data['price'], data['unit'], data['photoadress']) self.cur.execute(str_sql) self.mysql.commit() def __del__(self): self.cur.close() self.mysql.close() # 入口函数 def run(self): try: for i in range(1,5): url = self.url.format(i) print(i) # self.get_html(url) self.parse_html(url) time.sleep(random.randint(2, 4)) # 每次抓取一页要初始化一次self.blog self.blog = 1 except Exception as e: print('发生错误', e) if __name__ == '__main__': spider = ZhankunSpider() spider.run()
2.结果展示
你可能想看：
最新文章
-
王长绅紫微斗数之廉贞天相在子午守财帛宫，命宫为武曲星
19分钟前 -
磁共振扫描操作磁共振冠脉成像（一）
33分钟前 -
“病急乱投医”
50分钟前 -
择日——吊用“太阳、太阴”吉神到宫的方法
1小时前 -
古诗词中的十大人生境界，古今一付笑谈中
1小时前 -
史上最全K线图大全：70种+42个K线组合图解
2小时前 -
明朝最大遗憾：崇祯若答应李自成条件，明朝不会灭亡，还灭了满清
2小时前 -
孔子语录100句（附释文）国学经典
2小时前
热门文章
-
业医必修——孙思邈《大医精诚》
1周前 -
怎样从门口看房屋吉凶
1周前 -
低估了广东的“甘蔗马蹄茅根水”，喝完半个小时就舒服了！
1周前 -
痰湿体质如何调理 最全痰湿体质调理方法(8)
1周前 -
自制家庭小咸菜
1周前 -
七彩神龙指标源码（八戒神龙主图无未来函数）
7天前 -
《奇门遁甲预测术》奇门：格局庚
6天前 -
10种朝鲜咸菜的腌制方法
7天前
有话要说...