master
pan 4 years ago
parent ebe2382d4a
commit ae0f735438
  1. 17
      main.py
  2. 8
      run.py

@ -40,6 +40,7 @@ class MysqlDB:
finally:
connection.close()
# 美团美食爬虫
class MeiTuanCrawler:
# 城市列表网址
@ -71,9 +72,14 @@ class MeiTuanCrawler:
else:
writeError("无法爬取城市链接,响应码:{0}".format(res.status_code))
# 爬取城市美食
def meishi(self):
for city in self.cityMap.keys():
def meishiInputCity(self):
city = input("请输入要爬取的城市:\n")
while city not in self.cityMap:
writeInfo("城市:{}不合法".format(city))
city = input("请输入要爬取的城市:\n")
self.meishiWithCity(city)
def meishiWithCity(self, city):
# 获取请求参数
c = CityParam(self.cityMap[city])
GET_PARAM = c.getParam()
@ -121,3 +127,8 @@ class MeiTuanCrawler:
GET_PARAM['page'] = GET_PARAM['page'] + 1
# 限制请求速率
time.sleep(1)
# 爬取城市美食
def meishi(self):
for city in self.cityMap.keys():
self.meishiWithCity(city)

@ -1,5 +1,11 @@
from config.log import writeInfo, writeError
from main import MeiTuanCrawler, MysqlDB
c = MeiTuanCrawler()
c.getCity()
c.meishi()
option=input('选择爬虫选项:\n1.爬取所有城市\n2.爬取指定城市\n')
methods={'1':c.meishi,'2':c.meishiInputCity}
while option not in methods:
writeError('选项{}不合法!!!'.format(option))
option = input('选择爬虫选项:\n1.爬取所有城市\n2.爬取指定城市\n')
methods[option]()

Loading…
Cancel
Save