master
pan 5 years ago
parent ebe2382d4a
commit ae0f735438
  1. 17
      main.py
  2. 8
      run.py

@ -40,6 +40,7 @@ class MysqlDB:
finally: finally:
connection.close() connection.close()
# 美团美食爬虫 # 美团美食爬虫
class MeiTuanCrawler: class MeiTuanCrawler:
# 城市列表网址 # 城市列表网址
@ -71,9 +72,14 @@ class MeiTuanCrawler:
else: else:
writeError("无法爬取城市链接,响应码:{0}".format(res.status_code)) writeError("无法爬取城市链接,响应码:{0}".format(res.status_code))
# 爬取城市美食 def meishiInputCity(self):
def meishi(self): city = input("请输入要爬取的城市:\n")
for city in self.cityMap.keys(): while city not in self.cityMap:
writeInfo("城市:{}不合法".format(city))
city = input("请输入要爬取的城市:\n")
self.meishiWithCity(city)
def meishiWithCity(self, city):
# 获取请求参数 # 获取请求参数
c = CityParam(self.cityMap[city]) c = CityParam(self.cityMap[city])
GET_PARAM = c.getParam() GET_PARAM = c.getParam()
@ -121,3 +127,8 @@ class MeiTuanCrawler:
GET_PARAM['page'] = GET_PARAM['page'] + 1 GET_PARAM['page'] = GET_PARAM['page'] + 1
# 限制请求速率 # 限制请求速率
time.sleep(1) time.sleep(1)
# 爬取城市美食
def meishi(self):
for city in self.cityMap.keys():
self.meishiWithCity(city)

@ -1,5 +1,11 @@
from config.log import writeInfo, writeError
from main import MeiTuanCrawler, MysqlDB from main import MeiTuanCrawler, MysqlDB
c = MeiTuanCrawler() c = MeiTuanCrawler()
c.getCity() c.getCity()
c.meishi() option=input('选择爬虫选项:\n1.爬取所有城市\n2.爬取指定城市\n')
methods={'1':c.meishi,'2':c.meishiInputCity}
while option not in methods:
writeError('选项{}不合法!!!'.format(option))
option = input('选择爬虫选项:\n1.爬取所有城市\n2.爬取指定城市\n')
methods[option]()

Loading…
Cancel
Save