From bad777ad6c320e8a98e85fd7db77eba0ea49862a Mon Sep 17 00:00:00 2001 From: 10295 <1029559041@qq.com> Date: Thu, 22 Mar 2018 03:40:37 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8F=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- PixivSearch/dao/bangumi.py | 61 +++++++++ PixivSearch/model/config.py | 44 +++++++ PixivSearch/module/__init__.py | 0 PixivSearch/module/config.py | 5 - PixivSearch/pixiv.py | 4 +- PixivSearch/settings.py | 9 +- PixivSearch/templates/addConfig.html | 27 ++++ .../templates/{test.html => index.html} | 124 +++++++++--------- PixivSearch/urls.py | 7 +- PixivSearch/view.py | 48 +++++-- PixivSearch/wsgi.py | 2 +- 11 files changed, 248 insertions(+), 83 deletions(-) create mode 100644 PixivSearch/dao/bangumi.py create mode 100644 PixivSearch/model/config.py delete mode 100644 PixivSearch/module/__init__.py delete mode 100644 PixivSearch/module/config.py create mode 100644 PixivSearch/templates/addConfig.html rename PixivSearch/templates/{test.html => index.html} (96%) diff --git a/PixivSearch/dao/bangumi.py b/PixivSearch/dao/bangumi.py new file mode 100644 index 0000000..471f1c1 --- /dev/null +++ b/PixivSearch/dao/bangumi.py @@ -0,0 +1,61 @@ +import json +import os +import time + +import django +import requests +import threading +from bs4 import BeautifulSoup + +from PixivSearch.settings import logger + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "PixivSearch.settings") +django.setup() +from PixivSearch.model.config import mediaInfo, stat + +current_mediaInfo = mediaInfo(id=0, chn_name='null') +flag = True + + +class bangumi(threading.Thread): + begin = 0 + end = 0 + id = 0 + + def __init__(self, begin, end): + threading.Thread.__init__(self) + self.begin = begin + self.end = end + + def save(self): + req = requests.get("https://www.bilibili.com/bangumi/media/md%d" % self.id) + if (req.status_code == 200): + tag = BeautifulSoup(req.text, 'lxml') + script = tag.select("script")[3].text + json_str = script[script.index("=") + 1:script.index("function") - 2] + json_obj = json.loads(json_str) + stat_info = json_obj['mediaInfo']['stat'] + print(json_obj['mediaInfo']['chn_name']) + print(stat_info) + mediaInfo(id=self.id, chn_name=json_obj['mediaInfo']['chn_name']).save() + global current_mediaInfo + current_mediaInfo = mediaInfo.objects.get(pk=self.id) + stat(id=self.id, danmakus=int(stat_info['danmakus']), favorites=stat_info['favorites'], + views=stat_info['views']).save() + + def run(self) -> None: + self.go(self.begin, self.end) + + def get(self): + return current_mediaInfo + + def go(self, start, end): + for num in range(start, end): + if (flag): + time.sleep(1) + logger.info("爬虫进度：%d" % num) + self.id = num + self.save() + else: + logger.info("停止爬虫") + break diff --git a/PixivSearch/model/config.py b/PixivSearch/model/config.py new file mode 100644 index 0000000..2b8e7b0 --- /dev/null +++ b/PixivSearch/model/config.py @@ -0,0 +1,44 @@ +from django.db import models +import json + + +class param(models.Model): + param_name = models.CharField(max_length=10, primary_key=True) + param_value = models.CharField(max_length=128) + + +class stat(models.Model): + id = models.IntegerField(primary_key=True) + danmakus = models.IntegerField() + favorites = models.IntegerField() + views = models.IntegerField() + + # def json(self): + # return bangumi_stat(self['danmakus'],self['favorites'],self['views']) + + +class mediaInfo(models.Model): + id = models.IntegerField(primary_key=True) + chn_name = models.CharField(max_length=128) + + def __str__(self) -> str: + i = {} + i['media_id'] = self.id + i['chn_name'] = self.chn_name + return i + + +class bangumi_list(models.Model): + season_id = models.IntegerField(primary_key=True) + badge = models.CharField(max_length=128) + brief = models.CharField(max_length=128) + copyright = models.CharField(max_length=128) + cover = models.CharField(max_length=128) + favorites = models.IntegerField() + is_finish = models.IntegerField() + newest_ep_index = models.IntegerField() + pub_time = models.DateTimeField() + season_status = models.IntegerField() + title = models.CharField(max_length=128) + total_count = models.IntegerField() + trailer_aid = models.IntegerField() diff --git a/PixivSearch/module/__init__.py b/PixivSearch/module/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/PixivSearch/module/config.py b/PixivSearch/module/config.py deleted file mode 100644 index 2eb0653..0000000 --- a/PixivSearch/module/config.py +++ /dev/null @@ -1,5 +0,0 @@ -from django.db import models - -class param(models.Model): - param_name = models.CharField(max_length=10,primary_key=True) - param_value = models.CharField(max_length=128) diff --git a/PixivSearch/pixiv.py b/PixivSearch/pixiv.py index 4a11cd5..1e3822d 100644 --- a/PixivSearch/pixiv.py +++ b/PixivSearch/pixiv.py @@ -11,6 +11,8 @@ import os import zipfile import logging.config +from PixivSearch.model import config + headers = { 'X-Requested-With': 'XMLHttpRequest', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) ' @@ -24,7 +26,7 @@ logger = logging.getLogger('file') def get_cookies(): _cookies = {} - array = "p_ab_id=5; p_ab_id_2=9; login_ever=yes; device_token=c8c37fdf24b917b4e7fb191fe11c5ca5; search_tools_toggle=1; _ga=GA1.2.887334537.1498628532; PHPSESSID=25745470_75a76e86ff3145b53e21b440183b4822; a_type=0; is_sensei_service_user=1; module_orders_mypage=%5B%7B%22name%22%3A%22recommended_illusts%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22everyone_new_illusts%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22following_new_illusts%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22mypixiv_new_illusts%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22fanbox%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22featured_tags%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22contests%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22sensei_courses%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22spotlight%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22booth_follow_items%22%2C%22visible%22%3Atrue%7D%5D; __utmt=1; __utma=235335808.887334537.1498628532.1498720739.1498819600.5; __utmb=235335808.1.10.1498819600; __utmc=235335808; __utmz=235335808.1498713152.2.2.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; __utmv=235335808.|2=login%20ever=yes=1^3=plan=normal=1^5=gender=male=1^6=user_id=25745470=1^9=p_ab_id=5=1^10=p_ab_id_2=9=1^11=lang=zh_tw=1".split(';') + array = config.param.objects.get(param_name='pixiv_cookie').param_value.split(';') for row in array: k, v = row.strip().split('=', 1) _cookies[k] = v diff --git a/PixivSearch/settings.py b/PixivSearch/settings.py index 8dc8ad5..928f83d 100644 --- a/PixivSearch/settings.py +++ b/PixivSearch/settings.py @@ -9,7 +9,7 @@ https://docs.djangoproject.com/en/1.11/topics/settings/ For the full list of settings and their values, see https://docs.djangoproject.com/en/1.11/ref/settings/ """ - +import logging.config import os # Build paths inside the project like this: os.path.join(BASE_DIR, ...) @@ -37,7 +37,7 @@ INSTALLED_APPS = [ 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', - 'PixivSearch.module', + 'PixivSearch.model' ] MIDDLEWARE = [ @@ -122,3 +122,8 @@ STATIC_URL = '/static/' STATICFILES_DIRS = [ os.path.join(os.path.dirname(__file__), 'static').replace('\\', '/'), ] + +curr_dir = os.path.dirname(os.path.realpath(__file__)) +logging.config.fileConfig('%s%slogging.conf' % (curr_dir,os.sep)) +logger = logging.getLogger('file') + diff --git a/PixivSearch/templates/addConfig.html b/PixivSearch/templates/addConfig.html new file mode 100644 index 0000000..63d9962 --- /dev/null +++ b/PixivSearch/templates/addConfig.html @@ -0,0 +1,27 @@ + + + + 参数管理 + {% load staticfiles %} + + + + +
+ {% csrf_token %} + + +
+ + +
+
+ + \ No newline at end of file diff --git a/PixivSearch/templates/test.html b/PixivSearch/templates/index.html similarity index 96% rename from PixivSearch/templates/test.html rename to PixivSearch/templates/index.html index 3561152..9d2d15b 100644 --- a/PixivSearch/templates/test.html +++ b/PixivSearch/templates/index.html @@ -1,63 +1,63 @@ - - - - {% load staticfiles %} - - - - -
- {% csrf_token %} - - - - - - - - {% if download %} - 图包提取(文件大小:{{ size}}) -{# #} - {% endif %} -
- -
-

{{ msg}}

-
-
    - {% for imageNode in imageNodes %} -
  1. -
    -

    标题：{{imageNode.illustTitle}}，画师：{{imageNode.userName}}，收藏数：{{imageNode.bookmarkCount}}

    - {% if imageNode.localName %} - - {% endif %} -
    -
    2. - {% empty %} -

    {{tip}}

    - {% endfor %} - -
- + + + + {% load staticfiles %} + + + + +
+ {% csrf_token %} + + + + + + + + {% if download %} + 图包提取(文件大小:{{ size}}) +{# #} + {% endif %} +
+ +
+

{{ msg}}

+
+
    + {% for imageNode in imageNodes %} +
  1. +
    +

    标题：{{imageNode.illustTitle}}，画师：{{imageNode.userName}}，收藏数：{{imageNode.bookmarkCount}}

    + {% if imageNode.localName %} + + {% endif %} +
    +
    2. + {% empty %} +

    {{tip}}

    + {% endfor %} + +
+ \ No newline at end of file diff --git a/PixivSearch/urls.py b/PixivSearch/urls.py index db9a4b9..5808d3f 100644 --- a/PixivSearch/urls.py +++ b/PixivSearch/urls.py @@ -14,11 +14,14 @@ Including another URLconf 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) """ from django.conf.urls import url -from PixivSearch.view import search, index, download, haha +from PixivSearch.view import search, index, download, saveConfig, save, get, start urlpatterns = [ url(r'^$', index), url(r'^pixiv/search', search), url(r'^pixiv/download', download), - url(r'^tsdm',haha) + url(r'^tsdm', saveConfig), + url(r'^bangumi/save', save), + url(r'^bangumi/get', get), + url(r'^bangumi/start', start), ] diff --git a/PixivSearch/view.py b/PixivSearch/view.py index a1f227d..b89f988 100644 --- a/PixivSearch/view.py +++ b/PixivSearch/view.py @@ -1,14 +1,18 @@ # coding=utf-8 import os +import django from django.http import Http404, StreamingHttpResponse, HttpResponse from django.shortcuts import render -from PixivSearch import pixiv -from PixivSearch.module import config -from PixivSearch.pixiv import get_nodes +from PixivSearch.dao.bangumi import bangumi +from PixivSearch.settings import logger + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "PixivSearch.settings") +django.setup() # 添加的代码 -logger = pixiv.logging.getLogger('file') +from PixivSearch.model import config +from PixivSearch.pixiv import get_nodes def search(request): @@ -38,13 +42,13 @@ def search(request): nodes = {'imageNodes': array[0], 'msg': array[1]} else: nodes = {'tip': '没有返回结果'} - return render(request, 'test.html', nodes) + return render(request, 'index.html', nodes) except ValueError: raise Http404() def index(request): - return render(request, 'test.html', {'tip': '输入参数进行搜索'}) + return render(request, 'index.html', {'tip': '输入参数进行搜索'}) def download(request): @@ -66,7 +70,31 @@ def download(request): return response -def haha(request): - p=config.param(param_name='123',param_value='456') - p.save() - return HttpResponse("%s" % (config.param.param_name)) +def saveConfig(request): + if 'GET' == request.method and request.GET.get('param_name'): + p = config.param.objects.get(param_name=request.GET.get('param_name')) + return render(request, 'addConfig.html', p.__dict__) + elif 'POST' == request.method and request.POST.get('param_name') and request.POST.get('param_value'): + p = config.param(param_name=request.POST.get('param_name'), param_value=request.POST.get('param_value')) + p.save() + return render(request, 'addConfig.html', p.__dict__) + else: + return render(request, 'addConfig.html') + + +def save(request): + media_id = int(request.GET.get('id')) + bangumi.save(media_id) + return HttpResponse("save success") + + +def get(request): + return HttpResponse(str(bangumi.get(None).__str__())) + + +def start(request): + begin = int(request.GET.get('start')) + end = int(request.GET.get('end')) + bangumi(begin=begin, end=end).start() + + return HttpResponse("start success") diff --git a/PixivSearch/wsgi.py b/PixivSearch/wsgi.py index f079af7..59e2109 100644 --- a/PixivSearch/wsgi.py +++ b/PixivSearch/wsgi.py @@ -1,7 +1,7 @@ """ WSGI config for PixivSearch project. -It exposes the WSGI callable as a module-level variable named ``application``. +It exposes the WSGI callable as a model-level variable named ``application``. For more information on this file, see https://docs.djangoproject.com/en/1.11/howto/deployment/wsgi/