提交

PixivSearch/dao/bangumi.py

@@ -0,0 +1,61 @@
+import json
+import os
+import time
+
+import django
+import requests
+import threading
+from bs4 import BeautifulSoup
+
+from PixivSearch.settings import logger
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "PixivSearch.settings")
+django.setup()
+from PixivSearch.model.config import mediaInfo, stat
+
+current_mediaInfo = mediaInfo(id=0, chn_name='null')
+flag = True
+
+
+class bangumi(threading.Thread):
+    begin = 0
+    end = 0
+    id = 0
+
+    def __init__(self, begin, end):
+        threading.Thread.__init__(self)
+        self.begin = begin
+        self.end = end
+
+    def save(self):
+        req = requests.get("https://www.bilibili.com/bangumi/media/md%d" % self.id)
+        if (req.status_code == 200):
+            tag = BeautifulSoup(req.text, 'lxml')
+            script = tag.select("script")[3].text
+            json_str = script[script.index("=") + 1:script.index("function") - 2]
+            json_obj = json.loads(json_str)
+            stat_info = json_obj['mediaInfo']['stat']
+            print(json_obj['mediaInfo']['chn_name'])
+            print(stat_info)
+            mediaInfo(id=self.id, chn_name=json_obj['mediaInfo']['chn_name']).save()
+            global current_mediaInfo
+            current_mediaInfo = mediaInfo.objects.get(pk=self.id)
+            stat(id=self.id, danmakus=int(stat_info['danmakus']), favorites=stat_info['favorites'],
+                 views=stat_info['views']).save()
+
+    def run(self) -> None:
+        self.go(self.begin, self.end)
+
+    def get(self):
+        return current_mediaInfo
+
+    def go(self, start, end):
+        for num in range(start, end):
+            if (flag):
+                time.sleep(1)
+                logger.info("爬虫进度：%d" % num)
+                self.id = num
+                self.save()
+            else:
+                logger.info("停止爬虫")
+                break

PixivSearch/model/config.py

@@ -0,0 +1,44 @@
+from django.db import models
+import json
+
+
+class param(models.Model):
+    param_name = models.CharField(max_length=10, primary_key=True)
+    param_value = models.CharField(max_length=128)
+
+
+class stat(models.Model):
+    id = models.IntegerField(primary_key=True)
+    danmakus = models.IntegerField()
+    favorites = models.IntegerField()
+    views = models.IntegerField()
+
+    # def json(self):
+    #     return bangumi_stat(self['danmakus'],self['favorites'],self['views'])
+
+
+class mediaInfo(models.Model):
+    id = models.IntegerField(primary_key=True)
+    chn_name = models.CharField(max_length=128)
+
+    def __str__(self) -> str:
+        i = {}
+        i['media_id'] = self.id
+        i['chn_name'] = self.chn_name
+        return i
+
+
+class bangumi_list(models.Model):
+    season_id = models.IntegerField(primary_key=True)
+    badge = models.CharField(max_length=128)
+    brief = models.CharField(max_length=128)
+    copyright = models.CharField(max_length=128)
+    cover = models.CharField(max_length=128)
+    favorites = models.IntegerField()
+    is_finish = models.IntegerField()
+    newest_ep_index = models.IntegerField()
+    pub_time = models.DateTimeField()
+    season_status = models.IntegerField()
+    title = models.CharField(max_length=128)
+    total_count = models.IntegerField()
+    trailer_aid = models.IntegerField()

PixivSearch/module/config.py

@@ -1,5 +0,0 @@
-from django.db import models
-
-class param(models.Model):
-    param_name = models.CharField(max_length=10,primary_key=True)
-    param_value = models.CharField(max_length=128)

PixivSearch/pixiv.py

@@ -11,6 +11,8 @@ import os
 import zipfile
 import logging.config
 
+from PixivSearch.model import config
+
 headers = {
     'X-Requested-With': 'XMLHttpRequest',
     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) '
@@ -24,7 +26,7 @@ logger = logging.getLogger('file')
 
 def get_cookies():
     _cookies = {}
-    array = "p_ab_id=5; p_ab_id_2=9; login_ever=yes; device_token=c8c37fdf24b917b4e7fb191fe11c5ca5; search_tools_toggle=1; _ga=GA1.2.887334537.1498628532; PHPSESSID=25745470_75a76e86ff3145b53e21b440183b4822; a_type=0; is_sensei_service_user=1; module_orders_mypage=%5B%7B%22name%22%3A%22recommended_illusts%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22everyone_new_illusts%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22following_new_illusts%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22mypixiv_new_illusts%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22fanbox%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22featured_tags%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22contests%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22sensei_courses%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22spotlight%22%2C%22visible%22%3Atrue%7D%2C%7B%22name%22%3A%22booth_follow_items%22%2C%22visible%22%3Atrue%7D%5D; __utmt=1; __utma=235335808.887334537.1498628532.1498720739.1498819600.5; __utmb=235335808.1.10.1498819600; __utmc=235335808; __utmz=235335808.1498713152.2.2.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; __utmv=235335808.|2=login%20ever=yes=1^3=plan=normal=1^5=gender=male=1^6=user_id=25745470=1^9=p_ab_id=5=1^10=p_ab_id_2=9=1^11=lang=zh_tw=1".split(';')
+    array = config.param.objects.get(param_name='pixiv_cookie').param_value.split(';')
     for row in array:
         k, v = row.strip().split('=', 1)
         _cookies[k] = v

PixivSearch/settings.py

@@ -9,7 +9,7 @@ https://docs.djangoproject.com/en/1.11/topics/settings/
 For the full list of settings and their values, see
 https://docs.djangoproject.com/en/1.11/ref/settings/
 """
-
+import logging.config
 import os
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
@@ -37,7 +37,7 @@ INSTALLED_APPS = [
     'django.contrib.sessions',
     'django.contrib.messages',
     'django.contrib.staticfiles',
-    'PixivSearch.module',
+    'PixivSearch.model'
 ]
 
 MIDDLEWARE = [
@@ -122,3 +122,8 @@ STATIC_URL = '/static/'
 STATICFILES_DIRS = [
     os.path.join(os.path.dirname(__file__), 'static').replace('\\', '/'),
 ]
+
+curr_dir = os.path.dirname(os.path.realpath(__file__))
+logging.config.fileConfig('%s%slogging.conf' % (curr_dir,os.sep))
+logger = logging.getLogger('file')
+

PixivSearch/templates/addConfig.html

@@ -0,0 +1,27 @@
+<html xmlns="http://www.w3.org/1999/html">
+<head>
+    <meta charset="UTF-8">
+    <title>参数管理</title>
+    {% load staticfiles %}
+    <script src="{% static "js/jquery-3.2.1.min.js"%}"></script>
+    <script>
+        $(function () {
+            $("button").click(function(){
+                $("form").attr("method",$(this).attr("id")).submit();
+            });
+
+        })
+    </script>
+</head>
+<body>
+<form action="/tsdm">
+    {% csrf_token %}
+    <label>参数名<input name="param_name" value="{{param_name}}"/></label>
+    <label>参数值<input name="param_value" value="{{param_value}}"/></label>
+    <div>
+        <button id="POST">设置</button>
+        <button id="GET">获取</button>
+    </div>
+</form>
+</body>
+</html>

PixivSearch/templates/index.html

@@ -1,63 +1,63 @@
-<html xmlns="http://www.w3.org/1999/html">
+<html xmlns="http://www.w3.org/1999/html">
-<head>
+<head>
-    <style>
+    <style>
-        div{
+        div{
-            text-align:center;
+            text-align:center;
-        }
+        }
-        ol{list-style-type:demical;}
+        ol{list-style-type:demical;}
-        ol li{ list-style-position:outside;}
+        ol li{ list-style-position:outside;}
-    </style>
+    </style>
-    {% load staticfiles %}
+    {% load staticfiles %}
-    <script src="{% static "js/jquery-3.2.1.min.js"%}"></script>
+    <script src="{% static "js/jquery-3.2.1.min.js"%}"></script>
-    <script>
+    <script>
-        $(function () {
+        $(function () {
-
+
-        });
+        });
-
+
-        function check() {
+        function check() {
-            $("[name=pageSize],[name=order]").each(function () {
+            $("[name=pageSize],[name=order]").each(function () {
-                if($(this).val()!=''&&$(this).val()<=0){
+                if($(this).val()!=''&&$(this).val()<=0){
-                    $(this).val('');
+                    $(this).val('');
-                    alert($(this).parent().text()+'不能小于等于0');
+                    alert($(this).parent().text()+'不能小于等于0');
-                    $(this).focus();
+                    $(this).focus();
-                }
+                }
-            });
+            });
-        }
+        }
-    </script>
+    </script>
-</head>
+</head>
-<body>
+<body>
-<form  action="/pixiv/search"  method="post" onsubmit="check()">
+<form  action="/pixiv/search"  method="post" onsubmit="check()">
-    {% csrf_token %}
+    {% csrf_token %}
-    <label>关键字<input name="word" placeholder="R-18"/></label>
+    <label>关键字<input name="word" placeholder="R-18"/></label>
-    <label>页数<input name="pageSize" type="number" placeholder="10"/></label>
+    <label>页数<input name="pageSize" type="number" placeholder="10"/></label>
-    <label>排行数<input name="order" type="number" placeholder="10"/></label>
+    <label>排行数<input name="order" type="number" placeholder="10"/></label>
-    <label>结果显示类型：文本信息<input type="radio" value="" name="type" checked/></label>
+    <label>结果显示类型：文本信息<input type="radio" value="" name="type" checked/></label>
-    <label>图文信息<input type="radio" value="img" name="type"/></label>
+    <label>图文信息<input type="radio" value="img" name="type"/></label>
-
+
-    <input type="submit" />
+    <input type="submit" />
-    {% if  download %}
+    {% if  download %}
-        <a href="{% static "/download/" %}{{download}}">图包提取(文件大小:{{ size}})</a>
+        <a href="{% static "/download/" %}{{download}}">图包提取(文件大小:{{ size}})</a>
-{#        <button id="download" onclick="window.open('{% static "/download/" %}{{download}}')">图包提取(文件大小:{{ size}})</button>#}
+{#        <button id="download" onclick="window.open('{% static "/download/" %}{{download}}')">图包提取(文件大小:{{ size}})</button>#}
-    {% endif %}
+    {% endif %}
-</form>
+</form>
-
+
-<div>
+<div>
-    <h1>{{ msg}}</h1>
+    <h1>{{ msg}}</h1>
-</div>
+</div>
-<ol>
+<ol>
-    {% for imageNode in imageNodes %}
+    {% for imageNode in imageNodes %}
-        <li>
+        <li>
-            <div>
+            <div>
-                <h1>标题：<a href="https://www.pixiv.net/member_illust.php?mode=medium&illust_id={{imageNode.illustId}}" target="_blank">{{imageNode.illustTitle}}</a>，画师：<a href="https://www.pixiv.net/member.php?id={{ imageNode.userId}}">{{imageNode.userName}}</a>，收藏数：{{imageNode.bookmarkCount}}</h1>
+                <h1>标题：<a href="https://www.pixiv.net/member_illust.php?mode=medium&illust_id={{imageNode.illustId}}" target="_blank">{{imageNode.illustTitle}}</a>，画师：<a href="https://www.pixiv.net/member.php?id={{ imageNode.userId}}">{{imageNode.userName}}</a>，收藏数：{{imageNode.bookmarkCount}}</h1>
-                {% if imageNode.localName %}
+                {% if imageNode.localName %}
-                    <img src="{% static "images/"%}{{imageNode.localName}}">
+                    <img src="{% static "images/"%}{{imageNode.localName}}">
-                {% endif %}
+                {% endif %}
-            </div>
+            </div>
-        </li>
+        </li>
-    {% empty %}
+    {% empty %}
-        <p>{{tip}}</p>
+        <p>{{tip}}</p>
-    {% endfor %}
+    {% endfor %}
-
+
-</ol>
+</ol>
-</body>
+</body>
 </html>

PixivSearch/urls.py

@@ -14,11 +14,14 @@ Including another URLconf
     2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
 """
 from django.conf.urls import url
-from PixivSearch.view import search, index, download, haha
+from PixivSearch.view import search, index, download, saveConfig, save, get, start
 
 urlpatterns = [
     url(r'^$', index),
     url(r'^pixiv/search', search),
     url(r'^pixiv/download', download),
-    url(r'^tsdm',haha)
+    url(r'^tsdm', saveConfig),
+    url(r'^bangumi/save', save),
+    url(r'^bangumi/get', get),
+    url(r'^bangumi/start', start),
 ]

PixivSearch/view.py

@@ -1,14 +1,18 @@
 # coding=utf-8
 import os
 
+import django
 from django.http import Http404, StreamingHttpResponse, HttpResponse
 from django.shortcuts import render
 
-from PixivSearch import pixiv
+from PixivSearch.dao.bangumi import bangumi
-from PixivSearch.module import config
+from PixivSearch.settings import logger
-from PixivSearch.pixiv import get_nodes
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "PixivSearch.settings")
+django.setup()  # 添加的代码
 
-logger = pixiv.logging.getLogger('file')
+from PixivSearch.model import config
+from PixivSearch.pixiv import get_nodes
 
 
 def search(request):
@@ -38,13 +42,13 @@ def search(request):
                 nodes = {'imageNodes': array[0], 'msg': array[1]}
         else:
             nodes = {'tip': '没有返回结果'}
-        return render(request, 'test.html', nodes)
+        return render(request, 'index.html', nodes)
     except ValueError:
         raise Http404()
 
 
 def index(request):
-    return render(request, 'test.html', {'tip': '输入参数进行搜索'})
+    return render(request, 'index.html', {'tip': '输入参数进行搜索'})
 
 
 def download(request):
@@ -66,7 +70,31 @@ def download(request):
     return response
 
 
-def haha(request):
+def saveConfig(request):
-    p=config.param(param_name='123',param_value='456')
+    if 'GET' == request.method and request.GET.get('param_name'):
-    p.save()
+        p = config.param.objects.get(param_name=request.GET.get('param_name'))
-    return HttpResponse("<html><body>%s</body></html>" % (config.param.param_name))
+        return render(request, 'addConfig.html', p.__dict__)
+    elif 'POST' == request.method and request.POST.get('param_name') and request.POST.get('param_value'):
+        p = config.param(param_name=request.POST.get('param_name'), param_value=request.POST.get('param_value'))
+        p.save()
+        return render(request, 'addConfig.html', p.__dict__)
+    else:
+        return render(request, 'addConfig.html')
+
+
+def save(request):
+    media_id = int(request.GET.get('id'))
+    bangumi.save(media_id)
+    return HttpResponse("save success")
+
+
+def get(request):
+    return HttpResponse(str(bangumi.get(None).__str__()))
+
+
+def start(request):
+    begin = int(request.GET.get('start'))
+    end = int(request.GET.get('end'))
+    bangumi(begin=begin, end=end).start()
+
+    return HttpResponse("start success")

PixivSearch/wsgi.py

@@ -1,7 +1,7 @@
 """
 WSGI config for PixivSearch project.
 
-It exposes the WSGI callable as a module-level variable named ``application``.
+It exposes the WSGI callable as a model-level variable named ``application``.
 
 For more information on this file, see
 https://docs.djangoproject.com/en/1.11/howto/deployment/wsgi/