update

PixivSearch/aliyun/photo/AliyunPhoto.py

@@ -0,0 +1,140 @@
+import hashlib
+import json
+import os
+import time
+
+import aliyunsdkcore
+import oss2 as oss2
+from aliyunsdkcloudphoto.request.v20170711 import ListPhotoStoresRequest, FetchLibrariesRequest, \
+    CreateTransactionRequest, CreatePhotoRequest, EditPhotosRequest, GetPublicAccessUrlsRequest, ListPhotosRequest, \
+    DeletePhotosRequest, InactivatePhotosRequest, GetLibraryRequest
+from aliyunsdkcore.auth.credentials import RamRoleArnCredential
+from aliyunsdkcore.client import AcsClient
+from aliyunsdkcore.profile import region_provider
+from aliyunsdkcore.request import CommonRequest
+
+region_id = 'cn-shanghai'
+ak = 'LTAIWzPnmkJs2qpL'
+secret = 'LIIq3HumctXPp0WT8c06yDiFbKKiVe'
+region_provider.add_endpoint('Oss', region_id, 'oss-cn-shanghai.aliyuncs.com')
+region_provider.add_endpoint('CloudPhoto', region_id, 'cloudphoto.cn-shanghai.aliyuncs.com')
+aliyunsdkcore.request.set_default_protocol_type("https")
+
+
+class MyClient(AcsClient):
+
+    def __init__(self, arn, sessionName):
+        super().__init__(region_id=region_id, credential=RamRoleArnCredential(ak, secret, arn, sessionName))
+        self.StoreName = None
+
+    def get_md5_01(self, file_path):
+        md5 = None
+        if os.path.isfile(file_path):
+            f = open(file_path, 'rb')
+            md5_obj = hashlib.md5()
+            md5_obj.update(f.read())
+            hash_code = md5_obj.hexdigest()
+            f.close()
+            md5 = str(hash_code).lower()
+        return md5
+
+    def do_action_with_exception(self, acs_request):
+        return json.loads(super().do_action_with_exception(acs_request).decode())
+
+    def showOption(self, options, key, desc):
+        choose = []
+        for index, option in enumerate(options):
+            choose.append(option[key])
+            print('{index}:{name}'.format(index=index + 1, name=option[key]))
+        return choose[(int)(input('输入{desc}\n'.format(desc=desc))) - 1]
+
+    def listPhotoStores(self):
+        request = ListPhotoStoresRequest.ListPhotoStoresRequest()
+        response = self.do_action_with_exception(request)
+        print('PhotoStores:')
+        photoStores = response['PhotoStores']
+        self.StoreName = self.showOption(photoStores, 'Name', 'StoreName')
+
+    def listLibraries(self):
+        request = FetchLibrariesRequest.FetchLibrariesRequest()
+        request.set_StoreName(self.StoreName)
+        response = self.do_action_with_exception(request)
+        self.Libraries = response['Libraries']
+
+    def uploadPhoto(self):
+        if self.StoreName is None:
+            self.listPhotoStores()
+        request = CreateTransactionRequest.CreateTransactionRequest()
+        filePath = input('输入上传文件路径\n').replace('\\', '/')
+        fileName = filePath.split('/')[-1]
+        request.set_Size(os.path.getsize(filePath))
+        request.set_Ext(fileName[-fileName[::-1].index('.'):])
+        request.set_Md5(self.get_md5_01(filePath))
+        request.set_StoreName(self.StoreName)
+        response = self.do_action_with_exception(request)
+        print(response)
+        Upload = response['Transaction']['Upload']
+        FileId = Upload['FileId']
+        SessionId = Upload['SessionId']
+        Bucket = Upload['Bucket']
+        OssEndpoint = Upload['OssEndpoint']
+        ObjectKey = Upload['ObjectKey']
+
+        auth = oss2.StsAuth(self._signer._session_credential[0], self._signer._session_credential[1],
+                            self._signer._session_credential[2])
+        bucket = oss2.Bucket(auth, OssEndpoint, Bucket)
+        with open(filePath, 'rb') as fileobj:
+            result = bucket.put_object(ObjectKey, fileobj)
+            print('文件上传状态{status}'.format(status=result.status))
+
+        request = CreatePhotoRequest.CreatePhotoRequest()
+        request.set_FileId(FileId)
+        request.set_PhotoTitle(fileName)
+        request.set_SessionId(SessionId)
+        request.set_StoreName(self.StoreName)
+        request.set_UploadType('manual')
+        response = self.do_action_with_exception(request)
+        print(response)
+
+    def listPhotos(self):
+        if self.StoreName == None:
+            self.listPhotoStores()
+        request = ListPhotosRequest.ListPhotosRequest()
+        request.set_StoreName(self.StoreName)
+        request.set_State('all')
+        response = self.do_action_with_exception(request)
+        print(response)
+        return response['Photos']
+
+    def getPhoto(self):
+        return self.showOption(self.listPhotos(), 'IdStr', '照片Id')
+
+    def sharePhoto(self):
+        IdStr = self.getPhoto()
+        request = EditPhotosRequest.EditPhotosRequest()
+        request.set_PhotoIds([IdStr])
+        request.set_StoreName(self.StoreName)
+        request.set_ShareExpireTime((int(round(time.time())) + 60 * 60) * 1000)
+        response = self.do_action_with_exception(request)
+        print(response)
+
+        request = GetPublicAccessUrlsRequest.GetPublicAccessUrlsRequest()
+        request.set_DomainType('OSS')
+        request.set_PhotoIds([IdStr])
+        request.set_StoreName(self.StoreName)
+        request.set_ZoomType('style/2')
+        response = self.do_action_with_exception(request)
+        print(response)
+
+
+def client(arn, sessionName):
+    ram_role_arn_credential = RamRoleArnCredential('LTAIWzPnmkJs2qpL', 'LIIq3HumctXPp0WT8c06yDiFbKKiVe',
+                                                   arn, sessionName)
+    return AcsClient(region_id='cn-shanghai', credential=ram_role_arn_credential)
+
+
+if __name__ == '__main__':
+
+    myClient = MyClient('acs:ram::1098806312754985:role/aliyunosstokengeneratorrole', 'pqh001')
+    myClient.listPhotoStores()
+    myClient.sharePhoto()

PixivSearch/baidu/downLoadBduImg.py

@@ -0,0 +1,118 @@
+import json
+import os
+import re
+from concurrent import futures
+from datetime import datetime
+
+import requests
+
+
+def get_cookies():
+    _cookies = {}
+    array = "BDqhfp=fate%26%26NaN%26%260%26%261; BIDUPSID=8689C23BFD1526702A4EF173F3A809DD; BDRCVFR[dG2JNJb_ajR]=mk3SLVN4HKm; userFrom=null; BAIDUID=8689C23BFD152670722FAAEB4DDC55FA:FG=1; BDRCVFR[-pGxjrCMryR]=mk3SLVN4HKm".split(
+        ';')
+    for row in array:
+        k, v = row.strip().split('=', 1)
+        _cookies[k] = v
+    return _cookies
+
+
+# 图片保存路径
+savePath = None
+threadNum = 10
+startTime = None
+
+
+def getBaiduImage(word):
+    global startTime, savePath
+    params = []
+    startTime = datetime.now()
+    start = threadNum
+    i = 0
+    filepath = None
+    savePath = r'{savePath}/{word}'.format(savePath=savePath, word=word)
+    if not os.path.exists(savePath):
+        os.makedirs(savePath)
+    while True:
+        try:
+            url = r"https://image.baidu.com/search/acjson?tn=resultjson_com&ipn=rj&ct=201326592&is=&fp=result&queryWord={" \
+                  r"queryWord}&cl=2&lm=-1&ie=utf-8&oe=utf-8&adpicid=&st=-1&z=&ic=0&word={" \
+                  r"word}&s=&se=&tab=&width=&height=&face=0&istype=2&qc=&nc=1&fr=&pn={pn}&rn={rn}&gsm=3c&1523890541764= "
+            url = url.format(queryWord=word, word=word, pn=start, rn=threadNum)
+            print('request url:%s' % url)
+            req = requests.get(url)
+            if req.status_code == 200:
+                req.encoding = 'utf-8'
+                obj = json.loads(req.text.replace('\\\'', ''))
+                if len(obj['data']) == 1:
+                    break
+                for img in obj['data']:
+                    if 'fromPageTitle' in img:
+                        print('图片:%s\t添加到下载队列' % img['fromPageTitleEnc'])
+                        if 'replaceUrl' in img:
+                            url = img['replaceUrl'][0]['ObjURL']
+                            params.append((url, i))
+                            i += 1
+                            if not filepath is None and os.path.exists(filepath):
+                                os.remove(filepath)
+                            filepath = r'{savePath}/图片下载队列填充：{i}'.format(savePath=savePath, word=word, i=i)
+                            file = open(filepath, 'w')
+                            file.close()
+                start += threadNum
+        except BaseException as e:
+            print(repr(e))
+    if not filepath is None and os.path.exists(filepath):
+        os.remove(filepath)
+    executors = futures.ThreadPoolExecutor(threadNum)
+    try:
+        with executors as executor:
+            executor.map(downImage, params)
+    except BaseException as e:
+        print(repr(e))
+
+
+def downImage(params):
+    try:
+        url = params[0]
+        index = params[1]
+        print(r'开始下载图片{url}'.format(url=url))
+        imgurl = requests.get(url, headers={"Referer": "image.baidu.com"})
+        if imgurl.status_code == 200:
+            format = url[-url[::-1].index('.'):]
+            imgPath = r'{savePath}/fate_{index}.{format}'.format(savePath=savePath,
+                                                                 index=index,
+                                                                 format=format)
+            f = open(imgPath, 'wb')
+            f.write(imgurl.content)
+            f.close()
+            print(r'图片{url}成功下载到{imgPath}'.format(url=url, imgPath=imgPath))
+    except BaseException as e:
+        print(repr(e))
+
+
+if __name__ == '__main__':
+    # str = ''
+    # while True:
+    #     str = input('输入要下载图片的关键字，输入 exit  退出程序\n')
+    #     if not str == 'exit':
+    #         while True:
+    #             savePath = input('输入图片存放目录：例如 E:/图片,注意目录之间使用正斜杠隔开"/"\n')
+    #             if re.fullmatch(r"[a-zA-z]:(/[\u4e00-\u9fa5_a-zA-Z0-9]+)+", savePath) is None:
+    #                 print(r'图片目录{savePath}不合法请重新输入'.format(savePath=savePath))
+    #             else:
+    #                 break
+    #         getBaiduImage(str)
+    #         print(r'使用{threadNum}线程成功下载{count}张图片到目录{path}下，耗时：{second}'.format(threadNum=threadNum,
+    #                                                                             count=len(os.listdir(savePath)),
+    #                                                                             path=savePath,
+    #                                                                             second=datetime.now() - startTime))
+    #         flag = False
+    #     else:
+    #         print('exit')
+    #         break
+    req = requests.post('https://gifmaker.develophelper.com/gif/make', {'tplid': 1,
+                                                                        'content': '好啊##$@?$?@$##就算你是一流工程师##$@?$?@$##就算你出报告再完美##$@?$?@$##我叫你改报告你就要改##$@?$?@$##毕竟我是客户##$@?$?@$##客户了不起啊##$@?$?@$##sorry 客户真的了不起##$@?$?@$##以后叫他天天改报告##$@?$?@$##天天改 天天改'})
+    if req.status_code==200:
+        response = json.loads(req.content.decode())
+        if 'd' in response:
+            print(response['d'])

PixivSearch/dao/Comment.py

@@ -0,0 +1,119 @@
+import json
+import os
+import shutil
+import threading
+from concurrent import futures
+
+import requests
+from lxml import etree
+
+
+class Comment:
+    lock = threading.Lock()  # 多线程全局资源锁
+
+    def __init__(self, keywords_=None) -> None:
+        super().__init__()
+        self.obj = {'data': {}, 'flag': False}
+        self.keywords = keywords_
+
+    # 获取番剧合集弹幕排行榜
+    def getCommentSort(self, cids):
+
+        urls = []
+        for cid in cids:
+            urls.extend(getCidUrls(cid))
+        with futures.ThreadPoolExecutor(32) as executor:
+            executor.map(self.count, urls)
+        for index, data in enumerate(
+                sorted(self.obj["data"].items(), key=lambda d: d[1], reverse=True)[
+                :50]):
+            print('{index}:{data}'.format(index=index + 1, data=data))
+
+    # 获取番剧合集弹幕排行榜
+    def count(self, url, desc=None):
+        bytes = requests.get(url).content
+        comment_selector = etree.HTML(bytes)
+        if not desc is None:
+            print(desc)
+        print("url=%s" % url)
+        for comment in comment_selector.xpath('//i//d/text()'):
+            if comment in self.obj["data"]:
+                with self.lock:
+                    self.obj["data"][comment] = self.obj["data"][comment] + 1
+            else:
+                with self.lock:
+                    self.obj["data"][comment] = 1
+            if not self.obj["flag"]:
+                for keyword in self.keywords:
+                    if keyword in comment:
+                        self.obj["flag"] = True
+
+
+# 根据cid获取历史弹幕地址
+def getCidUrls(cid):
+    urls = []
+    url = "https://comment.bilibili.com/rolldate,%d" % cid
+    req = requests.get(url)
+    if len(req.text) > 0:
+        for i in json.loads(req.text):
+            urls.append("https://comment.bilibili.com/dmroll,%s,%d" % (i['timestamp'], cid))
+    else:
+        urls.append("https://comment.bilibili.com/%d.xml" % cid)
+    return urls
+
+
+# 下载弹幕
+def downloadXml(path, cid, size=None, histroy=True):
+    dlist = set()
+    flag = None
+    if histroy:
+        flag = parseXml(getCidUrls(cid), dlist, size)
+    else:
+        parseXml("https://comment.bilibili.com/%d.xml" % cid, dlist, size)
+    if size is None or (histroy and not size is None and flag):
+        if os.path.exists(path):
+            shutil.rmtree(path)
+        os.makedirs(path)
+        f = open('{path}/{cid}.xml'.format(path=path, cid=cid), 'wb')
+        f.write(b'<?xml version="1.0" encoding="UTF-8"?><i>')
+        for i in dlist:
+            f.write(('\r\n' + i).encode())
+        f.write(b'\r\n</i>')
+        f.close()
+
+
+def xml(url):
+    bytes = requests.get(url).content
+    return etree.HTML(bytes)
+
+
+def parseXml(urls, dlist, size=None):
+    if isinstance(urls, str):
+        urls = [urls]
+    if not size is None:
+        size = float(size.strip('%')) / 100.0
+    for url in urls:
+        comment_selector = xml(url)
+        list = comment_selector.xpath('//i//d/text()')
+        maxlimit = int(comment_selector.xpath('//i//maxlimit/text()')[0])
+
+        if len(list) > 0:
+            print('弹幕数：{list},最大弹幕数：{maxlimit},弹幕池填充：{p}'.format(list=len(list), maxlimit=maxlimit,
+                                                                 p='%.2f%%' % (len(list) / maxlimit * 100)))
+            for element in comment_selector.xpath('//i//d'):
+                if len(element.xpath("text()")) > 0:
+                    fstr = '<d p="{p}">{content}</d>'.format(p=str(element.xpath("@p")[0]),
+                                                             content=str(element.xpath("text()")[0]))
+                    dlist.add(fstr)
+
+                currentSize = len(dlist) / maxlimit
+                print('填充率：{l}'.format(l='%.2f%%' % (currentSize * 100)))
+                if not size is None and currentSize >= size:
+                    return True
+    return False
+
+
+if __name__ == '__main__':
+    cids = [7636499, 7636501, 7636500, 7636503, 7636504, 7636502, 7636509, 7636508, 7636506, 7636507, 7636505]
+
+    downloadXml('F:/ABC',12026697,histroy=False)

PixivSearch/dao/bangumi.py

@@ -0,0 +1,185 @@
+import _thread
+import json
+import math
+import os
+from concurrent import futures
+from queue import Queue
+
+import django
+import requests
+from bs4 import BeautifulSoup
+
+from PixivSearch.settings import logger
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "PixivSearch.settings")
+django.setup()
+from PixivSearch.model.config import mediaInfo, stat
+
+current_mediaInfo = None
+
+isStop = None
+
+
+def stop_():
+    global isStop
+    isStop = True
+
+
+def save(params):
+    if isStop:
+        return
+    logger.info(params)
+    bangumi_id = params[0]
+    season_id = params[1]
+    media_id = params[2]
+
+    url = "https://www.bilibili.com/bangumi/media/md%d" % media_id
+    try:
+        req = requests.get(url, timeout=10)
+    except BaseException as e:
+        logger.error(repr(e))
+        save(params)
+    logger.info("request_url=%s,status_code=%d" % (url, req.status_code))
+    if req.status_code == 200:
+        json_obj = getJsonText(req, 3)
+        try:
+            if 'mediaInfo' in json_obj and 'stat' in json_obj['mediaInfo'] and 'chn_name' in json_obj['mediaInfo']:
+                stat_info = json_obj['mediaInfo']['stat']
+                print(stat_info)
+                mediaInfo(bangumi_id=bangumi_id, season_id=season_id, media_id=media_id,
+                          chn_name=json_obj['mediaInfo']['chn_name']).save()
+                global current_mediaInfo
+                current_mediaInfo = mediaInfo.objects.get(pk=season_id)
+                stat(id=season_id, danmakus=int(stat_info['danmakus']), favorites=stat_info['favorites'],
+                     views=stat_info['views']).save()
+        except BaseException as e:
+            logger.error(repr(e))
+
+
+def getJsonText(req, index):
+    tag = BeautifulSoup(req.text, 'lxml')
+    script = tag.select("script")[index].text
+    json_str = script[script.index("=") + 1:script.index("function") - 2]
+    return json.loads(json_str)
+
+
+def get_():
+    global current_mediaInfo
+    return current_mediaInfo
+
+
+page_size = 10
+queue = Queue(page_size)
+
+
+def listen():
+    while True:
+        ids = queue.get()
+        try:
+            executors = futures.ThreadPoolExecutor(page_size)
+            with executors as executor:
+                executor.map(save, ids)
+            logger.info('结束爬虫')
+        except BaseException as e:
+            logger.error(repr(e))
+
+
+_thread.start_new_thread(listen, ())
+
+#遍历所有专题视频收藏数信息
+def getIds():
+    seasonIdList = []
+    page = 1
+    pages = None
+    name = 'seasonListCallback'
+    global isStop
+    isStop = False
+    while isStop == False and (pages is None or page <= pages):
+
+        url = 'https://bangumi.bilibili.com/web_api/season/index_global?page=%d&page_size=%d' % (page, page_size)
+        logger.info(url)
+        try:
+            req = requests.get(url, timeout=10)
+            if req.status_code == 200:
+                json_obj = json.loads(req.text)
+                if 'result' in json_obj and 'list' in json_obj['result']:
+                    bangumiList = json_obj['result']['list']
+                    ids = []
+                    for bangumi in bangumiList:
+                        if isStop:
+                            break
+                        if 'season_id' in bangumi:
+                            season_id = int(bangumi['season_id'])
+                            if season_id in seasonIdList:
+                                continue
+                            url = 'https://bangumi.bilibili.com/jsonp/seasoninfo/%d.ver?callback=%s&jsonp=jsonp' % (
+                                season_id, name)
+                            logger.info(url)
+                            req = requests.get(url, timeout=10)
+                            if req.status_code == 200:
+                                child_json_obj = json.loads(
+                                    req.text.replace('seasonListCallback(', '').replace(');', ''))
+                                if 'result' in child_json_obj and 'bangumi_id' in child_json_obj['result']:
+                                    bangumi_id = int(child_json_obj['result']['bangumi_id'])
+                                    if 'media' in child_json_obj['result']:
+                                        media_id = int(child_json_obj['result']['media']['media_id'])
+                                        ids.append((bangumi_id, season_id, media_id))
+                                        seasonIdList.append(season_id)
+                if pages is None and 'count' in json_obj['result']:
+                    pages = int(math.ceil(int(json_obj['result']['count']) / page_size))
+                page = page + 1
+                logger.info('获取id数量%d' % len(ids))
+                queue.put(ids)
+        except BaseException as e:
+            logger.error(repr(e))
+            continue
+
+#根据aid获取cid
+def getCid(aid, type=None):
+    while True and aid > 0:
+        url = "https://api.bilibili.com/x/web-interface/archive/stat?aid=%d" % aid
+        print(url)
+        req = requests.get(url)
+        code = json.loads(req.text)["code"]
+        if code == 0:
+            req = requests.get("https://www.bilibili.com/video/av%d" % aid)
+            if req.status_code == 200:
+                json_obj = getJsonText(req, 9)
+                if "videoData" in json_obj and "pages" in json_obj['videoData'] and len(
+                        json_obj['videoData']['pages']) > 0 and "cid" in json_obj['videoData']['pages'][0]:
+                    cid = json_obj['videoData']['pages'][0]['cid']
+                    print('cid=%s' % cid)
+                    return cid
+        if type is None:
+            break
+        else:
+            if type:
+                aid = aid - 1
+            else:
+                aid = aid + 1
+
+#根据aid获取cid
+def getCids(aid):
+    s = {"min": getCid(aid, True), "max": getCid(aid, False)}
+    return s
+
+#获取专题所有cid
+def episodeIdToCid(episode_id):
+    cids = []
+    url = "https://www.bilibili.com/bangumi/play/ep%d" % episode_id
+    print("url=%s" % url)
+    req = requests.get(url)
+    json_obj = getJsonText(req, 8)
+    if "epList" in json_obj:
+        for i in json_obj["epList"]:
+            cids.append(i['cid'])
+    return cids
+
+
+
+if __name__ == '__main__':
+    # print(getCids(29416))
+
+    req=requests.post('https://api.bilibili.com/x/v2/history/shadow/set','')
+    # obj = loadData([34807341], [])
+

PixivSearch/logging.conf

@@ -0,0 +1,33 @@
+[loggers]
+keys=root,main
+
+[handlers]
+keys=consoleHandler,fileHandler
+
+[formatters]
+keys=fmt
+
+[logger_root]
+level=DEBUG
+handlers=consoleHandler
+
+[logger_main]
+level=DEBUG
+qualname=file
+handlers=fileHandler
+
+[handler_consoleHandler]
+class=StreamHandler
+level=INFO
+formatter=fmt
+args=(sys.stdout,)
+
+[handler_fileHandler]
+class=FileHandler
+level=DEBUG
+formatter=fmt
+args=('pixiv.log','a','utf-8',False)
+
+[formatter_fmt]
+format=%(asctime)s - %(name)s - %(levelname)s - %(module)s:line:%(lineno)d - %(message)s
+datefmt=%Y-%m-%d %H:%M:%S

PixivSearch/model/config.py

@@ -0,0 +1,43 @@
+from django.db import models
+
+
+class param(models.Model):
+    param_name = models.CharField(max_length=10, primary_key=True)
+    param_value = models.CharField(max_length=128)
+
+
+class stat(models.Model):
+    id = models.IntegerField(primary_key=True)
+    danmakus = models.IntegerField()
+    favorites = models.IntegerField()
+    views = models.IntegerField()
+
+    # def json(self):
+    #     return bangumi_stat(self['danmakus'],self['favorites'],self['views'])
+
+
+class mediaInfo(models.Model):
+    bangumi_id = models.IntegerField()
+    season_id = models.IntegerField(primary_key=True)
+    media_id = models.IntegerField()
+    chn_name = models.CharField(max_length=128)
+
+    def __str__(self) -> str:
+        i = {'media_id': self.id, 'chn_name': self.chn_name}
+        return i
+
+
+class bangumi_list(models.Model):
+    season_id = models.IntegerField(primary_key=True)
+    badge = models.CharField(max_length=128)
+    brief = models.CharField(max_length=128)
+    copyright = models.CharField(max_length=128)
+    cover = models.CharField(max_length=128)
+    favorites = models.IntegerField()
+    is_finish = models.IntegerField()
+    newest_ep_index = models.IntegerField()
+    pub_time = models.DateTimeField()
+    season_status = models.IntegerField()
+    title = models.CharField(max_length=128)
+    total_count = models.IntegerField()
+    trailer_aid = models.IntegerField()

PixivSearch/model/migrations/0001_initial.py

@@ -0,0 +1,57 @@
+# Generated by Django 2.0 on 2018-03-24 17:02
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='bangumi_list',
+            fields=[
+                ('season_id', models.IntegerField(primary_key=True, serialize=False)),
+                ('badge', models.CharField(max_length=128)),
+                ('brief', models.CharField(max_length=128)),
+                ('copyright', models.CharField(max_length=128)),
+                ('cover', models.CharField(max_length=128)),
+                ('favorites', models.IntegerField()),
+                ('is_finish', models.IntegerField()),
+                ('newest_ep_index', models.IntegerField()),
+                ('pub_time', models.DateTimeField()),
+                ('season_status', models.IntegerField()),
+                ('title', models.CharField(max_length=128)),
+                ('total_count', models.IntegerField()),
+                ('trailer_aid', models.IntegerField()),
+            ],
+        ),
+        migrations.CreateModel(
+            name='mediaInfo',
+            fields=[
+                ('bangumi_id', models.IntegerField(primary_key=True, serialize=False)),
+                ('season_id', models.IntegerField()),
+                ('media_id', models.IntegerField()),
+                ('chn_name', models.CharField(max_length=128)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='param',
+            fields=[
+                ('param_name', models.CharField(max_length=10, primary_key=True, serialize=False)),
+                ('param_value', models.CharField(max_length=128)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='stat',
+            fields=[
+                ('id', models.IntegerField(primary_key=True, serialize=False)),
+                ('danmakus', models.IntegerField()),
+                ('favorites', models.IntegerField()),
+                ('views', models.IntegerField()),
+            ],
+        ),
+    ]

PixivSearch/pixiv/pixiv.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python
+# coding:utf-8
+import json
+import os
+import sys
+import threading
+from concurrent import futures
+from datetime import datetime
+
+import requests
+from bs4 import BeautifulSoup
+
+import root
+from PixivSearch.model import config
+from PixivSearch.settings import logger
+from PixivSearch.util import Util
+
+
+headers = {
+    'X-Requested-With': 'XMLHttpRequest',
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) '
+                  'Chrome/56.0.2924.87 Safari/537.36'
+}
+
+lock = threading.Lock()  # 多线程全局资源锁
+total = 1
+
+
+def get_cookies():
+    _cookies = {}
+    array = config.param.objects.get(param_name='pixiv_cookie').param_value.split(';')
+    for row in array:
+        k, v = row.strip().split('=', 1)
+        _cookies[k] = v
+    return _cookies
+
+
+def crawl(url):
+    global total
+    req = requests.get(url, headers=headers, cookies=get_cookies()).text
+    tag = BeautifulSoup(req, 'lxml').select('#js-mount-point-search-result-list')[0].attrs['data-items']
+    imageNodes = json.loads(tag)
+    for imageNode in imageNodes:
+        with lock:
+            nodes.append(imageNode)
+
+
+def get_urls(search, page):
+    fmt = 'https://www.pixiv.net/search.php?word={}&order=date_d&p={}'
+    return [fmt.format(search, p) for p in range(1, page)]
+
+
+def get_Img(params):
+    params[1]['imgUrl'] = 'https://i.pximg.net/img-original/img/' + params[1]['url'][-46:-15]
+
+    headers['referer'] = 'https://www.pixiv.net/member_illust.php?mode=medium&illust_id=' + params[1]['illustId']
+
+    suffix = ".jpg"
+    logger.info('开始下载图片：%s%s' % (params[1]['imgUrl'], suffix))
+
+    s = requests.get(params[1]['imgUrl'] + suffix, headers=headers, cookies=get_cookies())
+    if (s.status_code == 404):
+        suffix = '.png'
+        s = requests.get(params[1]['imgUrl'] + suffix, headers=headers, cookies=get_cookies())
+    if (s.status_code == 404):
+        logger.error('无法下载图片：%s' % (params[1]['illustTitle']))
+        return
+
+    logger.info('下载图片："%s"到%s' % (
+    params[1]['illustTitle'], os.getcwd().replace('\\', '/') + '/' + imgPath + params[1]['illustId'] + suffix))
+    f = open(imgPath + params[1]['illustId'] + suffix, 'wb')  # 写入多媒体文件要 b 这个参数
+    f.write(s.content)  # 多媒体文件要是用conctent
+    f.close()
+
+    params[1]['localName'] = params[1]['illustId'] + suffix
+    logger.info('排行第%d名，收藏数%d，标题：%s，标签：%s，(%s)前投稿，链接：%s' % (
+    params[0], params[1]['bookmarkCount'], params[1]['illustTitle'], ','.join(params[1]['tags']), '',
+    params[1]['imgUrl']))
+
+fsize = ''
+
+
+def get_nodes(param):
+    global nodes, fsize, imgPath
+    nodes = []
+    start = datetime.now()
+    urls = get_urls(param[1], int(param[2]) + 1)
+    logger.info('开始从P站获取图片数据')
+    with futures.ThreadPoolExecutor(32) as executor:
+        executor.map(crawl, urls)
+
+    # for url in urls:
+    #     crawl(url)
+
+    length = len(nodes)
+    logger.info('获取到%d张图片' % (length))
+    logger.info('对图片收藏数进行排序')
+    nodes = sorted(nodes, key=lambda v: v.get('bookmarkCount'), reverse=True)[:int(param[3])]  # 按star数降序排序
+    if (param[4] != None and param[4] == 'img'):
+        imgPath = root.getConfigValue('imgPath')
+        if not os.path.exists(imgPath):
+            os.makedirs(imgPath)
+        for file in os.listdir(imgPath):
+            os.remove(imgPath + file)
+        nodes_tup = []
+        start_d = datetime.now()
+        for index, img in enumerate(nodes):
+            nodes_tup.append((index + 1, img))
+            # get_Img((index+1,img))
+
+        with futures.ThreadPoolExecutor(32) as executor:
+            executor.map(get_Img, nodes_tup)
+        print('下载图片花费时间：%s' % (datetime.now() - start_d))
+        logger.info('%s张图片下载完毕' % (len(os.listdir(imgPath))))
+
+        baseZipPath=root.getConfigValue('baseZipPath')
+        if not os.path.exists(baseZipPath):
+            os.makedirs(baseZipPath)
+        zipPath =  baseZipPath+ param[1] + '.zip'
+        logger.info('图片打包到：%s' % (zipPath))
+        Util.zip(imgPath, zipPath)
+        fsize = str(round(os.path.getsize(zipPath) / float(1024 * 1024), 2)) + 'MB'
+        logger.info('图包大小：%s' % (fsize))
+
+    tip = '从%d张图片中筛选出收藏数前%s的图片,处理耗时：%s' % (length, param[3], datetime.now() - start)
+    logger.info(tip)
+    return [nodes, tip, fsize]
+
+
+if __name__ == "__main__":
+    if (len(sys.argv)) == 5 and sys.argv[2].isdigit() and sys.argv[3].isdigit():
+        try:
+            get_nodes(sys.argv)
+        except BaseException as e:
+            repr(e)
+
+    else:
+        logger.error('参数不合法')

PixivSearch/settings.py

@@ -0,0 +1,141 @@
+"""
+Django settings for PixivSearch project.
+
+Generated by 'django-admin startproject' using Django 1.11.7.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/1.11/topics/settings/
+
+For the full list of settings and their values, see
+https://docs.djangoproject.com/en/1.11/ref/settings/
+"""
+import logging.config
+import os
+
+# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+# Quick-start development settings - unsuitable for production
+# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = 'dh3^+=iugoo*+p_ea4u3dh&b!_zlgs8*m9kc+#*f2eozglsqjh'
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = True
+
+ALLOWED_HOSTS = ['*']
+
+
+# Application definition
+
+INSTALLED_APPS = [
+    'django.contrib.admin',
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.messages',
+    'django.contrib.staticfiles',
+    'PixivSearch.model'
+]
+
+MIDDLEWARE = [
+    'django.middleware.security.SecurityMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+]
+
+ROOT_URLCONF = 'PixivSearch.urls'
+
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': [os.path.join(os.path.dirname(__file__), 'templates').replace('\\','/')],
+        'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                'django.template.context_processors.debug',
+                'django.template.context_processors.request',
+                'django.contrib.auth.context_processors.auth',
+                'django.contrib.messages.context_processors.messages',
+            ],
+        },
+    },
+]
+
+WSGI_APPLICATION = 'PixivSearch.wsgi.application'
+
+
+# Database
+# https://docs.djangoproject.com/en/1.11/ref/settings/#databases
+
+# DATABASES = {
+#     'default': {
+#         'ENGINE': 'django.db.backends.mysql',
+#         'NAME': 'bangumi',
+#         'USER': 'root',
+#         'PASSWORD': '',
+#         'HOST': '184.170.212.72',
+#         # 'HOST': 'sukura.top',
+#         'PORT': '3306',
+#     }
+# }
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': 'mydatabase',
+    }
+}
+
+
+# Password validation
+# https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators
+
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
+    },
+]
+
+
+# Internationalization
+# https://docs.djangoproject.com/en/1.11/topics/i18n/
+
+LANGUAGE_CODE = 'en-us'
+
+TIME_ZONE = 'Asia/Shanghai'
+
+USE_I18N = True
+
+USE_L10N = True
+
+USE_TZ = False
+
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/1.11/howto/static-files/
+
+STATIC_URL = '/static/'
+STATICFILES_DIRS = [
+    os.path.join(os.path.dirname(__file__), 'static').replace('\\', '/'),
+]
+
+configPath = '%s/logging.conf' % os.path.dirname(__file__).replace('\\', '/')
+logging.config.fileConfig(configPath)
+logger = logging.getLogger('file')
+

PixivSearch/templates/addConfig.html

@@ -0,0 +1,27 @@
+<html xmlns="http://www.w3.org/1999/html">
+<head>
+    <meta charset="UTF-8">
+    <title>参数管理</title>
+    {% load staticfiles %}
+    <script src="{% static "js/jquery-3.2.1.min.js"%}"></script>
+    <script>
+        $(function () {
+            $("button").click(function(){
+                $("form").attr("method",$(this).attr("id")).submit();
+            });
+
+        })
+    </script>
+</head>
+<body>
+<form action="/tsdm">
+    {% csrf_token %}
+    <label>参数名<input name="param_name" value="{{param_name}}"/></label>
+    <label>参数值<input name="param_value" value="{{param_value}}"/></label>
+    <div>
+        <button id="POST">设置</button>
+        <button id="GET">获取</button>
+    </div>
+</form>
+</body>
+</html>

PixivSearch/templates/index.html

@@ -0,0 +1,107 @@
+<html xmlns="http://www.w3.org/1999/html">
+<head>
+    <style>
+        div {
+            text-align: center;
+        }
+
+        ol {
+            list-style-type: demical;
+        }
+
+        ol li {
+            list-style-position: outside;
+        }
+    </style>
+    {% load staticfiles %}
+    <script src="{% static "js/jquery-3.2.1.min.js" %}"></script>
+    <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css"
+          integrity="sha384-MCw98/SFnGE8fJT3GXwEOngsV7Zt27NXFoaoApmYm81iuXoPkFOJwJ8ERdknLPMO" crossorigin="anonymous">
+    <script src="https://code.jquery.com/jquery-3.3.1.slim.min.js"
+            integrity="sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo"
+            crossorigin="anonymous"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.3/umd/popper.min.js"
+            integrity="sha384-ZMP7rVo3mIykV+2+9J3UJ46jBk0WLaUAdn689aCwoqbBJiSnjAK/l8WvCWPIPm49"
+            crossorigin="anonymous"></script>
+    <script src="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js"
+            integrity="sha384-ChfqqxuZUCnJSK3+MXmPNIyE6ZbWh2IMqE241rYiqJxyMiZ6OW/JmZQ5stwEULTy"
+            crossorigin="anonymous"></script>
+    <script>
+
+        function check() {
+            $("[name=pageSize],[name=order]").each(function () {
+                if ($(this).val() != '' && $(this).val() <= 0) {
+                    $(this).val('');
+                    alert($(this).parent().text() + '不能小于等于0');
+                    $(this).focus();
+                }
+            });
+        }
+    </script>
+</head>
+<body>
+
+
+<form action="/pixiv/search" method="post" onsubmit="check()">
+    {% csrf_token %}
+
+    <div class="form-inline">
+        <div class="form-group">
+            <label for="word">关键字</label>
+            <input type="text" class="form-control" id="word" name="word" value="{{ word }}" placeholder="R-18">
+        </div>
+
+        <div class="form-group">
+            <label for="word">页数</label>
+            <input type="number" class="form-control" id="pageSize" name="pageSize" value="{{ pageSize }}" placeholder="10">
+        </div>
+
+        <div class="form-group">
+            <label for="word">统计数</label>
+            <input type="number" class="form-control" id="order" name="order" value="{{ order }}" placeholder="10">
+        </div>
+
+        <div class="custom-control custom-radio custom-control-inline">
+            <input type="radio" id="customRadio1" name="type" value="text" class="custom-control-input" {% if type == 'text' or type is None %}checked{% endif %}>
+            <label class="custom-control-label" for="customRadio1">不显示大图</label>
+        </div>
+        <div class="custom-control custom-radio custom-control-inline">
+            <input type="radio" id="customRadio2" name="type" value="img" class="custom-control-input" {% ifequal type 'img' %}checked{% endifequal %}>
+            <label class="custom-control-label" for="customRadio2">显示大图</label>
+        </div>
+    </div>
+
+    <div class="form-inline">
+        <button class="btn btn-primary offset-md-3" id="button" type="submit">查询</button>
+
+        {% if  download %}
+            <a href="{% static "/download/" %}{{ download }}">图包提取(文件大小:{{ size }})</a>
+        {% endif %}
+    </div>
+
+
+</form>
+
+<div>
+    <h1>{{ msg }}</h1>
+</div>
+<ol>
+    {% for imageNode in imageNodes %}
+        <li>
+            <div>
+                <h1>标题：<a href="https://www.pixiv.net/member_illust.php?mode=medium&illust_id={{ imageNode.illustId }}"
+                          target="_blank">{{ imageNode.illustTitle }}</a>，画师：<a
+                        href="https://www.pixiv.net/member.php?id={{ imageNode.userId }}">{{ imageNode.userName }}</a>，收藏数：{{ imageNode.bookmarkCount }}
+                </h1>
+                {% if imageNode.localName %}
+                    <img src="{% static "images/" %}{{ imageNode.localName }}">
+                {% endif %}
+            </div>
+        </li>
+    {% empty %}
+        <p>{{ tip }}</p>
+    {% endfor %}
+
+</ol>
+</body>
+</html>

PixivSearch/thrift/QueryComment/Client.py

@@ -0,0 +1,23 @@
+from thrift.protocol import TBinaryProtocol
+from thrift.transport import TSocket, TTransport
+
+from PixivSearch.thrift.QueryComment.QueryComment import Client
+
+if __name__ == '__main__':
+
+    socket = TSocket.TSocket('sukura.top', 2233)
+
+    # Buffering is critical. Raw sockets are very slow
+    transport = TTransport.TFramedTransport(socket)
+
+    if not transport.isOpen():
+        transport.open()
+
+    # Wrap in a protocol
+    protocol = TBinaryProtocol.TBinaryProtocol(transport)
+
+    # Create a client to use the protocol encoder
+    client = Client(protocol)
+
+    cids = [7636499, 7636501, 7636500, 7636503, 7636504, 7636502, 7636509, 7636508, 7636506, 7636507, 7636505]
+    print(client.commentSumMap(cids))

PixivSearch/thrift/QueryComment/QueryComment-remote

@@ -0,0 +1,131 @@
+#!/usr/bin/env python
+#
+# Autogenerated by Thrift Compiler (0.11.0)
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+#  options string: py
+#
+
+import sys
+import pprint
+if sys.version_info[0] > 2:
+    from urllib.parse import urlparse
+else:
+    from urlparse import urlparse
+from thrift.transport import TTransport, TSocket, TSSLSocket, THttpClient
+from thrift.protocol.TBinaryProtocol import TBinaryProtocol
+
+from QueryComment import QueryComment
+from QueryComment.ttypes import *
+
+if len(sys.argv) <= 1 or sys.argv[1] == '--help':
+    print('')
+    print('Usage: ' + sys.argv[0] + ' [-h host[:port]] [-u url] [-f[ramed]] [-s[sl]] [-novalidate] [-ca_certs certs] [-keyfile keyfile] [-certfile certfile] function [arg1 [arg2...]]')
+    print('')
+    print('Functions:')
+    print('  i32 commentSum(i32 cid)')
+    print('   commentSumList( cids)')
+    print('  string download( cids, string fileName)')
+    print('')
+    sys.exit(0)
+
+pp = pprint.PrettyPrinter(indent=2)
+host = 'localhost'
+port = 9090
+uri = ''
+framed = False
+ssl = False
+validate = True
+ca_certs = None
+keyfile = None
+certfile = None
+http = False
+argi = 1
+
+if sys.argv[argi] == '-h':
+    parts = sys.argv[argi + 1].split(':')
+    host = parts[0]
+    if len(parts) > 1:
+        port = int(parts[1])
+    argi += 2
+
+if sys.argv[argi] == '-u':
+    url = urlparse(sys.argv[argi + 1])
+    parts = url[1].split(':')
+    host = parts[0]
+    if len(parts) > 1:
+        port = int(parts[1])
+    else:
+        port = 80
+    uri = url[2]
+    if url[4]:
+        uri += '?%s' % url[4]
+    http = True
+    argi += 2
+
+if sys.argv[argi] == '-f' or sys.argv[argi] == '-framed':
+    framed = True
+    argi += 1
+
+if sys.argv[argi] == '-s' or sys.argv[argi] == '-ssl':
+    ssl = True
+    argi += 1
+
+if sys.argv[argi] == '-novalidate':
+    validate = False
+    argi += 1
+
+if sys.argv[argi] == '-ca_certs':
+    ca_certs = sys.argv[argi+1]
+    argi += 2
+
+if sys.argv[argi] == '-keyfile':
+    keyfile = sys.argv[argi+1]
+    argi += 2
+
+if sys.argv[argi] == '-certfile':
+    certfile = sys.argv[argi+1]
+    argi += 2
+
+cmd = sys.argv[argi]
+args = sys.argv[argi + 1:]
+
+if http:
+    transport = THttpClient.THttpClient(host, port, uri)
+else:
+    if ssl:
+        socket = TSSLSocket.TSSLSocket(host, port, validate=validate, ca_certs=ca_certs, keyfile=keyfile, certfile=certfile)
+    else:
+        socket = TSocket.TSocket(host, port)
+    if framed:
+        transport = TTransport.TFramedTransport(socket)
+    else:
+        transport = TTransport.TBufferedTransport(socket)
+protocol = TBinaryProtocol(transport)
+client = QueryComment.Client(protocol)
+transport.open()
+
+if cmd == 'commentSum':
+    if len(args) != 1:
+        print('commentSum requires 1 args')
+        sys.exit(1)
+    pp.pprint(client.commentSum(eval(args[0]),))
+
+elif cmd == 'commentSumList':
+    if len(args) != 1:
+        print('commentSumList requires 1 args')
+        sys.exit(1)
+    pp.pprint(client.commentSumList(eval(args[0]),))
+
+elif cmd == 'download':
+    if len(args) != 2:
+        print('download requires 2 args')
+        sys.exit(1)
+    pp.pprint(client.download(eval(args[0]), args[1],))
+
+else:
+    print('Unrecognized method %s' % cmd)
+    sys.exit(1)
+
+transport.close()

PixivSearch/thrift/QueryComment/QueryComment.py

@@ -0,0 +1,660 @@
+#
+# Autogenerated by Thrift Compiler (0.11.0)
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+#  options string: py
+#
+
+from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException
+from thrift.protocol.TProtocol import TProtocolException
+from thrift.TRecursive import fix_spec
+
+import sys
+import logging
+from .ttypes import *
+from thrift.Thrift import TProcessor
+from thrift.transport import TTransport
+all_structs = []
+
+
+class Iface(object):
+    def commentSum(self, cid):
+        """
+        查询单个cid弹幕数
+
+
+        Parameters:
+         - cid
+        """
+        pass
+
+    def commentSumList(self, cids):
+        """
+        * 批量查询多个cid弹幕数
+         *
+
+        Parameters:
+         - cids
+        """
+        pass
+
+    def download(self, cids, fileName):
+        """
+          * 批量下载弹幕
+        *
+
+        Parameters:
+         - cids
+         - fileName
+        """
+        pass
+
+
+class Client(Iface):
+    def __init__(self, iprot, oprot=None):
+        self._iprot = self._oprot = iprot
+        if oprot is not None:
+            self._oprot = oprot
+        self._seqid = 0
+
+    def commentSum(self, cid):
+        """
+        查询单个cid弹幕数
+
+
+        Parameters:
+         - cid
+        """
+        self.send_commentSum(cid)
+        return self.recv_commentSum()
+
+    def send_commentSum(self, cid):
+        self._oprot.writeMessageBegin('commentSum', TMessageType.CALL, self._seqid)
+        args = commentSum_args()
+        args.cid = cid
+        args.write(self._oprot)
+        self._oprot.writeMessageEnd()
+        self._oprot.trans.flush()
+
+    def recv_commentSum(self):
+        iprot = self._iprot
+        (fname, mtype, rseqid) = iprot.readMessageBegin()
+        if mtype == TMessageType.EXCEPTION:
+            x = TApplicationException()
+            x.read(iprot)
+            iprot.readMessageEnd()
+            raise x
+        result = commentSum_result()
+        result.read(iprot)
+        iprot.readMessageEnd()
+        if result.success is not None:
+            return result.success
+        raise TApplicationException(TApplicationException.MISSING_RESULT, "commentSum failed: unknown result")
+
+    def commentSumList(self, cids):
+        """
+        * 批量查询多个cid弹幕数
+         *
+
+        Parameters:
+         - cids
+        """
+        self.send_commentSumList(cids)
+        return self.recv_commentSumList()
+
+    def send_commentSumList(self, cids):
+        self._oprot.writeMessageBegin('commentSumList', TMessageType.CALL, self._seqid)
+        args = commentSumList_args()
+        args.cids = cids
+        args.write(self._oprot)
+        self._oprot.writeMessageEnd()
+        self._oprot.trans.flush()
+
+    def recv_commentSumList(self):
+        iprot = self._iprot
+        (fname, mtype, rseqid) = iprot.readMessageBegin()
+        if mtype == TMessageType.EXCEPTION:
+            x = TApplicationException()
+            x.read(iprot)
+            iprot.readMessageEnd()
+            raise x
+        result = commentSumList_result()
+        result.read(iprot)
+        iprot.readMessageEnd()
+        if result.success is not None:
+            return result.success
+        raise TApplicationException(TApplicationException.MISSING_RESULT, "commentSumList failed: unknown result")
+
+    def download(self, cids, fileName):
+        """
+          * 批量下载弹幕
+        *
+
+        Parameters:
+         - cids
+         - fileName
+        """
+        self.send_download(cids, fileName)
+        return self.recv_download()
+
+    def send_download(self, cids, fileName):
+        self._oprot.writeMessageBegin('download', TMessageType.CALL, self._seqid)
+        args = download_args()
+        args.cids = cids
+        args.fileName = fileName
+        args.write(self._oprot)
+        self._oprot.writeMessageEnd()
+        self._oprot.trans.flush()
+
+    def recv_download(self):
+        iprot = self._iprot
+        (fname, mtype, rseqid) = iprot.readMessageBegin()
+        if mtype == TMessageType.EXCEPTION:
+            x = TApplicationException()
+            x.read(iprot)
+            iprot.readMessageEnd()
+            raise x
+        result = download_result()
+        result.read(iprot)
+        iprot.readMessageEnd()
+        if result.success is not None:
+            return result.success
+        raise TApplicationException(TApplicationException.MISSING_RESULT, "download failed: unknown result")
+
+
+class Processor(Iface, TProcessor):
+    def __init__(self, handler):
+        self._handler = handler
+        self._processMap = {}
+        self._processMap["commentSum"] = Processor.process_commentSum
+        self._processMap["commentSumList"] = Processor.process_commentSumList
+        self._processMap["download"] = Processor.process_download
+
+    def process(self, iprot, oprot):
+        (name, type, seqid) = iprot.readMessageBegin()
+        if name not in self._processMap:
+            iprot.skip(TType.STRUCT)
+            iprot.readMessageEnd()
+            x = TApplicationException(TApplicationException.UNKNOWN_METHOD, 'Unknown function %s' % (name))
+            oprot.writeMessageBegin(name, TMessageType.EXCEPTION, seqid)
+            x.write(oprot)
+            oprot.writeMessageEnd()
+            oprot.trans.flush()
+            return
+        else:
+            self._processMap[name](self, seqid, iprot, oprot)
+        return True
+
+    def process_commentSum(self, seqid, iprot, oprot):
+        args = commentSum_args()
+        args.read(iprot)
+        iprot.readMessageEnd()
+        result = commentSum_result()
+        try:
+            result.success = self._handler.commentSum(args.cid)
+            msg_type = TMessageType.REPLY
+        except TTransport.TTransportException:
+            raise
+        except TApplicationException as ex:
+            logging.exception('TApplication exception in handler')
+            msg_type = TMessageType.EXCEPTION
+            result = ex
+        except Exception:
+            logging.exception('Unexpected exception in handler')
+            msg_type = TMessageType.EXCEPTION
+            result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error')
+        oprot.writeMessageBegin("commentSum", msg_type, seqid)
+        result.write(oprot)
+        oprot.writeMessageEnd()
+        oprot.trans.flush()
+
+    def process_commentSumList(self, seqid, iprot, oprot):
+        args = commentSumList_args()
+        args.read(iprot)
+        iprot.readMessageEnd()
+        result = commentSumList_result()
+        try:
+            result.success = self._handler.commentSumList(args.cids)
+            msg_type = TMessageType.REPLY
+        except TTransport.TTransportException:
+            raise
+        except TApplicationException as ex:
+            logging.exception('TApplication exception in handler')
+            msg_type = TMessageType.EXCEPTION
+            result = ex
+        except Exception:
+            logging.exception('Unexpected exception in handler')
+            msg_type = TMessageType.EXCEPTION
+            result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error')
+        oprot.writeMessageBegin("commentSumList", msg_type, seqid)
+        result.write(oprot)
+        oprot.writeMessageEnd()
+        oprot.trans.flush()
+
+    def process_download(self, seqid, iprot, oprot):
+        args = download_args()
+        args.read(iprot)
+        iprot.readMessageEnd()
+        result = download_result()
+        try:
+            result.success = self._handler.download(args.cids, args.fileName)
+            msg_type = TMessageType.REPLY
+        except TTransport.TTransportException:
+            raise
+        except TApplicationException as ex:
+            logging.exception('TApplication exception in handler')
+            msg_type = TMessageType.EXCEPTION
+            result = ex
+        except Exception:
+            logging.exception('Unexpected exception in handler')
+            msg_type = TMessageType.EXCEPTION
+            result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error')
+        oprot.writeMessageBegin("download", msg_type, seqid)
+        result.write(oprot)
+        oprot.writeMessageEnd()
+        oprot.trans.flush()
+
+# HELPER FUNCTIONS AND STRUCTURES
+
+
+class commentSum_args(object):
+    """
+    Attributes:
+     - cid
+    """
+
+
+    def __init__(self, cid=None,):
+        self.cid = cid
+
+    def read(self, iprot):
+        if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
+            iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
+            return
+        iprot.readStructBegin()
+        while True:
+            (fname, ftype, fid) = iprot.readFieldBegin()
+            if ftype == TType.STOP:
+                break
+            if fid == 1:
+                if ftype == TType.I32:
+                    self.cid = iprot.readI32()
+                else:
+                    iprot.skip(ftype)
+            else:
+                iprot.skip(ftype)
+            iprot.readFieldEnd()
+        iprot.readStructEnd()
+
+    def write(self, oprot):
+        if oprot._fast_encode is not None and self.thrift_spec is not None:
+            oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
+            return
+        oprot.writeStructBegin('commentSum_args')
+        if self.cid is not None:
+            oprot.writeFieldBegin('cid', TType.I32, 1)
+            oprot.writeI32(self.cid)
+            oprot.writeFieldEnd()
+        oprot.writeFieldStop()
+        oprot.writeStructEnd()
+
+    def validate(self):
+        return
+
+    def __repr__(self):
+        L = ['%s=%r' % (key, value)
+             for key, value in self.__dict__.items()]
+        return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+    def __eq__(self, other):
+        return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+    def __ne__(self, other):
+        return not (self == other)
+all_structs.append(commentSum_args)
+commentSum_args.thrift_spec = (
+    None,  # 0
+    (1, TType.I32, 'cid', None, None, ),  # 1
+)
+
+
+class commentSum_result(object):
+    """
+    Attributes:
+     - success
+    """
+
+
+    def __init__(self, success=None,):
+        self.success = success
+
+    def read(self, iprot):
+        if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
+            iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
+            return
+        iprot.readStructBegin()
+        while True:
+            (fname, ftype, fid) = iprot.readFieldBegin()
+            if ftype == TType.STOP:
+                break
+            if fid == 0:
+                if ftype == TType.I32:
+                    self.success = iprot.readI32()
+                else:
+                    iprot.skip(ftype)
+            else:
+                iprot.skip(ftype)
+            iprot.readFieldEnd()
+        iprot.readStructEnd()
+
+    def write(self, oprot):
+        if oprot._fast_encode is not None and self.thrift_spec is not None:
+            oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
+            return
+        oprot.writeStructBegin('commentSum_result')
+        if self.success is not None:
+            oprot.writeFieldBegin('success', TType.I32, 0)
+            oprot.writeI32(self.success)
+            oprot.writeFieldEnd()
+        oprot.writeFieldStop()
+        oprot.writeStructEnd()
+
+    def validate(self):
+        return
+
+    def __repr__(self):
+        L = ['%s=%r' % (key, value)
+             for key, value in self.__dict__.items()]
+        return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+    def __eq__(self, other):
+        return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+    def __ne__(self, other):
+        return not (self == other)
+all_structs.append(commentSum_result)
+commentSum_result.thrift_spec = (
+    (0, TType.I32, 'success', None, None, ),  # 0
+)
+
+
+class commentSumList_args(object):
+    """
+    Attributes:
+     - cids
+    """
+
+
+    def __init__(self, cids=None,):
+        self.cids = cids
+
+    def read(self, iprot):
+        if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
+            iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
+            return
+        iprot.readStructBegin()
+        while True:
+            (fname, ftype, fid) = iprot.readFieldBegin()
+            if ftype == TType.STOP:
+                break
+            if fid == 1:
+                if ftype == TType.LIST:
+                    self.cids = []
+                    (_etype3, _size0) = iprot.readListBegin()
+                    for _i4 in range(_size0):
+                        _elem5 = iprot.readI32()
+                        self.cids.append(_elem5)
+                    iprot.readListEnd()
+                else:
+                    iprot.skip(ftype)
+            else:
+                iprot.skip(ftype)
+            iprot.readFieldEnd()
+        iprot.readStructEnd()
+
+    def write(self, oprot):
+        if oprot._fast_encode is not None and self.thrift_spec is not None:
+            oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
+            return
+        oprot.writeStructBegin('commentSumList_args')
+        if self.cids is not None:
+            oprot.writeFieldBegin('cids', TType.LIST, 1)
+            oprot.writeListBegin(TType.I32, len(self.cids))
+            for iter6 in self.cids:
+                oprot.writeI32(iter6)
+            oprot.writeListEnd()
+            oprot.writeFieldEnd()
+        oprot.writeFieldStop()
+        oprot.writeStructEnd()
+
+    def validate(self):
+        return
+
+    def __repr__(self):
+        L = ['%s=%r' % (key, value)
+             for key, value in self.__dict__.items()]
+        return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+    def __eq__(self, other):
+        return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+    def __ne__(self, other):
+        return not (self == other)
+all_structs.append(commentSumList_args)
+commentSumList_args.thrift_spec = (
+    None,  # 0
+    (1, TType.LIST, 'cids', (TType.I32, None, False), None, ),  # 1
+)
+
+
+class commentSumList_result(object):
+    """
+    Attributes:
+     - success
+    """
+
+
+    def __init__(self, success=None,):
+        self.success = success
+
+    def read(self, iprot):
+        if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
+            iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
+            return
+        iprot.readStructBegin()
+        while True:
+            (fname, ftype, fid) = iprot.readFieldBegin()
+            if ftype == TType.STOP:
+                break
+            if fid == 0:
+                if ftype == TType.LIST:
+                    self.success = []
+                    (_etype10, _size7) = iprot.readListBegin()
+                    for _i11 in range(_size7):
+                        _elem12 = iprot.readI32()
+                        self.success.append(_elem12)
+                    iprot.readListEnd()
+                else:
+                    iprot.skip(ftype)
+            else:
+                iprot.skip(ftype)
+            iprot.readFieldEnd()
+        iprot.readStructEnd()
+
+    def write(self, oprot):
+        if oprot._fast_encode is not None and self.thrift_spec is not None:
+            oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
+            return
+        oprot.writeStructBegin('commentSumList_result')
+        if self.success is not None:
+            oprot.writeFieldBegin('success', TType.LIST, 0)
+            oprot.writeListBegin(TType.I32, len(self.success))
+            for iter13 in self.success:
+                oprot.writeI32(iter13)
+            oprot.writeListEnd()
+            oprot.writeFieldEnd()
+        oprot.writeFieldStop()
+        oprot.writeStructEnd()
+
+    def validate(self):
+        return
+
+    def __repr__(self):
+        L = ['%s=%r' % (key, value)
+             for key, value in self.__dict__.items()]
+        return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+    def __eq__(self, other):
+        return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+    def __ne__(self, other):
+        return not (self == other)
+all_structs.append(commentSumList_result)
+commentSumList_result.thrift_spec = (
+    (0, TType.LIST, 'success', (TType.I32, None, False), None, ),  # 0
+)
+
+
+class download_args(object):
+    """
+    Attributes:
+     - cids
+     - fileName
+    """
+
+
+    def __init__(self, cids=None, fileName=None,):
+        self.cids = cids
+        self.fileName = fileName
+
+    def read(self, iprot):
+        if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
+            iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
+            return
+        iprot.readStructBegin()
+        while True:
+            (fname, ftype, fid) = iprot.readFieldBegin()
+            if ftype == TType.STOP:
+                break
+            if fid == 1:
+                if ftype == TType.LIST:
+                    self.cids = []
+                    (_etype17, _size14) = iprot.readListBegin()
+                    for _i18 in range(_size14):
+                        _elem19 = iprot.readI32()
+                        self.cids.append(_elem19)
+                    iprot.readListEnd()
+                else:
+                    iprot.skip(ftype)
+            elif fid == 2:
+                if ftype == TType.STRING:
+                    self.fileName = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString()
+                else:
+                    iprot.skip(ftype)
+            else:
+                iprot.skip(ftype)
+            iprot.readFieldEnd()
+        iprot.readStructEnd()
+
+    def write(self, oprot):
+        if oprot._fast_encode is not None and self.thrift_spec is not None:
+            oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
+            return
+        oprot.writeStructBegin('download_args')
+        if self.cids is not None:
+            oprot.writeFieldBegin('cids', TType.LIST, 1)
+            oprot.writeListBegin(TType.I32, len(self.cids))
+            for iter20 in self.cids:
+                oprot.writeI32(iter20)
+            oprot.writeListEnd()
+            oprot.writeFieldEnd()
+        if self.fileName is not None:
+            oprot.writeFieldBegin('fileName', TType.STRING, 2)
+            oprot.writeString(self.fileName.encode('utf-8') if sys.version_info[0] == 2 else self.fileName)
+            oprot.writeFieldEnd()
+        oprot.writeFieldStop()
+        oprot.writeStructEnd()
+
+    def validate(self):
+        return
+
+    def __repr__(self):
+        L = ['%s=%r' % (key, value)
+             for key, value in self.__dict__.items()]
+        return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+    def __eq__(self, other):
+        return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+    def __ne__(self, other):
+        return not (self == other)
+all_structs.append(download_args)
+download_args.thrift_spec = (
+    None,  # 0
+    (1, TType.LIST, 'cids', (TType.I32, None, False), None, ),  # 1
+    (2, TType.STRING, 'fileName', 'UTF8', None, ),  # 2
+)
+
+
+class download_result(object):
+    """
+    Attributes:
+     - success
+    """
+
+
+    def __init__(self, success=None,):
+        self.success = success
+
+    def read(self, iprot):
+        if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
+            iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
+            return
+        iprot.readStructBegin()
+        while True:
+            (fname, ftype, fid) = iprot.readFieldBegin()
+            if ftype == TType.STOP:
+                break
+            if fid == 0:
+                if ftype == TType.STRING:
+                    self.success = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString()
+                else:
+                    iprot.skip(ftype)
+            else:
+                iprot.skip(ftype)
+            iprot.readFieldEnd()
+        iprot.readStructEnd()
+
+    def write(self, oprot):
+        if oprot._fast_encode is not None and self.thrift_spec is not None:
+            oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
+            return
+        oprot.writeStructBegin('download_result')
+        if self.success is not None:
+            oprot.writeFieldBegin('success', TType.STRING, 0)
+            oprot.writeString(self.success.encode('utf-8') if sys.version_info[0] == 2 else self.success)
+            oprot.writeFieldEnd()
+        oprot.writeFieldStop()
+        oprot.writeStructEnd()
+
+    def validate(self):
+        return
+
+    def __repr__(self):
+        L = ['%s=%r' % (key, value)
+             for key, value in self.__dict__.items()]
+        return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+    def __eq__(self, other):
+        return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+    def __ne__(self, other):
+        return not (self == other)
+all_structs.append(download_result)
+download_result.thrift_spec = (
+    (0, TType.STRING, 'success', 'UTF8', None, ),  # 0
+)
+fix_spec(all_structs)
+del all_structs
+

PixivSearch/thrift/QueryComment/Server.py

@@ -0,0 +1,48 @@
+import os
+import sys
+
+from thrift.server.TNonblockingServer import TNonblockingServer
+from thrift.transport import TSocket
+
+sys.path.append('/root/PixivSearch')
+from PixivSearch.util import Util
+from PixivSearch.dao.Comment import xml, downloadXml
+from PixivSearch.thrift.QueryComment import QueryComment
+from PixivSearch.thrift.QueryComment.QueryComment import Iface
+
+commentPath = '/root/PixivSearch/PixivSearch/thrift/tmpFile/comment'
+#commentPath='/home/hua/下载/comment'
+
+class Server(Iface):
+    def commentSumList(self, cids):
+        result = []
+        for cid in cids:
+            comment_selector = xml('https://comment.bilibili.com/{cid}.xml'.format(cid=cid))
+            length = len(comment_selector.xpath('//i//d/text()'))
+            print('cid:{cid},弹幕数：{length}'.format(cid=cid, length=length))
+            result.append(length)
+        return result
+
+    def commentSum(self, cid):
+        return self.commentSumList([cid])[0];
+
+    def download(self, cids, fileName):
+        path = '{commentPath}/{fileName}'.format(commentPath=commentPath, fileName=fileName)
+        for cid in cids:
+            downloadXml(path, cid,
+                        histroy=False)
+        zipFile = '{commentPath}/{fileName}.zip'.format(commentPath=commentPath,fileName=fileName)
+        print(zipFile)
+        Util.zip(path, zipFile)
+        if os.path.isfile(zipFile):
+            print('压缩包成功生成到{zipFile}'.format(zipFile=zipFile))
+            return zipFile
+        else:
+            return None
+
+
+if __name__ == '__main__':
+    socket = TSocket.TServerSocket(port=2233)
+    processor = QueryComment.Processor(Server())
+    server = TNonblockingServer(processor, socket)
+    server.serve()

PixivSearch/thrift/QueryComment/constants.py

@@ -0,0 +1,14 @@
+#
+# Autogenerated by Thrift Compiler (0.11.0)
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+#  options string: py
+#
+
+from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException
+from thrift.protocol.TProtocol import TProtocolException
+from thrift.TRecursive import fix_spec
+
+import sys
+from .ttypes import *

PixivSearch/thrift/QueryComment/ttypes.py

@@ -0,0 +1,18 @@
+#
+# Autogenerated by Thrift Compiler (0.11.0)
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+#  options string: py
+#
+
+from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException
+from thrift.protocol.TProtocol import TProtocolException
+from thrift.TRecursive import fix_spec
+
+import sys
+
+from thrift.transport import TTransport
+all_structs = []
+fix_spec(all_structs)
+del all_structs

PixivSearch/thrift/task/Server.py

@@ -0,0 +1,22 @@
+import sys
+
+from thrift.server.TNonblockingServer import TNonblockingServer
+from thrift.transport import TSocket
+
+sys.path.append('/root/PixivSearch')
+from PixivSearch.thrift.task import TSDM
+from PixivSearch.thrift.task.TSDM import Iface
+
+
+class Server(Iface):
+    def qiandao(self):
+        return False
+
+    def word(self):
+        return True
+
+if __name__ == '__main__':
+    socket = TSocket.TServerSocket(port=2233)
+    processor = TSDM.Processor(Server())
+    server = TNonblockingServer(processor, socket)
+    server.serve()

PixivSearch/thrift/task/TSDM-remote

@@ -0,0 +1,124 @@
+#!/usr/bin/env python
+#
+# Autogenerated by Thrift Compiler (0.11.0)
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+#  options string: py
+#
+
+import sys
+import pprint
+if sys.version_info[0] > 2:
+    from urllib.parse import urlparse
+else:
+    from urlparse import urlparse
+from thrift.transport import TTransport, TSocket, TSSLSocket, THttpClient
+from thrift.protocol.TBinaryProtocol import TBinaryProtocol
+
+from core.thrift.task import TSDM
+from core.thrift.task.ttypes import *
+
+if len(sys.argv) <= 1 or sys.argv[1] == '--help':
+    print('')
+    print('Usage: ' + sys.argv[0] + ' [-h host[:port]] [-u url] [-f[ramed]] [-s[sl]] [-novalidate] [-ca_certs certs] [-keyfile keyfile] [-certfile certfile] function [arg1 [arg2...]]')
+    print('')
+    print('Functions:')
+    print('  bool qiandao()')
+    print('  bool word()')
+    print('')
+    sys.exit(0)
+
+pp = pprint.PrettyPrinter(indent=2)
+host = 'localhost'
+port = 9090
+uri = ''
+framed = False
+ssl = False
+validate = True
+ca_certs = None
+keyfile = None
+certfile = None
+http = False
+argi = 1
+
+if sys.argv[argi] == '-h':
+    parts = sys.argv[argi + 1].split(':')
+    host = parts[0]
+    if len(parts) > 1:
+        port = int(parts[1])
+    argi += 2
+
+if sys.argv[argi] == '-u':
+    url = urlparse(sys.argv[argi + 1])
+    parts = url[1].split(':')
+    host = parts[0]
+    if len(parts) > 1:
+        port = int(parts[1])
+    else:
+        port = 80
+    uri = url[2]
+    if url[4]:
+        uri += '?%s' % url[4]
+    http = True
+    argi += 2
+
+if sys.argv[argi] == '-f' or sys.argv[argi] == '-framed':
+    framed = True
+    argi += 1
+
+if sys.argv[argi] == '-s' or sys.argv[argi] == '-ssl':
+    ssl = True
+    argi += 1
+
+if sys.argv[argi] == '-novalidate':
+    validate = False
+    argi += 1
+
+if sys.argv[argi] == '-ca_certs':
+    ca_certs = sys.argv[argi+1]
+    argi += 2
+
+if sys.argv[argi] == '-keyfile':
+    keyfile = sys.argv[argi+1]
+    argi += 2
+
+if sys.argv[argi] == '-certfile':
+    certfile = sys.argv[argi+1]
+    argi += 2
+
+cmd = sys.argv[argi]
+args = sys.argv[argi + 1:]
+
+if http:
+    transport = THttpClient.THttpClient(host, port, uri)
+else:
+    if ssl:
+        socket = TSSLSocket.TSSLSocket(host, port, validate=validate, ca_certs=ca_certs, keyfile=keyfile, certfile=certfile)
+    else:
+        socket = TSocket.TSocket(host, port)
+    if framed:
+        transport = TTransport.TFramedTransport(socket)
+    else:
+        transport = TTransport.TBufferedTransport(socket)
+protocol = TBinaryProtocol(transport)
+client = TSDM.Client(protocol)
+transport.open()
+
+if cmd == 'qiandao':
+    if len(args) != 0:
+        print('qiandao requires 0 args')
+        sys.exit(1)
+    pp.pprint(client.qiandao())
+
+elif cmd == 'word':
+    if len(args) != 0:
+        print('word requires 0 args')
+        sys.exit(1)
+    pp.pprint(client.word())
+
+else:
+    print('Unrecognized method %s' % cmd)
+    sys.exit(1)
+
+transport.close()

PixivSearch/thrift/task/TSDM.py

@@ -0,0 +1,366 @@
+#
+# Autogenerated by Thrift Compiler (0.11.0)
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+#  options string: py
+#
+
+from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException
+from thrift.protocol.TProtocol import TProtocolException
+from thrift.TRecursive import fix_spec
+
+import sys
+import logging
+from .ttypes import *
+from thrift.Thrift import TProcessor
+from thrift.transport import TTransport
+all_structs = []
+
+
+class Iface(object):
+    def qiandao(self):
+        pass
+
+    def word(self):
+        pass
+
+
+class Client(Iface):
+    def __init__(self, iprot, oprot=None):
+        self._iprot = self._oprot = iprot
+        if oprot is not None:
+            self._oprot = oprot
+        self._seqid = 0
+
+    def qiandao(self):
+        self.send_qiandao()
+        return self.recv_qiandao()
+
+    def send_qiandao(self):
+        self._oprot.writeMessageBegin('qiandao', TMessageType.CALL, self._seqid)
+        args = qiandao_args()
+        args.write(self._oprot)
+        self._oprot.writeMessageEnd()
+        self._oprot.trans.flush()
+
+    def recv_qiandao(self):
+        iprot = self._iprot
+        (fname, mtype, rseqid) = iprot.readMessageBegin()
+        if mtype == TMessageType.EXCEPTION:
+            x = TApplicationException()
+            x.read(iprot)
+            iprot.readMessageEnd()
+            raise x
+        result = qiandao_result()
+        result.read(iprot)
+        iprot.readMessageEnd()
+        if result.success is not None:
+            return result.success
+        raise TApplicationException(TApplicationException.MISSING_RESULT, "qiandao failed: unknown result")
+
+    def word(self):
+        self.send_word()
+        return self.recv_word()
+
+    def send_word(self):
+        self._oprot.writeMessageBegin('word', TMessageType.CALL, self._seqid)
+        args = word_args()
+        args.write(self._oprot)
+        self._oprot.writeMessageEnd()
+        self._oprot.trans.flush()
+
+    def recv_word(self):
+        iprot = self._iprot
+        (fname, mtype, rseqid) = iprot.readMessageBegin()
+        if mtype == TMessageType.EXCEPTION:
+            x = TApplicationException()
+            x.read(iprot)
+            iprot.readMessageEnd()
+            raise x
+        result = word_result()
+        result.read(iprot)
+        iprot.readMessageEnd()
+        if result.success is not None:
+            return result.success
+        raise TApplicationException(TApplicationException.MISSING_RESULT, "word failed: unknown result")
+
+
+class Processor(Iface, TProcessor):
+    def __init__(self, handler):
+        self._handler = handler
+        self._processMap = {}
+        self._processMap["qiandao"] = Processor.process_qiandao
+        self._processMap["word"] = Processor.process_word
+
+    def process(self, iprot, oprot):
+        (name, type, seqid) = iprot.readMessageBegin()
+        if name not in self._processMap:
+            iprot.skip(TType.STRUCT)
+            iprot.readMessageEnd()
+            x = TApplicationException(TApplicationException.UNKNOWN_METHOD, 'Unknown function %s' % (name))
+            oprot.writeMessageBegin(name, TMessageType.EXCEPTION, seqid)
+            x.write(oprot)
+            oprot.writeMessageEnd()
+            oprot.trans.flush()
+            return
+        else:
+            self._processMap[name](self, seqid, iprot, oprot)
+        return True
+
+    def process_qiandao(self, seqid, iprot, oprot):
+        args = qiandao_args()
+        args.read(iprot)
+        iprot.readMessageEnd()
+        result = qiandao_result()
+        try:
+            result.success = self._handler.qiandao()
+            msg_type = TMessageType.REPLY
+        except TTransport.TTransportException:
+            raise
+        except TApplicationException as ex:
+            logging.exception('TApplication exception in handler')
+            msg_type = TMessageType.EXCEPTION
+            result = ex
+        except Exception:
+            logging.exception('Unexpected exception in handler')
+            msg_type = TMessageType.EXCEPTION
+            result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error')
+        oprot.writeMessageBegin("qiandao", msg_type, seqid)
+        result.write(oprot)
+        oprot.writeMessageEnd()
+        oprot.trans.flush()
+
+    def process_word(self, seqid, iprot, oprot):
+        args = word_args()
+        args.read(iprot)
+        iprot.readMessageEnd()
+        result = word_result()
+        try:
+            result.success = self._handler.word()
+            msg_type = TMessageType.REPLY
+        except TTransport.TTransportException:
+            raise
+        except TApplicationException as ex:
+            logging.exception('TApplication exception in handler')
+            msg_type = TMessageType.EXCEPTION
+            result = ex
+        except Exception:
+            logging.exception('Unexpected exception in handler')
+            msg_type = TMessageType.EXCEPTION
+            result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error')
+        oprot.writeMessageBegin("word", msg_type, seqid)
+        result.write(oprot)
+        oprot.writeMessageEnd()
+        oprot.trans.flush()
+
+# HELPER FUNCTIONS AND STRUCTURES
+
+
+class qiandao_args(object):
+
+
+    def read(self, iprot):
+        if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
+            iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
+            return
+        iprot.readStructBegin()
+        while True:
+            (fname, ftype, fid) = iprot.readFieldBegin()
+            if ftype == TType.STOP:
+                break
+            else:
+                iprot.skip(ftype)
+            iprot.readFieldEnd()
+        iprot.readStructEnd()
+
+    def write(self, oprot):
+        if oprot._fast_encode is not None and self.thrift_spec is not None:
+            oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
+            return
+        oprot.writeStructBegin('qiandao_args')
+        oprot.writeFieldStop()
+        oprot.writeStructEnd()
+
+    def validate(self):
+        return
+
+    def __repr__(self):
+        L = ['%s=%r' % (key, value)
+             for key, value in self.__dict__.items()]
+        return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+    def __eq__(self, other):
+        return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+    def __ne__(self, other):
+        return not (self == other)
+all_structs.append(qiandao_args)
+qiandao_args.thrift_spec = (
+)
+
+
+class qiandao_result(object):
+    """
+    Attributes:
+     - success
+    """
+
+
+    def __init__(self, success=None,):
+        self.success = success
+
+    def read(self, iprot):
+        if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
+            iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
+            return
+        iprot.readStructBegin()
+        while True:
+            (fname, ftype, fid) = iprot.readFieldBegin()
+            if ftype == TType.STOP:
+                break
+            if fid == 0:
+                if ftype == TType.BOOL:
+                    self.success = iprot.readBool()
+                else:
+                    iprot.skip(ftype)
+            else:
+                iprot.skip(ftype)
+            iprot.readFieldEnd()
+        iprot.readStructEnd()
+
+    def write(self, oprot):
+        if oprot._fast_encode is not None and self.thrift_spec is not None:
+            oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
+            return
+        oprot.writeStructBegin('qiandao_result')
+        if self.success is not None:
+            oprot.writeFieldBegin('success', TType.BOOL, 0)
+            oprot.writeBool(self.success)
+            oprot.writeFieldEnd()
+        oprot.writeFieldStop()
+        oprot.writeStructEnd()
+
+    def validate(self):
+        return
+
+    def __repr__(self):
+        L = ['%s=%r' % (key, value)
+             for key, value in self.__dict__.items()]
+        return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+    def __eq__(self, other):
+        return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+    def __ne__(self, other):
+        return not (self == other)
+all_structs.append(qiandao_result)
+qiandao_result.thrift_spec = (
+    (0, TType.BOOL, 'success', None, None, ),  # 0
+)
+
+
+class word_args(object):
+
+
+    def read(self, iprot):
+        if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
+            iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
+            return
+        iprot.readStructBegin()
+        while True:
+            (fname, ftype, fid) = iprot.readFieldBegin()
+            if ftype == TType.STOP:
+                break
+            else:
+                iprot.skip(ftype)
+            iprot.readFieldEnd()
+        iprot.readStructEnd()
+
+    def write(self, oprot):
+        if oprot._fast_encode is not None and self.thrift_spec is not None:
+            oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
+            return
+        oprot.writeStructBegin('word_args')
+        oprot.writeFieldStop()
+        oprot.writeStructEnd()
+
+    def validate(self):
+        return
+
+    def __repr__(self):
+        L = ['%s=%r' % (key, value)
+             for key, value in self.__dict__.items()]
+        return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+    def __eq__(self, other):
+        return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+    def __ne__(self, other):
+        return not (self == other)
+all_structs.append(word_args)
+word_args.thrift_spec = (
+)
+
+
+class word_result(object):
+    """
+    Attributes:
+     - success
+    """
+
+
+    def __init__(self, success=None,):
+        self.success = success
+
+    def read(self, iprot):
+        if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
+            iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
+            return
+        iprot.readStructBegin()
+        while True:
+            (fname, ftype, fid) = iprot.readFieldBegin()
+            if ftype == TType.STOP:
+                break
+            if fid == 0:
+                if ftype == TType.BOOL:
+                    self.success = iprot.readBool()
+                else:
+                    iprot.skip(ftype)
+            else:
+                iprot.skip(ftype)
+            iprot.readFieldEnd()
+        iprot.readStructEnd()
+
+    def write(self, oprot):
+        if oprot._fast_encode is not None and self.thrift_spec is not None:
+            oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
+            return
+        oprot.writeStructBegin('word_result')
+        if self.success is not None:
+            oprot.writeFieldBegin('success', TType.BOOL, 0)
+            oprot.writeBool(self.success)
+            oprot.writeFieldEnd()
+        oprot.writeFieldStop()
+        oprot.writeStructEnd()
+
+    def validate(self):
+        return
+
+    def __repr__(self):
+        L = ['%s=%r' % (key, value)
+             for key, value in self.__dict__.items()]
+        return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+    def __eq__(self, other):
+        return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+
+    def __ne__(self, other):
+        return not (self == other)
+all_structs.append(word_result)
+word_result.thrift_spec = (
+    (0, TType.BOOL, 'success', None, None, ),  # 0
+)
+fix_spec(all_structs)
+del all_structs
+

PixivSearch/thrift/task/constants.py

@@ -0,0 +1,14 @@
+#
+# Autogenerated by Thrift Compiler (0.11.0)
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+#  options string: py
+#
+
+from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException
+from thrift.protocol.TProtocol import TProtocolException
+from thrift.TRecursive import fix_spec
+
+import sys
+from .ttypes import *

PixivSearch/thrift/task/ttypes.py

@@ -0,0 +1,18 @@
+#
+# Autogenerated by Thrift Compiler (0.11.0)
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+#  options string: py
+#
+
+from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException
+from thrift.protocol.TProtocol import TProtocolException
+from thrift.TRecursive import fix_spec
+
+import sys
+
+from thrift.transport import TTransport
+all_structs = []
+fix_spec(all_structs)
+del all_structs

PixivSearch/urls.py

@@ -0,0 +1,27 @@
+"""PixivSearch URL Configuration
+
+The `urlpatterns` list routes URLs to views. For more information please see:
+    https://docs.djangoproject.com/en/1.11/topics/http/urls/
+Examples:
+Function views
+    1. Add an import:  from my_app import views
+    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
+Class-based views
+    1. Add an import:  from other_app.views import Home
+    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
+Including another URLconf
+    1. Import the include() function: from django.conf.urls import url, include
+    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
+"""
+from django.conf.urls import url
+from PixivSearch.view import search, index, download, saveConfig, get, start, stop
+
+urlpatterns = [
+    url(r'^$', index),
+    url(r'^pixiv/search', search),
+    url(r'^pixiv/download', download),
+    url(r'^tsdm', saveConfig),
+    url(r'^bangumi/get', get),
+    url(r'^bangumi/start', start),
+    url(r'^bangumi/stop', stop)
+]

PixivSearch/util/Util.py

@@ -0,0 +1,10 @@
+import os
+import zipfile
+
+
+def zip(inputFile, outFile):
+    f = zipfile.ZipFile(outFile, 'w', zipfile.ZIP_DEFLATED)
+    for dirpath, dirnames, filenames in os.walk(inputFile):
+        for filename in filenames:
+            f.write(os.path.join(dirpath, filename), filename)
+    f.close()

PixivSearch/view.py

@@ -0,0 +1,101 @@
+# coding=utf-8
+import _thread
+import os
+
+import django
+from django.http import Http404, StreamingHttpResponse, HttpResponse
+from django.shortcuts import render
+
+import root
+from PixivSearch.dao.bangumi import get_, stop_, getIds
+from PixivSearch.pixiv.pixiv import get_nodes
+from PixivSearch.settings import logger
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "PixivSearch.settings")
+django.setup()  # 添加的代码
+
+from PixivSearch.model import config
+
+
+def search(request):
+    word = 'R-18'
+    pageSize = 10
+    order = 10
+    type = 'text'
+
+    if request.POST.get('word') != '' and request.POST.get('word') is not None:
+        word = request.POST.get('word')
+    if request.POST.get('pageSize') != '' and request.POST.get('pageSize') is not None:
+        pageSize = int(request.POST.get('pageSize'))
+    if request.POST.get('order') != '' and request.POST.get('order') is not None:
+        order = int(request.POST.get('order'))
+    if request.POST.get('type') != '' and request.POST.get('type') is not None:
+        type = request.POST.get('type')
+
+    try:
+        logger.info("word:%s,pageSize:%d,order:%d,type:%s", word, pageSize, order, type)
+        array = get_nodes([0, word, pageSize, order, type])
+        if len(array[0]) > 0:
+            if request.POST.get('type') == 'img':
+                nodes = {'imageNodes': array[0], 'msg': array[1], 'download': word + '.zip', 'size': array[2],
+                         'word': word, 'pageSize': pageSize, 'order': order, 'type': type,
+                         'imgPath': root.getConfigValue('imgPath'),
+                         'baseZipPath': root.getConfigValue('baseZipPath')}
+            else:
+                nodes = {'imageNodes': array[0], 'msg': array[1], 'word': word, 'pageSize': pageSize, 'order': order,
+                         'type': type}
+        else:
+            nodes = {'tip': '没有返回结果'}
+        return render(request, 'index.html', nodes)
+    except ValueError:
+        raise Http404()
+
+
+def index(request):
+    return render(request, 'index.html', {'tip': '输入参数进行搜索'})
+
+
+def download(request):
+    # do something...
+    def file_iterator(file_name, chunk_size=512):
+        with open(file_name) as f:
+            while True:
+                c = f.read(chunk_size)
+                if c:
+                    yield c
+                else:
+                    break
+
+    fileName = os.path.join(os.path.dirname(__file__), 'static/download/').replace('\\', '/') + request.GET.get(
+        'fileName').encode('utf-8')
+    response = StreamingHttpResponse(file_iterator(fileName))
+    response['Content-Type'] = 'application/octet-stream'
+    response['Content-Disposition'] = 'attachment;filename="{0}"'.format(request.GET.get('fileName').encode('utf-8'))
+    return response
+
+
+def saveConfig(request):
+    if 'GET' == request.method and request.GET.get('param_name'):
+        p = config.param.objects.get(param_name=request.GET.get('param_name'))
+        return render(request, 'addConfig.html', p.__dict__)
+    elif 'POST' == request.method and request.POST.get('param_name') and request.POST.get('param_value'):
+        p = config.param(param_name=request.POST.get('param_name'), param_value=request.POST.get('param_value'))
+        p.save()
+        return render(request, 'addConfig.html', p.__dict__)
+    else:
+        return render(request, 'addConfig.html')
+
+
+def get(request):
+    return HttpResponse(str(get_().__str__()))
+
+
+# 测试方法
+def start(request):
+    _thread.start_new_thread(getIds, ())
+    return HttpResponse("start success")
+
+
+def stop(request):
+    stop_()
+    return HttpResponse("stop success")

PixivSearch/wsgi.py

@@ -0,0 +1,16 @@
+"""
+WSGI config for PixivSearch project.
+
+It exposes the WSGI callable as a model-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/1.11/howto/deployment/wsgi/
+"""
+
+import os
+
+from django.core.wsgi import get_wsgi_application
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "PixivSearch.settings")
+
+application = get_wsgi_application()

README.md

@@ -0,0 +1 @@
+# PixivSearch

init/packages.txt

@@ -0,0 +1,18 @@
+aliyun-python-sdk-cloudphoto==1.1.18
+aliyun-python-sdk-core-v3==2.8.6
+beautifulsoup4==4.6.0
+bs4==0.0.1
+certifi==2018.4.16
+chardet==3.0.4
+crcmod==1.7
+Django==2.0.5
+idna==2.6
+lxml==4.2.1
+mysqlclient==1.3.12
+oss2==2.4.0
+param==1.6.1
+pytz==2018.4
+requests==2.18.4
+six==1.11.0
+thrift==0.11.0
+urllib3==1.22

manage.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+import os
+import sys
+
+if __name__ == "__main__":
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "PixivSearch.settings")
+    try:
+        from django.core.management import execute_from_command_line
+    except ImportError:
+        # The above import may fail for some other reason. Ensure that the
+        # issue is really that Django is missing to avoid masking other
+        # exceptions on Python 2.
+        try:
+            import django
+        except ImportError:
+            raise ImportError(
+                "Couldn't import Django. Are you sure it's installed and "
+                "available on your PYTHONPATH environment variable? Did you "
+                "forget to activate a virtual environment?"
+            )
+        raise
+    execute_from_command_line(sys.argv)

root.ini

@@ -0,0 +1,5 @@
+[config]
+#图片保存目录
+imgPath = PixivSearch/static/images/
+#压缩包保存目录
+baseZipPath=PixivSearch/static/zip/

root.py

@@ -0,0 +1,13 @@
+import configparser
+
+cf = configparser.ConfigParser()
+
+
+def getConfigValue(name):
+    cf.read('root.ini', encoding='utf-8')
+    value = cf.get("config", name)
+    return value
+
+if __name__ == '__main__':
+    print(getConfigValue('imgPath'))
+    print(getConfigValue('baseZipPath'))

start.sh

@@ -0,0 +1,2 @@
+#!/bash/sh
+nohup python3 manage.py runserver 0.0.0.0:8000 > nohup.out &