diff --git a/PixivSearch/dao/Comment.py b/PixivSearch/dao/Comment.py index 294f423..2a9f5c7 100644 --- a/PixivSearch/dao/Comment.py +++ b/PixivSearch/dao/Comment.py @@ -1,4 +1,6 @@ import json +import os +import shutil import threading from concurrent import futures @@ -25,7 +27,7 @@ class Comment: for index, data in enumerate( sorted(self.obj["data"].items(), key=lambda d: d[1], reverse=True)[ :50]): - print('{index}:{data}'.format(index=index+1, data=data)) + print('{index}:{data}'.format(index=index + 1, data=data)) # 获取番剧合集弹幕排行榜 def count(self, url, desc=None): @@ -60,22 +62,31 @@ def getCidUrls(cid): return urls -# 下载历史弹幕 -def parseXmlByHistory(path, cid,size=None): +# 下载弹幕 +def downloadXml(path, cid, size=None, histroy=True): dlist = set() - flag=parseXml(getCidUrls(cid), dlist, size) - if size is None or not size is None and flag: - f = open(path, 'wb') + flag = None + if histroy: + flag = parseXml(getCidUrls(cid), dlist, size) + else: + parseXml("https://comment.bilibili.com/%d.xml" % cid, dlist, size) + if size is None or (histroy and not size is None and flag): + if os.path.exists(path): + shutil.rmtree(path) + os.makedirs(path) + f = open('{path}/{cid}.xml'.format(path=path, cid=cid), 'wb') f.write(b'') for i in dlist: f.write(('\r\n' + i).encode()) f.write(b'\r\n') f.close() + def xml(url): bytes = requests.get(url).content return etree.HTML(bytes) + def parseXml(urls, dlist, size=None): if isinstance(urls, str): urls = [urls] @@ -92,7 +103,7 @@ def parseXml(urls, dlist, size=None): for element in comment_selector.xpath('//i//d'): if len(element.xpath("text()")) > 0: fstr = '{content}'.format(p=str(element.xpath("@p")[0]), - content=str(element.xpath("text()")[0])) + content=str(element.xpath("text()")[0])) dlist.add(fstr) currentSize = len(dlist) / maxlimit @@ -101,28 +112,8 @@ def parseXml(urls, dlist, size=None): return True return False + if __name__ == '__main__': - # parseXmlByHistory('10815558.xml', 10815558) - # Comment('').getCommentSort(episodeIdToCid(172095)) - cids=[653701,653702,653703,653704,653705,653738,653739,653740,653741,653742,653743,655269,655270,655271,655272,655273,655274,655275,655276,655277,655278,655279,1153426,1160531,1160532,1160533,1160534,1166828,3462691,3526913,4566294,4566295,4566296,4566297,4566298,4566299,4997067,4997068,4997069,4997070,4997071,4997072,4997073,4997074,4997075,4997076,4997077,4997078,4997079,4997080,4997081,4997082,4997083,4997084,4997085,4997086,4997087,4997088,4997089,5679996,7091113,7091114,7091115,7091116,7091117,7091118,7091119,7091120,7102322,7102323,7102324,7105422,7105423,7105424,7105425,7105426,7105427,7105428,7105429,7105430,7105431,7105432,7111387,7862786,7862787,7862788,7862789,7862790,7862791,7862792,7862793,7862794,7862795,7862796,8097054,8462197,8462198,8463198,8463856,8464066,8465726,8466012,8466263,8466715,8467897,8468074,8469339,8470303,8470792,8471514,8471910,8472168,8472542,8473404,8474331,8476130,8974866,9006609,9914921,9914922,9930188,9930189,9930190,9935817,9935818,9935819,9946753,10240252,10240253,10240254,10240255,10240256,10240257,10240258,10240259,10240260,10240261,10240262,12029055,12107619,13109117,13109118,13109119,13109120,13109121,13109122,13109123,13109124,13109125,13109126,13109127,13109128,13109129,13109130,13109131,13109132,13109133,13109134,13109135,13109136,13109137,13109138,13109139,13109140,13109141,13109142,13109143,13109144,13109145,13109146,13109147,13109148,13109149,13109150,13109151,13109152,13109153,13109154,13109155,13280053,13386513,13386514,13386515,13386516,13386517,13386518,13386519,13386520,13386521,13386522,13386523,13494289,14369143,14842278] - - cids=[6675884,6675885,6675886,6722497,6722498,6722499,6722500,6761162,6761163,6761164,6761165,7008191,7008192,7008193,7008194,7008195,7008196,7008197,7008198,7008199,7008200,7008201,7018471,7018472,7018473,7018474,7018475,7018476,7018477,7018478,7018479,7018480,7018481,7636499,7636500,7636501,7636502,7636503,7636504,7636505,7636506,7636507,7636508,7636509,9019798,9019799,9019800,9019801,9019802,9019803,9019804,9019805,9019806,9019807,9019808,9019809,9832709] - for cid in cids: - comment_selector=xml('https://comment.bilibili.com/{cid}.xml'.format(cid=cid)) - print('cid:{cid},弹幕数:{length}'.format(cid=cid,length=len(comment_selector.xpath('//i//d/text()')))) - # parseXmlByHistory('{cid}.xml'.format(cid=cid),cid) - - # path = 'D:/QQ/1029559041/FileRecv/tmp001.xlsx' - # ExcelFile = xlrd.open_workbook(path) - # - # - # for sheetName in ExcelFile.sheet_names(): - # sheet = ExcelFile.sheet_by_name(sheetName) - # for row in sheet.get_rows(): - # try: - # row[5].value.index('日剧') - # cid = int(row[2].value) - # print('aid={aid},cid={cid},title={title}'.format(aid=int(row[1].value),cid=cid,title=row[5].value)) - # parseXmlByHistory('{cid}.xml'.format(cid=cid),cid,'50%') - # except BaseException as e: - # repr(e) + cids = [7636499, 7636501, 7636500, 7636503, 7636504, 7636502, 7636509, 7636508, 7636506, 7636507, 7636505] + + downloadXml('F:/ABC',12026697,histroy=False) diff --git a/PixivSearch/dao/bangumi.py b/PixivSearch/dao/bangumi.py index 6b02cce..4f048f6 100644 --- a/PixivSearch/dao/bangumi.py +++ b/PixivSearch/dao/bangumi.py @@ -184,7 +184,8 @@ def episodeIdToCid(episode_id): if __name__ == '__main__': - print(getCids(29416)) + # print(getCids(29416)) + req=requests.post('https://api.bilibili.com/x/v2/history/shadow/set','') # obj = loadData([34807341], []) diff --git a/PixivSearch/pixiv/pixiv.py b/PixivSearch/pixiv/pixiv.py index a3312b5..17c278f 100644 --- a/PixivSearch/pixiv/pixiv.py +++ b/PixivSearch/pixiv/pixiv.py @@ -1,17 +1,18 @@ #!/usr/bin/env python # coding:utf-8 -from concurrent import futures -import threading import json -import requests -from bs4 import BeautifulSoup +import os import sys +import threading +from concurrent import futures from datetime import datetime -import os -import zipfile + +import requests +from bs4 import BeautifulSoup from PixivSearch.model import config from PixivSearch.settings import logger +from PixivSearch.util import Util headers = { 'X-Requested-With': 'XMLHttpRequest', @@ -74,15 +75,6 @@ def get_Img(params): params[0], params[1]['bookmarkCount'], params[1]['illustTitle'], ','.join(params[1]['tags']), '', params[1]['imgUrl'])) - -def zip(inputFile, outFile): - f = zipfile.ZipFile(outFile, 'w', zipfile.ZIP_DEFLATED) - for dirpath, dirnames, filenames in os.walk(inputFile): - for filename in filenames: - f.write(os.path.join(dirpath, filename), filename) - f.close() - - fsize = '' @@ -119,7 +111,7 @@ def get_nodes(param): zipPath = 'PixivSearch/static/download/' + param[1] + '.zip' logger.info('图片打包到:%s' % (zipPath)) - zip(imgPath, zipPath) + Util.zip(imgPath, zipPath) fsize = str(round(os.path.getsize(zipPath) / float(1024 * 1024), 2)) + 'MB' logger.info('图包大小:%s' % (fsize)) diff --git a/PixivSearch/settings.py b/PixivSearch/settings.py index f584b76..9be76e0 100644 --- a/PixivSearch/settings.py +++ b/PixivSearch/settings.py @@ -79,9 +79,9 @@ DATABASES = { 'ENGINE': 'django.db.backends.mysql', 'NAME': 'bangumi', 'USER': 'root', - 'PASSWORD': 'Luffy9412!', - # 'HOST': '127.0.0.1', - 'HOST': 'mikuhime.xyz', + 'PASSWORD': '', + 'HOST': '127.0.0.1', + # 'HOST': 'sukura.top', 'PORT': '3306', } } diff --git a/PixivSearch/thrift/TestQry/Client.py b/PixivSearch/thrift/QueryComment/Client.py similarity index 63% rename from PixivSearch/thrift/TestQry/Client.py rename to PixivSearch/thrift/QueryComment/Client.py index 89fea10..9abc8e7 100644 --- a/PixivSearch/thrift/TestQry/Client.py +++ b/PixivSearch/thrift/QueryComment/Client.py @@ -1,14 +1,11 @@ -import sys - from thrift.protocol import TBinaryProtocol from thrift.transport import TSocket, TTransport -sys.path.append('/root/PixivSearch') -from PixivSearch.thrift.TestQry.TestQry import Client +from PixivSearch.thrift.QueryComment.QueryComment import Client if __name__ == '__main__': - # Make socket - socket = TSocket.TSocket('mikuhime.xyz', 2233) + + socket = TSocket.TSocket('sukura.top', 2233) # Buffering is critical. Raw sockets are very slow transport = TTransport.TFramedTransport(socket) @@ -22,4 +19,5 @@ if __name__ == '__main__': # Create a client to use the protocol encoder client = Client(protocol) - print(client.qryTest(1)) + cids = [7636499, 7636501, 7636500, 7636503, 7636504, 7636502, 7636509, 7636508, 7636506, 7636507, 7636505] + print(client.commentSumMap(cids)) \ No newline at end of file diff --git a/PixivSearch/thrift/TestQry/TestQry-remote b/PixivSearch/thrift/QueryComment/QueryComment-remote old mode 100755 new mode 100644 similarity index 74% rename from PixivSearch/thrift/TestQry/TestQry-remote rename to PixivSearch/thrift/QueryComment/QueryComment-remote index c12dd5a..a76303e --- a/PixivSearch/thrift/TestQry/TestQry-remote +++ b/PixivSearch/thrift/QueryComment/QueryComment-remote @@ -9,24 +9,24 @@ import sys import pprint -from urllib.parse import urlparse - -from PixivSearch.thrift.TestQry import TestQry - -sys.path.append('/root/PixivSearch') - - +if sys.version_info[0] > 2: + from urllib.parse import urlparse +else: + from urlparse import urlparse from thrift.transport import TTransport, TSocket, TSSLSocket, THttpClient from thrift.protocol.TBinaryProtocol import TBinaryProtocol - +from QueryComment import QueryComment +from QueryComment.ttypes import * if len(sys.argv) <= 1 or sys.argv[1] == '--help': print('') print('Usage: ' + sys.argv[0] + ' [-h host[:port]] [-u url] [-f[ramed]] [-s[sl]] [-novalidate] [-ca_certs certs] [-keyfile keyfile] [-certfile certfile] function [arg1 [arg2...]]') print('') print('Functions:') - print(' QryResult qryTest(i32 qryCode)') + print(' i32 commentSum(i32 cid)') + print(' commentSumList( cids)') + print(' string download( cids, string fileName)') print('') sys.exit(0) @@ -103,14 +103,26 @@ else: else: transport = TTransport.TBufferedTransport(socket) protocol = TBinaryProtocol(transport) -client = TestQry.Client(protocol) +client = QueryComment.Client(protocol) transport.open() -if cmd == 'qryTest': +if cmd == 'commentSum': if len(args) != 1: - print('qryTest requires 1 args') + print('commentSum requires 1 args') + sys.exit(1) + pp.pprint(client.commentSum(eval(args[0]),)) + +elif cmd == 'commentSumList': + if len(args) != 1: + print('commentSumList requires 1 args') + sys.exit(1) + pp.pprint(client.commentSumList(eval(args[0]),)) + +elif cmd == 'download': + if len(args) != 2: + print('download requires 2 args') sys.exit(1) - pp.pprint(client.qryTest(eval(args[0]),)) + pp.pprint(client.download(eval(args[0]), args[1],)) else: print('Unrecognized method %s' % cmd) diff --git a/PixivSearch/thrift/QueryComment/QueryComment.py b/PixivSearch/thrift/QueryComment/QueryComment.py new file mode 100644 index 0000000..9b4bffb --- /dev/null +++ b/PixivSearch/thrift/QueryComment/QueryComment.py @@ -0,0 +1,660 @@ +# +# Autogenerated by Thrift Compiler (0.11.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# +# options string: py +# + +from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException +from thrift.protocol.TProtocol import TProtocolException +from thrift.TRecursive import fix_spec + +import sys +import logging +from .ttypes import * +from thrift.Thrift import TProcessor +from thrift.transport import TTransport +all_structs = [] + + +class Iface(object): + def commentSum(self, cid): + """ + 查询单个cid弹幕数 + + + Parameters: + - cid + """ + pass + + def commentSumList(self, cids): + """ + * 批量查询多个cid弹幕数 + * + + Parameters: + - cids + """ + pass + + def download(self, cids, fileName): + """ + * 批量下载弹幕 + * + + Parameters: + - cids + - fileName + """ + pass + + +class Client(Iface): + def __init__(self, iprot, oprot=None): + self._iprot = self._oprot = iprot + if oprot is not None: + self._oprot = oprot + self._seqid = 0 + + def commentSum(self, cid): + """ + 查询单个cid弹幕数 + + + Parameters: + - cid + """ + self.send_commentSum(cid) + return self.recv_commentSum() + + def send_commentSum(self, cid): + self._oprot.writeMessageBegin('commentSum', TMessageType.CALL, self._seqid) + args = commentSum_args() + args.cid = cid + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_commentSum(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = commentSum_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "commentSum failed: unknown result") + + def commentSumList(self, cids): + """ + * 批量查询多个cid弹幕数 + * + + Parameters: + - cids + """ + self.send_commentSumList(cids) + return self.recv_commentSumList() + + def send_commentSumList(self, cids): + self._oprot.writeMessageBegin('commentSumList', TMessageType.CALL, self._seqid) + args = commentSumList_args() + args.cids = cids + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_commentSumList(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = commentSumList_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "commentSumList failed: unknown result") + + def download(self, cids, fileName): + """ + * 批量下载弹幕 + * + + Parameters: + - cids + - fileName + """ + self.send_download(cids, fileName) + return self.recv_download() + + def send_download(self, cids, fileName): + self._oprot.writeMessageBegin('download', TMessageType.CALL, self._seqid) + args = download_args() + args.cids = cids + args.fileName = fileName + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_download(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = download_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "download failed: unknown result") + + +class Processor(Iface, TProcessor): + def __init__(self, handler): + self._handler = handler + self._processMap = {} + self._processMap["commentSum"] = Processor.process_commentSum + self._processMap["commentSumList"] = Processor.process_commentSumList + self._processMap["download"] = Processor.process_download + + def process(self, iprot, oprot): + (name, type, seqid) = iprot.readMessageBegin() + if name not in self._processMap: + iprot.skip(TType.STRUCT) + iprot.readMessageEnd() + x = TApplicationException(TApplicationException.UNKNOWN_METHOD, 'Unknown function %s' % (name)) + oprot.writeMessageBegin(name, TMessageType.EXCEPTION, seqid) + x.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + return + else: + self._processMap[name](self, seqid, iprot, oprot) + return True + + def process_commentSum(self, seqid, iprot, oprot): + args = commentSum_args() + args.read(iprot) + iprot.readMessageEnd() + result = commentSum_result() + try: + result.success = self._handler.commentSum(args.cid) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception('TApplication exception in handler') + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception('Unexpected exception in handler') + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') + oprot.writeMessageBegin("commentSum", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_commentSumList(self, seqid, iprot, oprot): + args = commentSumList_args() + args.read(iprot) + iprot.readMessageEnd() + result = commentSumList_result() + try: + result.success = self._handler.commentSumList(args.cids) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception('TApplication exception in handler') + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception('Unexpected exception in handler') + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') + oprot.writeMessageBegin("commentSumList", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_download(self, seqid, iprot, oprot): + args = download_args() + args.read(iprot) + iprot.readMessageEnd() + result = download_result() + try: + result.success = self._handler.download(args.cids, args.fileName) + msg_type = TMessageType.REPLY + except TTransport.TTransportException: + raise + except TApplicationException as ex: + logging.exception('TApplication exception in handler') + msg_type = TMessageType.EXCEPTION + result = ex + except Exception: + logging.exception('Unexpected exception in handler') + msg_type = TMessageType.EXCEPTION + result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') + oprot.writeMessageBegin("download", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + +# HELPER FUNCTIONS AND STRUCTURES + + +class commentSum_args(object): + """ + Attributes: + - cid + """ + + + def __init__(self, cid=None,): + self.cid = cid + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.cid = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('commentSum_args') + if self.cid is not None: + oprot.writeFieldBegin('cid', TType.I32, 1) + oprot.writeI32(self.cid) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) +all_structs.append(commentSum_args) +commentSum_args.thrift_spec = ( + None, # 0 + (1, TType.I32, 'cid', None, None, ), # 1 +) + + +class commentSum_result(object): + """ + Attributes: + - success + """ + + + def __init__(self, success=None,): + self.success = success + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.I32: + self.success = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('commentSum_result') + if self.success is not None: + oprot.writeFieldBegin('success', TType.I32, 0) + oprot.writeI32(self.success) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) +all_structs.append(commentSum_result) +commentSum_result.thrift_spec = ( + (0, TType.I32, 'success', None, None, ), # 0 +) + + +class commentSumList_args(object): + """ + Attributes: + - cids + """ + + + def __init__(self, cids=None,): + self.cids = cids + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.cids = [] + (_etype3, _size0) = iprot.readListBegin() + for _i4 in range(_size0): + _elem5 = iprot.readI32() + self.cids.append(_elem5) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('commentSumList_args') + if self.cids is not None: + oprot.writeFieldBegin('cids', TType.LIST, 1) + oprot.writeListBegin(TType.I32, len(self.cids)) + for iter6 in self.cids: + oprot.writeI32(iter6) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) +all_structs.append(commentSumList_args) +commentSumList_args.thrift_spec = ( + None, # 0 + (1, TType.LIST, 'cids', (TType.I32, None, False), None, ), # 1 +) + + +class commentSumList_result(object): + """ + Attributes: + - success + """ + + + def __init__(self, success=None,): + self.success = success + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.LIST: + self.success = [] + (_etype10, _size7) = iprot.readListBegin() + for _i11 in range(_size7): + _elem12 = iprot.readI32() + self.success.append(_elem12) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('commentSumList_result') + if self.success is not None: + oprot.writeFieldBegin('success', TType.LIST, 0) + oprot.writeListBegin(TType.I32, len(self.success)) + for iter13 in self.success: + oprot.writeI32(iter13) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) +all_structs.append(commentSumList_result) +commentSumList_result.thrift_spec = ( + (0, TType.LIST, 'success', (TType.I32, None, False), None, ), # 0 +) + + +class download_args(object): + """ + Attributes: + - cids + - fileName + """ + + + def __init__(self, cids=None, fileName=None,): + self.cids = cids + self.fileName = fileName + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.cids = [] + (_etype17, _size14) = iprot.readListBegin() + for _i18 in range(_size14): + _elem19 = iprot.readI32() + self.cids.append(_elem19) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.fileName = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('download_args') + if self.cids is not None: + oprot.writeFieldBegin('cids', TType.LIST, 1) + oprot.writeListBegin(TType.I32, len(self.cids)) + for iter20 in self.cids: + oprot.writeI32(iter20) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.fileName is not None: + oprot.writeFieldBegin('fileName', TType.STRING, 2) + oprot.writeString(self.fileName.encode('utf-8') if sys.version_info[0] == 2 else self.fileName) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) +all_structs.append(download_args) +download_args.thrift_spec = ( + None, # 0 + (1, TType.LIST, 'cids', (TType.I32, None, False), None, ), # 1 + (2, TType.STRING, 'fileName', 'UTF8', None, ), # 2 +) + + +class download_result(object): + """ + Attributes: + - success + """ + + + def __init__(self, success=None,): + self.success = success + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRING: + self.success = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('download_result') + if self.success is not None: + oprot.writeFieldBegin('success', TType.STRING, 0) + oprot.writeString(self.success.encode('utf-8') if sys.version_info[0] == 2 else self.success) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) +all_structs.append(download_result) +download_result.thrift_spec = ( + (0, TType.STRING, 'success', 'UTF8', None, ), # 0 +) +fix_spec(all_structs) +del all_structs + diff --git a/PixivSearch/thrift/QueryComment/Server.py b/PixivSearch/thrift/QueryComment/Server.py new file mode 100644 index 0000000..956f024 --- /dev/null +++ b/PixivSearch/thrift/QueryComment/Server.py @@ -0,0 +1,48 @@ +import os +import sys + +from thrift.server.TNonblockingServer import TNonblockingServer +from thrift.transport import TSocket + +sys.path.append('/root/PixivSearch') +from PixivSearch.util import Util +from PixivSearch.dao.Comment import xml, downloadXml +from PixivSearch.thrift.QueryComment import QueryComment +from PixivSearch.thrift.QueryComment.QueryComment import Iface + +commentPath = '/root/PixivSearch/PixivSearch/thrift/tmpFile/comment' + + +class Server(Iface): + def commentSumList(self, cids): + result = [] + for cid in cids: + comment_selector = xml('https://comment.bilibili.com/{cid}.xml'.format(cid=cid)) + length = len(comment_selector.xpath('//i//d/text()')) + print('cid:{cid},弹幕数:{length}'.format(cid=cid, length=length)) + result.append(length) + return result + + def commentSum(self, cid): + return self.commentSumList([cid])[0]; + + def download(self, cids, fileName): + path = '{commentPath}/{fileName}'.format(commentPath=commentPath, fileName=fileName) + for cid in cids: + downloadXml(path, cid, + histroy=False) + zipFile = '{path}/{fileName}.zip'.format(path=path,fileName=fileName) + print(zipFile) + Util.zip(path, zipFile) + if os.path.isfile(zipFile): + print('压缩包成功生成到{zipFile}'.format(zipFile=zipFile)) + return zipFile + else: + return None + + +if __name__ == '__main__': + socket = TSocket.TServerSocket(port=2233) + processor = QueryComment.Processor(Server()) + server = TNonblockingServer(processor, socket) + server.serve() diff --git a/PixivSearch/thrift/QueryComment/__init__.py b/PixivSearch/thrift/QueryComment/__init__.py new file mode 100644 index 0000000..52b572e --- /dev/null +++ b/PixivSearch/thrift/QueryComment/__init__.py @@ -0,0 +1 @@ +__all__ = ['ttypes', 'constants', 'QueryComment'] diff --git a/PixivSearch/thrift/TestQry/constants.py b/PixivSearch/thrift/QueryComment/constants.py similarity index 100% rename from PixivSearch/thrift/TestQry/constants.py rename to PixivSearch/thrift/QueryComment/constants.py diff --git a/PixivSearch/thrift/QueryComment/ttypes.py b/PixivSearch/thrift/QueryComment/ttypes.py new file mode 100644 index 0000000..9fb0f57 --- /dev/null +++ b/PixivSearch/thrift/QueryComment/ttypes.py @@ -0,0 +1,18 @@ +# +# Autogenerated by Thrift Compiler (0.11.0) +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# +# options string: py +# + +from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException +from thrift.protocol.TProtocol import TProtocolException +from thrift.TRecursive import fix_spec + +import sys + +from thrift.transport import TTransport +all_structs = [] +fix_spec(all_structs) +del all_structs diff --git a/PixivSearch/thrift/TestQry/Server.py b/PixivSearch/thrift/TestQry/Server.py deleted file mode 100644 index a31e9e5..0000000 --- a/PixivSearch/thrift/TestQry/Server.py +++ /dev/null @@ -1,30 +0,0 @@ -import sys - -from thrift.server.TNonblockingServer import TNonblockingServer -from thrift.transport import TSocket - -sys.path.append('/root/PixivSearch') -from PixivSearch.thrift.TestQry import TestQry -from PixivSearch.thrift.TestQry.ttypes import QryResult - - -class QueryImpl(TestQry.Iface): - - def qryTest(self, qryCode): - result = QryResult() - if qryCode == 1: - result.code = 1 - result.msg = 'success' - - else: - result.code = 0 - result.msg = 'fail' - return result - - -if __name__ == '__main__': - socket = TSocket.TServerSocket(port=2233) - processor = TestQry.Processor(QueryImpl()) - server = TNonblockingServer(processor, socket) - - server.serve() diff --git a/PixivSearch/thrift/TestQry/TestQry.py b/PixivSearch/thrift/TestQry/TestQry.py deleted file mode 100644 index 228ef8d..0000000 --- a/PixivSearch/thrift/TestQry/TestQry.py +++ /dev/null @@ -1,245 +0,0 @@ -# -# Autogenerated by Thrift Compiler (0.11.0) -# -# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING -# -# options string: py -# -import logging - -from thrift.TRecursive import fix_spec -from thrift.Thrift import TProcessor -from thrift.Thrift import TType, TMessageType, TApplicationException -from thrift.transport import TTransport - -from PixivSearch.thrift.TestQry.ttypes import QryResult - -all_structs = [] - -class Iface(object): - def qryTest(self, qryCode): - """ - 测试查询接口,当qryCode值为1时返回"成功"的响应信息,qryCode值为其他值时返回"失败"的响应信息 - @param qryCode测试参数 - - Parameters: - - qryCode - """ - pass - - -class Client(Iface): - def __init__(self, iprot, oprot=None): - self._iprot = self._oprot = iprot - if oprot is not None: - self._oprot = oprot - self._seqid = 0 - - def qryTest(self, qryCode): - """ - 测试查询接口,当qryCode值为1时返回"成功"的响应信息,qryCode值为其他值时返回"失败"的响应信息 - @param qryCode测试参数 - - Parameters: - - qryCode - """ - self.send_qryTest(qryCode) - return self.recv_qryTest() - - def send_qryTest(self, qryCode): - self._oprot.writeMessageBegin('qryTest', TMessageType.CALL, self._seqid) - args = qryTest_args() - args.qryCode = qryCode - args.write(self._oprot) - self._oprot.writeMessageEnd() - self._oprot.trans.flush() - - def recv_qryTest(self): - iprot = self._iprot - (fname, mtype, rseqid) = iprot.readMessageBegin() - if mtype == TMessageType.EXCEPTION: - x = TApplicationException() - x.read(iprot) - iprot.readMessageEnd() - raise x - result = qryTest_result() - result.read(iprot) - iprot.readMessageEnd() - if result.success is not None: - return result.success - raise TApplicationException(TApplicationException.MISSING_RESULT, "qryTest failed: unknown result") - - -class Processor(Iface, TProcessor): - def __init__(self, handler): - self._handler = handler - self._processMap = {} - self._processMap["qryTest"] = Processor.process_qryTest - - def process(self, iprot, oprot): - (name, type, seqid) = iprot.readMessageBegin() - if name not in self._processMap: - iprot.skip(TType.STRUCT) - iprot.readMessageEnd() - x = TApplicationException(TApplicationException.UNKNOWN_METHOD, 'Unknown function %s' % (name)) - oprot.writeMessageBegin(name, TMessageType.EXCEPTION, seqid) - x.write(oprot) - oprot.writeMessageEnd() - oprot.trans.flush() - return - else: - self._processMap[name](self, seqid, iprot, oprot) - return True - - def process_qryTest(self, seqid, iprot, oprot): - args = qryTest_args() - args.read(iprot) - iprot.readMessageEnd() - result = qryTest_result() - try: - result.success = self._handler.qryTest(args.qryCode) - msg_type = TMessageType.REPLY - except TTransport.TTransportException: - raise - except TApplicationException as ex: - logging.exception('TApplication exception in handler') - msg_type = TMessageType.EXCEPTION - result = ex - except Exception: - logging.exception('Unexpected exception in handler') - msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') - oprot.writeMessageBegin("qryTest", msg_type, seqid) - result.write(oprot) - oprot.writeMessageEnd() - oprot.trans.flush() - -# HELPER FUNCTIONS AND STRUCTURES - - -class qryTest_args(object): - """ - Attributes: - - qryCode - """ - - - def __init__(self, qryCode=None,): - self.qryCode = qryCode - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.I32: - self.qryCode = iprot.readI32() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('qryTest_args') - if self.qryCode is not None: - oprot.writeFieldBegin('qryCode', TType.I32, 1) - oprot.writeI32(self.qryCode) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(qryTest_args) -qryTest_args.thrift_spec = ( - None, # 0 - (1, TType.I32, 'qryCode', None, None, ), # 1 -) - - -class qryTest_result(object): - """ - Attributes: - - success - """ - - - def __init__(self, success=None,): - self.success = success - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 0: - if ftype == TType.STRUCT: - self.success = QryResult() - self.success.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('qryTest_result') - if self.success is not None: - oprot.writeFieldBegin('success', TType.STRUCT, 0) - self.success.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(qryTest_result) -qryTest_result.thrift_spec = ( - (0, TType.STRUCT, 'success', [QryResult, None], None, ), # 0 -) -fix_spec(all_structs) -del all_structs - - - - diff --git a/PixivSearch/thrift/TestQry/__init__.py b/PixivSearch/thrift/TestQry/__init__.py deleted file mode 100644 index 22182f6..0000000 --- a/PixivSearch/thrift/TestQry/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__all__ = ['ttypes', 'constants', 'TestQry'] diff --git a/PixivSearch/thrift/TestQry/ttypes.py b/PixivSearch/thrift/TestQry/ttypes.py deleted file mode 100644 index 523a81b..0000000 --- a/PixivSearch/thrift/TestQry/ttypes.py +++ /dev/null @@ -1,91 +0,0 @@ -# -# Autogenerated by Thrift Compiler (0.11.0) -# -# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING -# -# options string: py -# - -from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException -from thrift.protocol.TProtocol import TProtocolException -from thrift.TRecursive import fix_spec - -import sys - -from thrift.transport import TTransport -all_structs = [] - - -class QryResult(object): - """ - Attributes: - - code: 返回码, 1成功,0失败 - - msg: 响应信息 - """ - - - def __init__(self, code=None, msg=None,): - self.code = code - self.msg = msg - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.I32: - self.code = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 2: - if ftype == TType.STRING: - self.msg = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('QryResult') - if self.code is not None: - oprot.writeFieldBegin('code', TType.I32, 1) - oprot.writeI32(self.code) - oprot.writeFieldEnd() - if self.msg is not None: - oprot.writeFieldBegin('msg', TType.STRING, 2) - oprot.writeString(self.msg.encode('utf-8') if sys.version_info[0] == 2 else self.msg) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(QryResult) -QryResult.thrift_spec = ( - None, # 0 - (1, TType.I32, 'code', None, None, ), # 1 - (2, TType.STRING, 'msg', 'UTF8', None, ), # 2 -) -fix_spec(all_structs) -del all_structs diff --git a/PixivSearch/util/Util.py b/PixivSearch/util/Util.py new file mode 100644 index 0000000..d69b89a --- /dev/null +++ b/PixivSearch/util/Util.py @@ -0,0 +1,10 @@ +import os +import zipfile + + +def zip(inputFile, outFile): + f = zipfile.ZipFile(outFile, 'w', zipfile.ZIP_DEFLATED) + for dirpath, dirnames, filenames in os.walk(inputFile): + for filename in filenames: + f.write(os.path.join(dirpath, filename), filename) + f.close() \ No newline at end of file diff --git a/PixivSearch/util/__init__.py b/PixivSearch/util/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/PixivSearch/view.py b/PixivSearch/view.py index d01f309..9153a04 100644 --- a/PixivSearch/view.py +++ b/PixivSearch/view.py @@ -7,13 +7,13 @@ from django.http import Http404, StreamingHttpResponse, HttpResponse from django.shortcuts import render from PixivSearch.dao.bangumi import get_, stop_, getIds +from PixivSearch.pixiv.pixiv import get_nodes from PixivSearch.settings import logger os.environ.setdefault("DJANGO_SETTINGS_MODULE", "PixivSearch.settings") django.setup() # 添加的代码 from PixivSearch.model import config -from PixivSearch.pixiv import get_nodes def search(request):