thrift弹幕下载

master
10295 7 years ago
parent 06a5a453f4
commit f0e5adb105
  1. 53
      PixivSearch/dao/Comment.py
  2. 3
      PixivSearch/dao/bangumi.py
  3. 24
      PixivSearch/pixiv/pixiv.py
  4. 6
      PixivSearch/settings.py
  5. 12
      PixivSearch/thrift/QueryComment/Client.py
  6. 38
      PixivSearch/thrift/QueryComment/QueryComment-remote
  7. 660
      PixivSearch/thrift/QueryComment/QueryComment.py
  8. 48
      PixivSearch/thrift/QueryComment/Server.py
  9. 1
      PixivSearch/thrift/QueryComment/__init__.py
  10. 0
      PixivSearch/thrift/QueryComment/constants.py
  11. 18
      PixivSearch/thrift/QueryComment/ttypes.py
  12. 30
      PixivSearch/thrift/TestQry/Server.py
  13. 245
      PixivSearch/thrift/TestQry/TestQry.py
  14. 1
      PixivSearch/thrift/TestQry/__init__.py
  15. 91
      PixivSearch/thrift/TestQry/ttypes.py
  16. 10
      PixivSearch/util/Util.py
  17. 0
      PixivSearch/util/__init__.py
  18. 2
      PixivSearch/view.py

@ -1,4 +1,6 @@
import json
import os
import shutil
import threading
from concurrent import futures
@ -25,7 +27,7 @@ class Comment:
for index, data in enumerate(
sorted(self.obj["data"].items(), key=lambda d: d[1], reverse=True)[
:50]):
print('{index}:{data}'.format(index=index+1, data=data))
print('{index}:{data}'.format(index=index + 1, data=data))
# 获取番剧合集弹幕排行榜
def count(self, url, desc=None):
@ -60,22 +62,31 @@ def getCidUrls(cid):
return urls
# 下载历史弹幕
def parseXmlByHistory(path, cid,size=None):
# 下载弹幕
def downloadXml(path, cid, size=None, histroy=True):
dlist = set()
flag=parseXml(getCidUrls(cid), dlist, size)
if size is None or not size is None and flag:
f = open(path, 'wb')
flag = None
if histroy:
flag = parseXml(getCidUrls(cid), dlist, size)
else:
parseXml("https://comment.bilibili.com/%d.xml" % cid, dlist, size)
if size is None or (histroy and not size is None and flag):
if os.path.exists(path):
shutil.rmtree(path)
os.makedirs(path)
f = open('{path}/{cid}.xml'.format(path=path, cid=cid), 'wb')
f.write(b'<?xml version="1.0" encoding="UTF-8"?><i>')
for i in dlist:
f.write(('\r\n' + i).encode())
f.write(b'\r\n</i>')
f.close()
def xml(url):
bytes = requests.get(url).content
return etree.HTML(bytes)
def parseXml(urls, dlist, size=None):
if isinstance(urls, str):
urls = [urls]
@ -92,7 +103,7 @@ def parseXml(urls, dlist, size=None):
for element in comment_selector.xpath('//i//d'):
if len(element.xpath("text()")) > 0:
fstr = '<d p="{p}">{content}</d>'.format(p=str(element.xpath("@p")[0]),
content=str(element.xpath("text()")[0]))
content=str(element.xpath("text()")[0]))
dlist.add(fstr)
currentSize = len(dlist) / maxlimit
@ -101,28 +112,8 @@ def parseXml(urls, dlist, size=None):
return True
return False
if __name__ == '__main__':
# parseXmlByHistory('10815558.xml', 10815558)
# Comment('').getCommentSort(episodeIdToCid(172095))
cids=[653701,653702,653703,653704,653705,653738,653739,653740,653741,653742,653743,655269,655270,655271,655272,655273,655274,655275,655276,655277,655278,655279,1153426,1160531,1160532,1160533,1160534,1166828,3462691,3526913,4566294,4566295,4566296,4566297,4566298,4566299,4997067,4997068,4997069,4997070,4997071,4997072,4997073,4997074,4997075,4997076,4997077,4997078,4997079,4997080,4997081,4997082,4997083,4997084,4997085,4997086,4997087,4997088,4997089,5679996,7091113,7091114,7091115,7091116,7091117,7091118,7091119,7091120,7102322,7102323,7102324,7105422,7105423,7105424,7105425,7105426,7105427,7105428,7105429,7105430,7105431,7105432,7111387,7862786,7862787,7862788,7862789,7862790,7862791,7862792,7862793,7862794,7862795,7862796,8097054,8462197,8462198,8463198,8463856,8464066,8465726,8466012,8466263,8466715,8467897,8468074,8469339,8470303,8470792,8471514,8471910,8472168,8472542,8473404,8474331,8476130,8974866,9006609,9914921,9914922,9930188,9930189,9930190,9935817,9935818,9935819,9946753,10240252,10240253,10240254,10240255,10240256,10240257,10240258,10240259,10240260,10240261,10240262,12029055,12107619,13109117,13109118,13109119,13109120,13109121,13109122,13109123,13109124,13109125,13109126,13109127,13109128,13109129,13109130,13109131,13109132,13109133,13109134,13109135,13109136,13109137,13109138,13109139,13109140,13109141,13109142,13109143,13109144,13109145,13109146,13109147,13109148,13109149,13109150,13109151,13109152,13109153,13109154,13109155,13280053,13386513,13386514,13386515,13386516,13386517,13386518,13386519,13386520,13386521,13386522,13386523,13494289,14369143,14842278]
cids=[6675884,6675885,6675886,6722497,6722498,6722499,6722500,6761162,6761163,6761164,6761165,7008191,7008192,7008193,7008194,7008195,7008196,7008197,7008198,7008199,7008200,7008201,7018471,7018472,7018473,7018474,7018475,7018476,7018477,7018478,7018479,7018480,7018481,7636499,7636500,7636501,7636502,7636503,7636504,7636505,7636506,7636507,7636508,7636509,9019798,9019799,9019800,9019801,9019802,9019803,9019804,9019805,9019806,9019807,9019808,9019809,9832709]
for cid in cids:
comment_selector=xml('https://comment.bilibili.com/{cid}.xml'.format(cid=cid))
print('cid:{cid},弹幕数:{length}'.format(cid=cid,length=len(comment_selector.xpath('//i//d/text()'))))
# parseXmlByHistory('{cid}.xml'.format(cid=cid),cid)
# path = 'D:/QQ/1029559041/FileRecv/tmp001.xlsx'
# ExcelFile = xlrd.open_workbook(path)
#
#
# for sheetName in ExcelFile.sheet_names():
# sheet = ExcelFile.sheet_by_name(sheetName)
# for row in sheet.get_rows():
# try:
# row[5].value.index('日剧')
# cid = int(row[2].value)
# print('aid={aid},cid={cid},title={title}'.format(aid=int(row[1].value),cid=cid,title=row[5].value))
# parseXmlByHistory('{cid}.xml'.format(cid=cid),cid,'50%')
# except BaseException as e:
# repr(e)
cids = [7636499, 7636501, 7636500, 7636503, 7636504, 7636502, 7636509, 7636508, 7636506, 7636507, 7636505]
downloadXml('F:/ABC',12026697,histroy=False)

@ -184,7 +184,8 @@ def episodeIdToCid(episode_id):
if __name__ == '__main__':
print(getCids(29416))
# print(getCids(29416))
req=requests.post('https://api.bilibili.com/x/v2/history/shadow/set','')
# obj = loadData([34807341], [])

@ -1,17 +1,18 @@
#!/usr/bin/env python
# coding:utf-8
from concurrent import futures
import threading
import json
import requests
from bs4 import BeautifulSoup
import os
import sys
import threading
from concurrent import futures
from datetime import datetime
import os
import zipfile
import requests
from bs4 import BeautifulSoup
from PixivSearch.model import config
from PixivSearch.settings import logger
from PixivSearch.util import Util
headers = {
'X-Requested-With': 'XMLHttpRequest',
@ -74,15 +75,6 @@ def get_Img(params):
params[0], params[1]['bookmarkCount'], params[1]['illustTitle'], ','.join(params[1]['tags']), '',
params[1]['imgUrl']))
def zip(inputFile, outFile):
f = zipfile.ZipFile(outFile, 'w', zipfile.ZIP_DEFLATED)
for dirpath, dirnames, filenames in os.walk(inputFile):
for filename in filenames:
f.write(os.path.join(dirpath, filename), filename)
f.close()
fsize = ''
@ -119,7 +111,7 @@ def get_nodes(param):
zipPath = 'PixivSearch/static/download/' + param[1] + '.zip'
logger.info('图片打包到:%s' % (zipPath))
zip(imgPath, zipPath)
Util.zip(imgPath, zipPath)
fsize = str(round(os.path.getsize(zipPath) / float(1024 * 1024), 2)) + 'MB'
logger.info('图包大小:%s' % (fsize))

@ -79,9 +79,9 @@ DATABASES = {
'ENGINE': 'django.db.backends.mysql',
'NAME': 'bangumi',
'USER': 'root',
'PASSWORD': 'Luffy9412!',
# 'HOST': '127.0.0.1',
'HOST': 'mikuhime.xyz',
'PASSWORD': '',
'HOST': '127.0.0.1',
# 'HOST': 'sukura.top',
'PORT': '3306',
}
}

@ -1,14 +1,11 @@
import sys
from thrift.protocol import TBinaryProtocol
from thrift.transport import TSocket, TTransport
sys.path.append('/root/PixivSearch')
from PixivSearch.thrift.TestQry.TestQry import Client
from PixivSearch.thrift.QueryComment.QueryComment import Client
if __name__ == '__main__':
# Make socket
socket = TSocket.TSocket('mikuhime.xyz', 2233)
socket = TSocket.TSocket('sukura.top', 2233)
# Buffering is critical. Raw sockets are very slow
transport = TTransport.TFramedTransport(socket)
@ -22,4 +19,5 @@ if __name__ == '__main__':
# Create a client to use the protocol encoder
client = Client(protocol)
print(client.qryTest(1))
cids = [7636499, 7636501, 7636500, 7636503, 7636504, 7636502, 7636509, 7636508, 7636506, 7636507, 7636505]
print(client.commentSumMap(cids))

@ -9,24 +9,24 @@
import sys
import pprint
from urllib.parse import urlparse
from PixivSearch.thrift.TestQry import TestQry
sys.path.append('/root/PixivSearch')
if sys.version_info[0] > 2:
from urllib.parse import urlparse
else:
from urlparse import urlparse
from thrift.transport import TTransport, TSocket, TSSLSocket, THttpClient
from thrift.protocol.TBinaryProtocol import TBinaryProtocol
from QueryComment import QueryComment
from QueryComment.ttypes import *
if len(sys.argv) <= 1 or sys.argv[1] == '--help':
print('')
print('Usage: ' + sys.argv[0] + ' [-h host[:port]] [-u url] [-f[ramed]] [-s[sl]] [-novalidate] [-ca_certs certs] [-keyfile keyfile] [-certfile certfile] function [arg1 [arg2...]]')
print('')
print('Functions:')
print(' QryResult qryTest(i32 qryCode)')
print(' i32 commentSum(i32 cid)')
print(' commentSumList( cids)')
print(' string download( cids, string fileName)')
print('')
sys.exit(0)
@ -103,14 +103,26 @@ else:
else:
transport = TTransport.TBufferedTransport(socket)
protocol = TBinaryProtocol(transport)
client = TestQry.Client(protocol)
client = QueryComment.Client(protocol)
transport.open()
if cmd == 'qryTest':
if cmd == 'commentSum':
if len(args) != 1:
print('qryTest requires 1 args')
print('commentSum requires 1 args')
sys.exit(1)
pp.pprint(client.commentSum(eval(args[0]),))
elif cmd == 'commentSumList':
if len(args) != 1:
print('commentSumList requires 1 args')
sys.exit(1)
pp.pprint(client.commentSumList(eval(args[0]),))
elif cmd == 'download':
if len(args) != 2:
print('download requires 2 args')
sys.exit(1)
pp.pprint(client.qryTest(eval(args[0]),))
pp.pprint(client.download(eval(args[0]), args[1],))
else:
print('Unrecognized method %s' % cmd)

@ -0,0 +1,660 @@
#
# Autogenerated by Thrift Compiler (0.11.0)
#
# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
#
# options string: py
#
from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException
from thrift.protocol.TProtocol import TProtocolException
from thrift.TRecursive import fix_spec
import sys
import logging
from .ttypes import *
from thrift.Thrift import TProcessor
from thrift.transport import TTransport
all_structs = []
class Iface(object):
def commentSum(self, cid):
"""
查询单个cid弹幕数
Parameters:
- cid
"""
pass
def commentSumList(self, cids):
"""
* 批量查询多个cid弹幕数
*
Parameters:
- cids
"""
pass
def download(self, cids, fileName):
"""
* 批量下载弹幕
*
Parameters:
- cids
- fileName
"""
pass
class Client(Iface):
def __init__(self, iprot, oprot=None):
self._iprot = self._oprot = iprot
if oprot is not None:
self._oprot = oprot
self._seqid = 0
def commentSum(self, cid):
"""
查询单个cid弹幕数
Parameters:
- cid
"""
self.send_commentSum(cid)
return self.recv_commentSum()
def send_commentSum(self, cid):
self._oprot.writeMessageBegin('commentSum', TMessageType.CALL, self._seqid)
args = commentSum_args()
args.cid = cid
args.write(self._oprot)
self._oprot.writeMessageEnd()
self._oprot.trans.flush()
def recv_commentSum(self):
iprot = self._iprot
(fname, mtype, rseqid) = iprot.readMessageBegin()
if mtype == TMessageType.EXCEPTION:
x = TApplicationException()
x.read(iprot)
iprot.readMessageEnd()
raise x
result = commentSum_result()
result.read(iprot)
iprot.readMessageEnd()
if result.success is not None:
return result.success
raise TApplicationException(TApplicationException.MISSING_RESULT, "commentSum failed: unknown result")
def commentSumList(self, cids):
"""
* 批量查询多个cid弹幕数
*
Parameters:
- cids
"""
self.send_commentSumList(cids)
return self.recv_commentSumList()
def send_commentSumList(self, cids):
self._oprot.writeMessageBegin('commentSumList', TMessageType.CALL, self._seqid)
args = commentSumList_args()
args.cids = cids
args.write(self._oprot)
self._oprot.writeMessageEnd()
self._oprot.trans.flush()
def recv_commentSumList(self):
iprot = self._iprot
(fname, mtype, rseqid) = iprot.readMessageBegin()
if mtype == TMessageType.EXCEPTION:
x = TApplicationException()
x.read(iprot)
iprot.readMessageEnd()
raise x
result = commentSumList_result()
result.read(iprot)
iprot.readMessageEnd()
if result.success is not None:
return result.success
raise TApplicationException(TApplicationException.MISSING_RESULT, "commentSumList failed: unknown result")
def download(self, cids, fileName):
"""
* 批量下载弹幕
*
Parameters:
- cids
- fileName
"""
self.send_download(cids, fileName)
return self.recv_download()
def send_download(self, cids, fileName):
self._oprot.writeMessageBegin('download', TMessageType.CALL, self._seqid)
args = download_args()
args.cids = cids
args.fileName = fileName
args.write(self._oprot)
self._oprot.writeMessageEnd()
self._oprot.trans.flush()
def recv_download(self):
iprot = self._iprot
(fname, mtype, rseqid) = iprot.readMessageBegin()
if mtype == TMessageType.EXCEPTION:
x = TApplicationException()
x.read(iprot)
iprot.readMessageEnd()
raise x
result = download_result()
result.read(iprot)
iprot.readMessageEnd()
if result.success is not None:
return result.success
raise TApplicationException(TApplicationException.MISSING_RESULT, "download failed: unknown result")
class Processor(Iface, TProcessor):
def __init__(self, handler):
self._handler = handler
self._processMap = {}
self._processMap["commentSum"] = Processor.process_commentSum
self._processMap["commentSumList"] = Processor.process_commentSumList
self._processMap["download"] = Processor.process_download
def process(self, iprot, oprot):
(name, type, seqid) = iprot.readMessageBegin()
if name not in self._processMap:
iprot.skip(TType.STRUCT)
iprot.readMessageEnd()
x = TApplicationException(TApplicationException.UNKNOWN_METHOD, 'Unknown function %s' % (name))
oprot.writeMessageBegin(name, TMessageType.EXCEPTION, seqid)
x.write(oprot)
oprot.writeMessageEnd()
oprot.trans.flush()
return
else:
self._processMap[name](self, seqid, iprot, oprot)
return True
def process_commentSum(self, seqid, iprot, oprot):
args = commentSum_args()
args.read(iprot)
iprot.readMessageEnd()
result = commentSum_result()
try:
result.success = self._handler.commentSum(args.cid)
msg_type = TMessageType.REPLY
except TTransport.TTransportException:
raise
except TApplicationException as ex:
logging.exception('TApplication exception in handler')
msg_type = TMessageType.EXCEPTION
result = ex
except Exception:
logging.exception('Unexpected exception in handler')
msg_type = TMessageType.EXCEPTION
result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error')
oprot.writeMessageBegin("commentSum", msg_type, seqid)
result.write(oprot)
oprot.writeMessageEnd()
oprot.trans.flush()
def process_commentSumList(self, seqid, iprot, oprot):
args = commentSumList_args()
args.read(iprot)
iprot.readMessageEnd()
result = commentSumList_result()
try:
result.success = self._handler.commentSumList(args.cids)
msg_type = TMessageType.REPLY
except TTransport.TTransportException:
raise
except TApplicationException as ex:
logging.exception('TApplication exception in handler')
msg_type = TMessageType.EXCEPTION
result = ex
except Exception:
logging.exception('Unexpected exception in handler')
msg_type = TMessageType.EXCEPTION
result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error')
oprot.writeMessageBegin("commentSumList", msg_type, seqid)
result.write(oprot)
oprot.writeMessageEnd()
oprot.trans.flush()
def process_download(self, seqid, iprot, oprot):
args = download_args()
args.read(iprot)
iprot.readMessageEnd()
result = download_result()
try:
result.success = self._handler.download(args.cids, args.fileName)
msg_type = TMessageType.REPLY
except TTransport.TTransportException:
raise
except TApplicationException as ex:
logging.exception('TApplication exception in handler')
msg_type = TMessageType.EXCEPTION
result = ex
except Exception:
logging.exception('Unexpected exception in handler')
msg_type = TMessageType.EXCEPTION
result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error')
oprot.writeMessageBegin("download", msg_type, seqid)
result.write(oprot)
oprot.writeMessageEnd()
oprot.trans.flush()
# HELPER FUNCTIONS AND STRUCTURES
class commentSum_args(object):
"""
Attributes:
- cid
"""
def __init__(self, cid=None,):
self.cid = cid
def read(self, iprot):
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
return
iprot.readStructBegin()
while True:
(fname, ftype, fid) = iprot.readFieldBegin()
if ftype == TType.STOP:
break
if fid == 1:
if ftype == TType.I32:
self.cid = iprot.readI32()
else:
iprot.skip(ftype)
else:
iprot.skip(ftype)
iprot.readFieldEnd()
iprot.readStructEnd()
def write(self, oprot):
if oprot._fast_encode is not None and self.thrift_spec is not None:
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
return
oprot.writeStructBegin('commentSum_args')
if self.cid is not None:
oprot.writeFieldBegin('cid', TType.I32, 1)
oprot.writeI32(self.cid)
oprot.writeFieldEnd()
oprot.writeFieldStop()
oprot.writeStructEnd()
def validate(self):
return
def __repr__(self):
L = ['%s=%r' % (key, value)
for key, value in self.__dict__.items()]
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
def __eq__(self, other):
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
def __ne__(self, other):
return not (self == other)
all_structs.append(commentSum_args)
commentSum_args.thrift_spec = (
None, # 0
(1, TType.I32, 'cid', None, None, ), # 1
)
class commentSum_result(object):
"""
Attributes:
- success
"""
def __init__(self, success=None,):
self.success = success
def read(self, iprot):
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
return
iprot.readStructBegin()
while True:
(fname, ftype, fid) = iprot.readFieldBegin()
if ftype == TType.STOP:
break
if fid == 0:
if ftype == TType.I32:
self.success = iprot.readI32()
else:
iprot.skip(ftype)
else:
iprot.skip(ftype)
iprot.readFieldEnd()
iprot.readStructEnd()
def write(self, oprot):
if oprot._fast_encode is not None and self.thrift_spec is not None:
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
return
oprot.writeStructBegin('commentSum_result')
if self.success is not None:
oprot.writeFieldBegin('success', TType.I32, 0)
oprot.writeI32(self.success)
oprot.writeFieldEnd()
oprot.writeFieldStop()
oprot.writeStructEnd()
def validate(self):
return
def __repr__(self):
L = ['%s=%r' % (key, value)
for key, value in self.__dict__.items()]
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
def __eq__(self, other):
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
def __ne__(self, other):
return not (self == other)
all_structs.append(commentSum_result)
commentSum_result.thrift_spec = (
(0, TType.I32, 'success', None, None, ), # 0
)
class commentSumList_args(object):
"""
Attributes:
- cids
"""
def __init__(self, cids=None,):
self.cids = cids
def read(self, iprot):
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
return
iprot.readStructBegin()
while True:
(fname, ftype, fid) = iprot.readFieldBegin()
if ftype == TType.STOP:
break
if fid == 1:
if ftype == TType.LIST:
self.cids = []
(_etype3, _size0) = iprot.readListBegin()
for _i4 in range(_size0):
_elem5 = iprot.readI32()
self.cids.append(_elem5)
iprot.readListEnd()
else:
iprot.skip(ftype)
else:
iprot.skip(ftype)
iprot.readFieldEnd()
iprot.readStructEnd()
def write(self, oprot):
if oprot._fast_encode is not None and self.thrift_spec is not None:
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
return
oprot.writeStructBegin('commentSumList_args')
if self.cids is not None:
oprot.writeFieldBegin('cids', TType.LIST, 1)
oprot.writeListBegin(TType.I32, len(self.cids))
for iter6 in self.cids:
oprot.writeI32(iter6)
oprot.writeListEnd()
oprot.writeFieldEnd()
oprot.writeFieldStop()
oprot.writeStructEnd()
def validate(self):
return
def __repr__(self):
L = ['%s=%r' % (key, value)
for key, value in self.__dict__.items()]
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
def __eq__(self, other):
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
def __ne__(self, other):
return not (self == other)
all_structs.append(commentSumList_args)
commentSumList_args.thrift_spec = (
None, # 0
(1, TType.LIST, 'cids', (TType.I32, None, False), None, ), # 1
)
class commentSumList_result(object):
"""
Attributes:
- success
"""
def __init__(self, success=None,):
self.success = success
def read(self, iprot):
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
return
iprot.readStructBegin()
while True:
(fname, ftype, fid) = iprot.readFieldBegin()
if ftype == TType.STOP:
break
if fid == 0:
if ftype == TType.LIST:
self.success = []
(_etype10, _size7) = iprot.readListBegin()
for _i11 in range(_size7):
_elem12 = iprot.readI32()
self.success.append(_elem12)
iprot.readListEnd()
else:
iprot.skip(ftype)
else:
iprot.skip(ftype)
iprot.readFieldEnd()
iprot.readStructEnd()
def write(self, oprot):
if oprot._fast_encode is not None and self.thrift_spec is not None:
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
return
oprot.writeStructBegin('commentSumList_result')
if self.success is not None:
oprot.writeFieldBegin('success', TType.LIST, 0)
oprot.writeListBegin(TType.I32, len(self.success))
for iter13 in self.success:
oprot.writeI32(iter13)
oprot.writeListEnd()
oprot.writeFieldEnd()
oprot.writeFieldStop()
oprot.writeStructEnd()
def validate(self):
return
def __repr__(self):
L = ['%s=%r' % (key, value)
for key, value in self.__dict__.items()]
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
def __eq__(self, other):
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
def __ne__(self, other):
return not (self == other)
all_structs.append(commentSumList_result)
commentSumList_result.thrift_spec = (
(0, TType.LIST, 'success', (TType.I32, None, False), None, ), # 0
)
class download_args(object):
"""
Attributes:
- cids
- fileName
"""
def __init__(self, cids=None, fileName=None,):
self.cids = cids
self.fileName = fileName
def read(self, iprot):
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
return
iprot.readStructBegin()
while True:
(fname, ftype, fid) = iprot.readFieldBegin()
if ftype == TType.STOP:
break
if fid == 1:
if ftype == TType.LIST:
self.cids = []
(_etype17, _size14) = iprot.readListBegin()
for _i18 in range(_size14):
_elem19 = iprot.readI32()
self.cids.append(_elem19)
iprot.readListEnd()
else:
iprot.skip(ftype)
elif fid == 2:
if ftype == TType.STRING:
self.fileName = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString()
else:
iprot.skip(ftype)
else:
iprot.skip(ftype)
iprot.readFieldEnd()
iprot.readStructEnd()
def write(self, oprot):
if oprot._fast_encode is not None and self.thrift_spec is not None:
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
return
oprot.writeStructBegin('download_args')
if self.cids is not None:
oprot.writeFieldBegin('cids', TType.LIST, 1)
oprot.writeListBegin(TType.I32, len(self.cids))
for iter20 in self.cids:
oprot.writeI32(iter20)
oprot.writeListEnd()
oprot.writeFieldEnd()
if self.fileName is not None:
oprot.writeFieldBegin('fileName', TType.STRING, 2)
oprot.writeString(self.fileName.encode('utf-8') if sys.version_info[0] == 2 else self.fileName)
oprot.writeFieldEnd()
oprot.writeFieldStop()
oprot.writeStructEnd()
def validate(self):
return
def __repr__(self):
L = ['%s=%r' % (key, value)
for key, value in self.__dict__.items()]
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
def __eq__(self, other):
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
def __ne__(self, other):
return not (self == other)
all_structs.append(download_args)
download_args.thrift_spec = (
None, # 0
(1, TType.LIST, 'cids', (TType.I32, None, False), None, ), # 1
(2, TType.STRING, 'fileName', 'UTF8', None, ), # 2
)
class download_result(object):
"""
Attributes:
- success
"""
def __init__(self, success=None,):
self.success = success
def read(self, iprot):
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
return
iprot.readStructBegin()
while True:
(fname, ftype, fid) = iprot.readFieldBegin()
if ftype == TType.STOP:
break
if fid == 0:
if ftype == TType.STRING:
self.success = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString()
else:
iprot.skip(ftype)
else:
iprot.skip(ftype)
iprot.readFieldEnd()
iprot.readStructEnd()
def write(self, oprot):
if oprot._fast_encode is not None and self.thrift_spec is not None:
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
return
oprot.writeStructBegin('download_result')
if self.success is not None:
oprot.writeFieldBegin('success', TType.STRING, 0)
oprot.writeString(self.success.encode('utf-8') if sys.version_info[0] == 2 else self.success)
oprot.writeFieldEnd()
oprot.writeFieldStop()
oprot.writeStructEnd()
def validate(self):
return
def __repr__(self):
L = ['%s=%r' % (key, value)
for key, value in self.__dict__.items()]
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
def __eq__(self, other):
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
def __ne__(self, other):
return not (self == other)
all_structs.append(download_result)
download_result.thrift_spec = (
(0, TType.STRING, 'success', 'UTF8', None, ), # 0
)
fix_spec(all_structs)
del all_structs

@ -0,0 +1,48 @@
import os
import sys
from thrift.server.TNonblockingServer import TNonblockingServer
from thrift.transport import TSocket
sys.path.append('/root/PixivSearch')
from PixivSearch.util import Util
from PixivSearch.dao.Comment import xml, downloadXml
from PixivSearch.thrift.QueryComment import QueryComment
from PixivSearch.thrift.QueryComment.QueryComment import Iface
commentPath = '/root/PixivSearch/PixivSearch/thrift/tmpFile/comment'
class Server(Iface):
def commentSumList(self, cids):
result = []
for cid in cids:
comment_selector = xml('https://comment.bilibili.com/{cid}.xml'.format(cid=cid))
length = len(comment_selector.xpath('//i//d/text()'))
print('cid:{cid},弹幕数:{length}'.format(cid=cid, length=length))
result.append(length)
return result
def commentSum(self, cid):
return self.commentSumList([cid])[0];
def download(self, cids, fileName):
path = '{commentPath}/{fileName}'.format(commentPath=commentPath, fileName=fileName)
for cid in cids:
downloadXml(path, cid,
histroy=False)
zipFile = '{path}/{fileName}.zip'.format(path=path,fileName=fileName)
print(zipFile)
Util.zip(path, zipFile)
if os.path.isfile(zipFile):
print('压缩包成功生成到{zipFile}'.format(zipFile=zipFile))
return zipFile
else:
return None
if __name__ == '__main__':
socket = TSocket.TServerSocket(port=2233)
processor = QueryComment.Processor(Server())
server = TNonblockingServer(processor, socket)
server.serve()

@ -0,0 +1 @@
__all__ = ['ttypes', 'constants', 'QueryComment']

@ -0,0 +1,18 @@
#
# Autogenerated by Thrift Compiler (0.11.0)
#
# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
#
# options string: py
#
from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException
from thrift.protocol.TProtocol import TProtocolException
from thrift.TRecursive import fix_spec
import sys
from thrift.transport import TTransport
all_structs = []
fix_spec(all_structs)
del all_structs

@ -1,30 +0,0 @@
import sys
from thrift.server.TNonblockingServer import TNonblockingServer
from thrift.transport import TSocket
sys.path.append('/root/PixivSearch')
from PixivSearch.thrift.TestQry import TestQry
from PixivSearch.thrift.TestQry.ttypes import QryResult
class QueryImpl(TestQry.Iface):
def qryTest(self, qryCode):
result = QryResult()
if qryCode == 1:
result.code = 1
result.msg = 'success'
else:
result.code = 0
result.msg = 'fail'
return result
if __name__ == '__main__':
socket = TSocket.TServerSocket(port=2233)
processor = TestQry.Processor(QueryImpl())
server = TNonblockingServer(processor, socket)
server.serve()

@ -1,245 +0,0 @@
#
# Autogenerated by Thrift Compiler (0.11.0)
#
# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
#
# options string: py
#
import logging
from thrift.TRecursive import fix_spec
from thrift.Thrift import TProcessor
from thrift.Thrift import TType, TMessageType, TApplicationException
from thrift.transport import TTransport
from PixivSearch.thrift.TestQry.ttypes import QryResult
all_structs = []
class Iface(object):
def qryTest(self, qryCode):
"""
测试查询接口当qryCode值为1时返回"成功"的响应信息qryCode值为其他值时返回"失败"的响应信息
@param qryCode测试参数
Parameters:
- qryCode
"""
pass
class Client(Iface):
def __init__(self, iprot, oprot=None):
self._iprot = self._oprot = iprot
if oprot is not None:
self._oprot = oprot
self._seqid = 0
def qryTest(self, qryCode):
"""
测试查询接口当qryCode值为1时返回"成功"的响应信息qryCode值为其他值时返回"失败"的响应信息
@param qryCode测试参数
Parameters:
- qryCode
"""
self.send_qryTest(qryCode)
return self.recv_qryTest()
def send_qryTest(self, qryCode):
self._oprot.writeMessageBegin('qryTest', TMessageType.CALL, self._seqid)
args = qryTest_args()
args.qryCode = qryCode
args.write(self._oprot)
self._oprot.writeMessageEnd()
self._oprot.trans.flush()
def recv_qryTest(self):
iprot = self._iprot
(fname, mtype, rseqid) = iprot.readMessageBegin()
if mtype == TMessageType.EXCEPTION:
x = TApplicationException()
x.read(iprot)
iprot.readMessageEnd()
raise x
result = qryTest_result()
result.read(iprot)
iprot.readMessageEnd()
if result.success is not None:
return result.success
raise TApplicationException(TApplicationException.MISSING_RESULT, "qryTest failed: unknown result")
class Processor(Iface, TProcessor):
def __init__(self, handler):
self._handler = handler
self._processMap = {}
self._processMap["qryTest"] = Processor.process_qryTest
def process(self, iprot, oprot):
(name, type, seqid) = iprot.readMessageBegin()
if name not in self._processMap:
iprot.skip(TType.STRUCT)
iprot.readMessageEnd()
x = TApplicationException(TApplicationException.UNKNOWN_METHOD, 'Unknown function %s' % (name))
oprot.writeMessageBegin(name, TMessageType.EXCEPTION, seqid)
x.write(oprot)
oprot.writeMessageEnd()
oprot.trans.flush()
return
else:
self._processMap[name](self, seqid, iprot, oprot)
return True
def process_qryTest(self, seqid, iprot, oprot):
args = qryTest_args()
args.read(iprot)
iprot.readMessageEnd()
result = qryTest_result()
try:
result.success = self._handler.qryTest(args.qryCode)
msg_type = TMessageType.REPLY
except TTransport.TTransportException:
raise
except TApplicationException as ex:
logging.exception('TApplication exception in handler')
msg_type = TMessageType.EXCEPTION
result = ex
except Exception:
logging.exception('Unexpected exception in handler')
msg_type = TMessageType.EXCEPTION
result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error')
oprot.writeMessageBegin("qryTest", msg_type, seqid)
result.write(oprot)
oprot.writeMessageEnd()
oprot.trans.flush()
# HELPER FUNCTIONS AND STRUCTURES
class qryTest_args(object):
"""
Attributes:
- qryCode
"""
def __init__(self, qryCode=None,):
self.qryCode = qryCode
def read(self, iprot):
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
return
iprot.readStructBegin()
while True:
(fname, ftype, fid) = iprot.readFieldBegin()
if ftype == TType.STOP:
break
if fid == 1:
if ftype == TType.I32:
self.qryCode = iprot.readI32()
else:
iprot.skip(ftype)
else:
iprot.skip(ftype)
iprot.readFieldEnd()
iprot.readStructEnd()
def write(self, oprot):
if oprot._fast_encode is not None and self.thrift_spec is not None:
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
return
oprot.writeStructBegin('qryTest_args')
if self.qryCode is not None:
oprot.writeFieldBegin('qryCode', TType.I32, 1)
oprot.writeI32(self.qryCode)
oprot.writeFieldEnd()
oprot.writeFieldStop()
oprot.writeStructEnd()
def validate(self):
return
def __repr__(self):
L = ['%s=%r' % (key, value)
for key, value in self.__dict__.items()]
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
def __eq__(self, other):
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
def __ne__(self, other):
return not (self == other)
all_structs.append(qryTest_args)
qryTest_args.thrift_spec = (
None, # 0
(1, TType.I32, 'qryCode', None, None, ), # 1
)
class qryTest_result(object):
"""
Attributes:
- success
"""
def __init__(self, success=None,):
self.success = success
def read(self, iprot):
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
return
iprot.readStructBegin()
while True:
(fname, ftype, fid) = iprot.readFieldBegin()
if ftype == TType.STOP:
break
if fid == 0:
if ftype == TType.STRUCT:
self.success = QryResult()
self.success.read(iprot)
else:
iprot.skip(ftype)
else:
iprot.skip(ftype)
iprot.readFieldEnd()
iprot.readStructEnd()
def write(self, oprot):
if oprot._fast_encode is not None and self.thrift_spec is not None:
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
return
oprot.writeStructBegin('qryTest_result')
if self.success is not None:
oprot.writeFieldBegin('success', TType.STRUCT, 0)
self.success.write(oprot)
oprot.writeFieldEnd()
oprot.writeFieldStop()
oprot.writeStructEnd()
def validate(self):
return
def __repr__(self):
L = ['%s=%r' % (key, value)
for key, value in self.__dict__.items()]
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
def __eq__(self, other):
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
def __ne__(self, other):
return not (self == other)
all_structs.append(qryTest_result)
qryTest_result.thrift_spec = (
(0, TType.STRUCT, 'success', [QryResult, None], None, ), # 0
)
fix_spec(all_structs)
del all_structs

@ -1 +0,0 @@
__all__ = ['ttypes', 'constants', 'TestQry']

@ -1,91 +0,0 @@
#
# Autogenerated by Thrift Compiler (0.11.0)
#
# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
#
# options string: py
#
from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException
from thrift.protocol.TProtocol import TProtocolException
from thrift.TRecursive import fix_spec
import sys
from thrift.transport import TTransport
all_structs = []
class QryResult(object):
"""
Attributes:
- code: 返回码, 1成功0失败
- msg: 响应信息
"""
def __init__(self, code=None, msg=None,):
self.code = code
self.msg = msg
def read(self, iprot):
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
return
iprot.readStructBegin()
while True:
(fname, ftype, fid) = iprot.readFieldBegin()
if ftype == TType.STOP:
break
if fid == 1:
if ftype == TType.I32:
self.code = iprot.readI32()
else:
iprot.skip(ftype)
elif fid == 2:
if ftype == TType.STRING:
self.msg = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString()
else:
iprot.skip(ftype)
else:
iprot.skip(ftype)
iprot.readFieldEnd()
iprot.readStructEnd()
def write(self, oprot):
if oprot._fast_encode is not None and self.thrift_spec is not None:
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
return
oprot.writeStructBegin('QryResult')
if self.code is not None:
oprot.writeFieldBegin('code', TType.I32, 1)
oprot.writeI32(self.code)
oprot.writeFieldEnd()
if self.msg is not None:
oprot.writeFieldBegin('msg', TType.STRING, 2)
oprot.writeString(self.msg.encode('utf-8') if sys.version_info[0] == 2 else self.msg)
oprot.writeFieldEnd()
oprot.writeFieldStop()
oprot.writeStructEnd()
def validate(self):
return
def __repr__(self):
L = ['%s=%r' % (key, value)
for key, value in self.__dict__.items()]
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
def __eq__(self, other):
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
def __ne__(self, other):
return not (self == other)
all_structs.append(QryResult)
QryResult.thrift_spec = (
None, # 0
(1, TType.I32, 'code', None, None, ), # 1
(2, TType.STRING, 'msg', 'UTF8', None, ), # 2
)
fix_spec(all_structs)
del all_structs

@ -0,0 +1,10 @@
import os
import zipfile
def zip(inputFile, outFile):
f = zipfile.ZipFile(outFile, 'w', zipfile.ZIP_DEFLATED)
for dirpath, dirnames, filenames in os.walk(inputFile):
for filename in filenames:
f.write(os.path.join(dirpath, filename), filename)
f.close()

@ -7,13 +7,13 @@ from django.http import Http404, StreamingHttpResponse, HttpResponse
from django.shortcuts import render
from PixivSearch.dao.bangumi import get_, stop_, getIds
from PixivSearch.pixiv.pixiv import get_nodes
from PixivSearch.settings import logger
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "PixivSearch.settings")
django.setup() # 添加的代码
from PixivSearch.model import config
from PixivSearch.pixiv import get_nodes
def search(request):

Loading…
Cancel
Save