1029559041@qq.com
6 years ago
commit
7322d7107a
@ -0,0 +1,140 @@ |
||||
import hashlib |
||||
import json |
||||
import os |
||||
import time |
||||
|
||||
import aliyunsdkcore |
||||
import oss2 as oss2 |
||||
from aliyunsdkcloudphoto.request.v20170711 import ListPhotoStoresRequest, FetchLibrariesRequest, \ |
||||
CreateTransactionRequest, CreatePhotoRequest, EditPhotosRequest, GetPublicAccessUrlsRequest, ListPhotosRequest, \ |
||||
DeletePhotosRequest, InactivatePhotosRequest, GetLibraryRequest |
||||
from aliyunsdkcore.auth.credentials import RamRoleArnCredential |
||||
from aliyunsdkcore.client import AcsClient |
||||
from aliyunsdkcore.profile import region_provider |
||||
from aliyunsdkcore.request import CommonRequest |
||||
|
||||
region_id = 'cn-shanghai' |
||||
ak = 'LTAIWzPnmkJs2qpL' |
||||
secret = 'LIIq3HumctXPp0WT8c06yDiFbKKiVe' |
||||
region_provider.add_endpoint('Oss', region_id, 'oss-cn-shanghai.aliyuncs.com') |
||||
region_provider.add_endpoint('CloudPhoto', region_id, 'cloudphoto.cn-shanghai.aliyuncs.com') |
||||
aliyunsdkcore.request.set_default_protocol_type("https") |
||||
|
||||
|
||||
class MyClient(AcsClient): |
||||
|
||||
def __init__(self, arn, sessionName): |
||||
super().__init__(region_id=region_id, credential=RamRoleArnCredential(ak, secret, arn, sessionName)) |
||||
self.StoreName = None |
||||
|
||||
def get_md5_01(self, file_path): |
||||
md5 = None |
||||
if os.path.isfile(file_path): |
||||
f = open(file_path, 'rb') |
||||
md5_obj = hashlib.md5() |
||||
md5_obj.update(f.read()) |
||||
hash_code = md5_obj.hexdigest() |
||||
f.close() |
||||
md5 = str(hash_code).lower() |
||||
return md5 |
||||
|
||||
def do_action_with_exception(self, acs_request): |
||||
return json.loads(super().do_action_with_exception(acs_request).decode()) |
||||
|
||||
def showOption(self, options, key, desc): |
||||
choose = [] |
||||
for index, option in enumerate(options): |
||||
choose.append(option[key]) |
||||
print('{index}:{name}'.format(index=index + 1, name=option[key])) |
||||
return choose[(int)(input('输入{desc}\n'.format(desc=desc))) - 1] |
||||
|
||||
def listPhotoStores(self): |
||||
request = ListPhotoStoresRequest.ListPhotoStoresRequest() |
||||
response = self.do_action_with_exception(request) |
||||
print('PhotoStores:') |
||||
photoStores = response['PhotoStores'] |
||||
self.StoreName = self.showOption(photoStores, 'Name', 'StoreName') |
||||
|
||||
def listLibraries(self): |
||||
request = FetchLibrariesRequest.FetchLibrariesRequest() |
||||
request.set_StoreName(self.StoreName) |
||||
response = self.do_action_with_exception(request) |
||||
self.Libraries = response['Libraries'] |
||||
|
||||
def uploadPhoto(self): |
||||
if self.StoreName is None: |
||||
self.listPhotoStores() |
||||
request = CreateTransactionRequest.CreateTransactionRequest() |
||||
filePath = input('输入上传文件路径\n').replace('\\', '/') |
||||
fileName = filePath.split('/')[-1] |
||||
request.set_Size(os.path.getsize(filePath)) |
||||
request.set_Ext(fileName[-fileName[::-1].index('.'):]) |
||||
request.set_Md5(self.get_md5_01(filePath)) |
||||
request.set_StoreName(self.StoreName) |
||||
response = self.do_action_with_exception(request) |
||||
print(response) |
||||
Upload = response['Transaction']['Upload'] |
||||
FileId = Upload['FileId'] |
||||
SessionId = Upload['SessionId'] |
||||
Bucket = Upload['Bucket'] |
||||
OssEndpoint = Upload['OssEndpoint'] |
||||
ObjectKey = Upload['ObjectKey'] |
||||
|
||||
auth = oss2.StsAuth(self._signer._session_credential[0], self._signer._session_credential[1], |
||||
self._signer._session_credential[2]) |
||||
bucket = oss2.Bucket(auth, OssEndpoint, Bucket) |
||||
with open(filePath, 'rb') as fileobj: |
||||
result = bucket.put_object(ObjectKey, fileobj) |
||||
print('文件上传状态{status}'.format(status=result.status)) |
||||
|
||||
request = CreatePhotoRequest.CreatePhotoRequest() |
||||
request.set_FileId(FileId) |
||||
request.set_PhotoTitle(fileName) |
||||
request.set_SessionId(SessionId) |
||||
request.set_StoreName(self.StoreName) |
||||
request.set_UploadType('manual') |
||||
response = self.do_action_with_exception(request) |
||||
print(response) |
||||
|
||||
def listPhotos(self): |
||||
if self.StoreName == None: |
||||
self.listPhotoStores() |
||||
request = ListPhotosRequest.ListPhotosRequest() |
||||
request.set_StoreName(self.StoreName) |
||||
request.set_State('all') |
||||
response = self.do_action_with_exception(request) |
||||
print(response) |
||||
return response['Photos'] |
||||
|
||||
def getPhoto(self): |
||||
return self.showOption(self.listPhotos(), 'IdStr', '照片Id') |
||||
|
||||
def sharePhoto(self): |
||||
IdStr = self.getPhoto() |
||||
request = EditPhotosRequest.EditPhotosRequest() |
||||
request.set_PhotoIds([IdStr]) |
||||
request.set_StoreName(self.StoreName) |
||||
request.set_ShareExpireTime((int(round(time.time())) + 60 * 60) * 1000) |
||||
response = self.do_action_with_exception(request) |
||||
print(response) |
||||
|
||||
request = GetPublicAccessUrlsRequest.GetPublicAccessUrlsRequest() |
||||
request.set_DomainType('OSS') |
||||
request.set_PhotoIds([IdStr]) |
||||
request.set_StoreName(self.StoreName) |
||||
request.set_ZoomType('style/2') |
||||
response = self.do_action_with_exception(request) |
||||
print(response) |
||||
|
||||
|
||||
def client(arn, sessionName): |
||||
ram_role_arn_credential = RamRoleArnCredential('LTAIWzPnmkJs2qpL', 'LIIq3HumctXPp0WT8c06yDiFbKKiVe', |
||||
arn, sessionName) |
||||
return AcsClient(region_id='cn-shanghai', credential=ram_role_arn_credential) |
||||
|
||||
|
||||
if __name__ == '__main__': |
||||
|
||||
myClient = MyClient('acs:ram::1098806312754985:role/aliyunosstokengeneratorrole', 'pqh001') |
||||
myClient.listPhotoStores() |
||||
myClient.sharePhoto() |
||||
@ -0,0 +1,118 @@ |
||||
import json |
||||
import os |
||||
import re |
||||
from concurrent import futures |
||||
from datetime import datetime |
||||
|
||||
import requests |
||||
|
||||
|
||||
def get_cookies(): |
||||
_cookies = {} |
||||
array = "BDqhfp=fate%26%26NaN%26%260%26%261; BIDUPSID=8689C23BFD1526702A4EF173F3A809DD; BDRCVFR[dG2JNJb_ajR]=mk3SLVN4HKm; userFrom=null; BAIDUID=8689C23BFD152670722FAAEB4DDC55FA:FG=1; BDRCVFR[-pGxjrCMryR]=mk3SLVN4HKm".split( |
||||
';') |
||||
for row in array: |
||||
k, v = row.strip().split('=', 1) |
||||
_cookies[k] = v |
||||
return _cookies |
||||
|
||||
|
||||
# 图片保存路径 |
||||
savePath = None |
||||
threadNum = 10 |
||||
startTime = None |
||||
|
||||
|
||||
def getBaiduImage(word): |
||||
global startTime, savePath |
||||
params = [] |
||||
startTime = datetime.now() |
||||
start = threadNum |
||||
i = 0 |
||||
filepath = None |
||||
savePath = r'{savePath}/{word}'.format(savePath=savePath, word=word) |
||||
if not os.path.exists(savePath): |
||||
os.makedirs(savePath) |
||||
while True: |
||||
try: |
||||
url = r"https://image.baidu.com/search/acjson?tn=resultjson_com&ipn=rj&ct=201326592&is=&fp=result&queryWord={" \ |
||||
r"queryWord}&cl=2&lm=-1&ie=utf-8&oe=utf-8&adpicid=&st=-1&z=&ic=0&word={" \ |
||||
r"word}&s=&se=&tab=&width=&height=&face=0&istype=2&qc=&nc=1&fr=&pn={pn}&rn={rn}&gsm=3c&1523890541764= " |
||||
url = url.format(queryWord=word, word=word, pn=start, rn=threadNum) |
||||
print('request url:%s' % url) |
||||
req = requests.get(url) |
||||
if req.status_code == 200: |
||||
req.encoding = 'utf-8' |
||||
obj = json.loads(req.text.replace('\\\'', '')) |
||||
if len(obj['data']) == 1: |
||||
break |
||||
for img in obj['data']: |
||||
if 'fromPageTitle' in img: |
||||
print('图片:%s\t添加到下载队列' % img['fromPageTitleEnc']) |
||||
if 'replaceUrl' in img: |
||||
url = img['replaceUrl'][0]['ObjURL'] |
||||
params.append((url, i)) |
||||
i += 1 |
||||
if not filepath is None and os.path.exists(filepath): |
||||
os.remove(filepath) |
||||
filepath = r'{savePath}/图片下载队列填充：{i}'.format(savePath=savePath, word=word, i=i) |
||||
file = open(filepath, 'w') |
||||
file.close() |
||||
start += threadNum |
||||
except BaseException as e: |
||||
print(repr(e)) |
||||
if not filepath is None and os.path.exists(filepath): |
||||
os.remove(filepath) |
||||
executors = futures.ThreadPoolExecutor(threadNum) |
||||
try: |
||||
with executors as executor: |
||||
executor.map(downImage, params) |
||||
except BaseException as e: |
||||
print(repr(e)) |
||||
|
||||
|
||||
def downImage(params): |
||||
try: |
||||
url = params[0] |
||||
index = params[1] |
||||
print(r'开始下载图片{url}'.format(url=url)) |
||||
imgurl = requests.get(url, headers={"Referer": "image.baidu.com"}) |
||||
if imgurl.status_code == 200: |
||||
format = url[-url[::-1].index('.'):] |
||||
imgPath = r'{savePath}/fate_{index}.{format}'.format(savePath=savePath, |
||||
index=index, |
||||
format=format) |
||||
f = open(imgPath, 'wb') |
||||
f.write(imgurl.content) |
||||
f.close() |
||||
print(r'图片{url}成功下载到{imgPath}'.format(url=url, imgPath=imgPath)) |
||||
except BaseException as e: |
||||
print(repr(e)) |
||||
|
||||
|
||||
if __name__ == '__main__': |
||||
# str = '' |
||||
# while True: |
||||
# str = input('输入要下载图片的关键字，输入 exit 退出程序\n') |
||||
# if not str == 'exit': |
||||
# while True: |
||||
# savePath = input('输入图片存放目录：例如 E:/图片,注意目录之间使用正斜杠隔开"/"\n') |
||||
# if re.fullmatch(r"[a-zA-z]:(/[\u4e00-\u9fa5_a-zA-Z0-9]+)+", savePath) is None: |
||||
# print(r'图片目录{savePath}不合法请重新输入'.format(savePath=savePath)) |
||||
# else: |
||||
# break |
||||
# getBaiduImage(str) |
||||
# print(r'使用{threadNum}线程成功下载{count}张图片到目录{path}下，耗时：{second}'.format(threadNum=threadNum, |
||||
# count=len(os.listdir(savePath)), |
||||
# path=savePath, |
||||
# second=datetime.now() - startTime)) |
||||
# flag = False |
||||
# else: |
||||
# print('exit') |
||||
# break |
||||
req = requests.post('https://gifmaker.develophelper.com/gif/make', {'tplid': 1, |
||||
'content': '好啊##$@?$?@$##就算你是一流工程师##$@?$?@$##就算你出报告再完美##$@?$?@$##我叫你改报告你就要改##$@?$?@$##毕竟我是客户##$@?$?@$##客户了不起啊##$@?$?@$##sorry 客户真的了不起##$@?$?@$##以后叫他天天改报告##$@?$?@$##天天改 天天改'}) |
||||
if req.status_code==200: |
||||
response = json.loads(req.content.decode()) |
||||
if 'd' in response: |
||||
print(response['d']) |
||||
@ -0,0 +1,119 @@ |
||||
import json |
||||
import os |
||||
import shutil |
||||
import threading |
||||
from concurrent import futures |
||||
|
||||
import requests |
||||
from lxml import etree |
||||
|
||||
|
||||
class Comment: |
||||
lock = threading.Lock() # 多线程全局资源锁 |
||||
|
||||
def __init__(self, keywords_=None) -> None: |
||||
super().__init__() |
||||
self.obj = {'data': {}, 'flag': False} |
||||
self.keywords = keywords_ |
||||
|
||||
# 获取番剧合集弹幕排行榜 |
||||
def getCommentSort(self, cids): |
||||
|
||||
urls = [] |
||||
for cid in cids: |
||||
urls.extend(getCidUrls(cid)) |
||||
with futures.ThreadPoolExecutor(32) as executor: |
||||
executor.map(self.count, urls) |
||||
for index, data in enumerate( |
||||
sorted(self.obj["data"].items(), key=lambda d: d[1], reverse=True)[ |
||||
:50]): |
||||
print('{index}:{data}'.format(index=index + 1, data=data)) |
||||
|
||||
# 获取番剧合集弹幕排行榜 |
||||
def count(self, url, desc=None): |
||||
bytes = requests.get(url).content |
||||
comment_selector = etree.HTML(bytes) |
||||
if not desc is None: |
||||
print(desc) |
||||
print("url=%s" % url) |
||||
for comment in comment_selector.xpath('//i//d/text()'): |
||||
if comment in self.obj["data"]: |
||||
with self.lock: |
||||
self.obj["data"][comment] = self.obj["data"][comment] + 1 |
||||
else: |
||||
with self.lock: |
||||
self.obj["data"][comment] = 1 |
||||
if not self.obj["flag"]: |
||||
for keyword in self.keywords: |
||||
if keyword in comment: |
||||
self.obj["flag"] = True |
||||
|
||||
|
||||
# 根据cid获取历史弹幕地址 |
||||
def getCidUrls(cid): |
||||
urls = [] |
||||
url = "https://comment.bilibili.com/rolldate,%d" % cid |
||||
req = requests.get(url) |
||||
if len(req.text) > 0: |
||||
for i in json.loads(req.text): |
||||
urls.append("https://comment.bilibili.com/dmroll,%s,%d" % (i['timestamp'], cid)) |
||||
else: |
||||
urls.append("https://comment.bilibili.com/%d.xml" % cid) |
||||
return urls |
||||
|
||||
|
||||
# 下载弹幕 |
||||
def downloadXml(path, cid, size=None, histroy=True): |
||||
dlist = set() |
||||
flag = None |
||||
if histroy: |
||||
flag = parseXml(getCidUrls(cid), dlist, size) |
||||
else: |
||||
parseXml("https://comment.bilibili.com/%d.xml" % cid, dlist, size) |
||||
if size is None or (histroy and not size is None and flag): |
||||
if os.path.exists(path): |
||||
shutil.rmtree(path) |
||||
os.makedirs(path) |
||||
f = open('{path}/{cid}.xml'.format(path=path, cid=cid), 'wb') |
||||
f.write(b'<?xml version="1.0" encoding="UTF-8"?><i>') |
||||
for i in dlist: |
||||
f.write(('\r\n' + i).encode()) |
||||
f.write(b'\r\n</i>') |
||||
f.close() |
||||
|
||||
|
||||
def xml(url): |
||||
bytes = requests.get(url).content |
||||
return etree.HTML(bytes) |
||||
|
||||
|
||||
def parseXml(urls, dlist, size=None): |
||||
if isinstance(urls, str): |
||||
urls = [urls] |
||||
if not size is None: |
||||
size = float(size.strip('%')) / 100.0 |
||||
for url in urls: |
||||
comment_selector = xml(url) |
||||
list = comment_selector.xpath('//i//d/text()') |
||||
maxlimit = int(comment_selector.xpath('//i//maxlimit/text()')[0]) |
||||
|
||||
if len(list) > 0: |
||||
print('弹幕数：{list},最大弹幕数：{maxlimit},弹幕池填充：{p}'.format(list=len(list), maxlimit=maxlimit, |
||||
p='%.2f%%' % (len(list) / maxlimit * 100))) |
||||
for element in comment_selector.xpath('//i//d'): |
||||
if len(element.xpath("text()")) > 0: |
||||
fstr = '<d p="{p}">{content}</d>'.format(p=str(element.xpath("@p")[0]), |
||||
content=str(element.xpath("text()")[0])) |
||||
dlist.add(fstr) |
||||
|
||||
currentSize = len(dlist) / maxlimit |
||||
print('填充率：{l}'.format(l='%.2f%%' % (currentSize * 100))) |
||||
if not size is None and currentSize >= size: |
||||
return True |
||||
return False |
||||
|
||||
|
||||
if __name__ == '__main__': |
||||
cids = [7636499, 7636501, 7636500, 7636503, 7636504, 7636502, 7636509, 7636508, 7636506, 7636507, 7636505] |
||||
|
||||
downloadXml('F:/ABC',12026697,histroy=False) |
||||
@ -0,0 +1,185 @@ |
||||
import _thread |
||||
import json |
||||
import math |
||||
import os |
||||
from concurrent import futures |
||||
from queue import Queue |
||||
|
||||
import django |
||||
import requests |
||||
from bs4 import BeautifulSoup |
||||
|
||||
from PixivSearch.settings import logger |
||||
|
||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "PixivSearch.settings") |
||||
django.setup() |
||||
from PixivSearch.model.config import mediaInfo, stat |
||||
|
||||
current_mediaInfo = None |
||||
|
||||
isStop = None |
||||
|
||||
|
||||
def stop_(): |
||||
global isStop |
||||
isStop = True |
||||
|
||||
|
||||
def save(params): |
||||
if isStop: |
||||
return |
||||
logger.info(params) |
||||
bangumi_id = params[0] |
||||
season_id = params[1] |
||||
media_id = params[2] |
||||
|
||||
url = "https://www.bilibili.com/bangumi/media/md%d" % media_id |
||||
try: |
||||
req = requests.get(url, timeout=10) |
||||
except BaseException as e: |
||||
logger.error(repr(e)) |
||||
save(params) |
||||
logger.info("request_url=%s,status_code=%d" % (url, req.status_code)) |
||||
if req.status_code == 200: |
||||
json_obj = getJsonText(req, 3) |
||||
try: |
||||
if 'mediaInfo' in json_obj and 'stat' in json_obj['mediaInfo'] and 'chn_name' in json_obj['mediaInfo']: |
||||
stat_info = json_obj['mediaInfo']['stat'] |
||||
print(stat_info) |
||||
mediaInfo(bangumi_id=bangumi_id, season_id=season_id, media_id=media_id, |
||||
chn_name=json_obj['mediaInfo']['chn_name']).save() |
||||
global current_mediaInfo |
||||
current_mediaInfo = mediaInfo.objects.get(pk=season_id) |
||||
stat(id=season_id, danmakus=int(stat_info['danmakus']), favorites=stat_info['favorites'], |
||||
views=stat_info['views']).save() |
||||
except BaseException as e: |
||||
logger.error(repr(e)) |
||||
|
||||
|
||||
def getJsonText(req, index): |
||||
tag = BeautifulSoup(req.text, 'lxml') |
||||
script = tag.select("script")[index].text |
||||
json_str = script[script.index("=") + 1:script.index("function") - 2] |
||||
return json.loads(json_str) |
||||
|
||||
|
||||
def get_(): |
||||
global current_mediaInfo |
||||
return current_mediaInfo |
||||
|
||||
|
||||
page_size = 10 |
||||
queue = Queue(page_size) |
||||
|
||||
|
||||
def listen(): |
||||
while True: |
||||
ids = queue.get() |
||||
try: |
||||
executors = futures.ThreadPoolExecutor(page_size) |
||||
with executors as executor: |
||||
executor.map(save, ids) |
||||
logger.info('结束爬虫') |
||||
except BaseException as e: |
||||
logger.error(repr(e)) |
||||
|
||||
|
||||
_thread.start_new_thread(listen, ()) |
||||
|
||||
#遍历所有专题视频收藏数信息 |
||||
def getIds(): |
||||
seasonIdList = [] |
||||
page = 1 |
||||
pages = None |
||||
name = 'seasonListCallback' |
||||
global isStop |
||||
isStop = False |
||||
while isStop == False and (pages is None or page <= pages): |
||||
|
||||
url = 'https://bangumi.bilibili.com/web_api/season/index_global?page=%d&page_size=%d' % (page, page_size) |
||||
logger.info(url) |
||||
try: |
||||
req = requests.get(url, timeout=10) |
||||
if req.status_code == 200: |
||||
json_obj = json.loads(req.text) |
||||
if 'result' in json_obj and 'list' in json_obj['result']: |
||||
bangumiList = json_obj['result']['list'] |
||||
ids = [] |
||||
for bangumi in bangumiList: |
||||
if isStop: |
||||
break |
||||
if 'season_id' in bangumi: |
||||
season_id = int(bangumi['season_id']) |
||||
if season_id in seasonIdList: |
||||
continue |
||||
url = 'https://bangumi.bilibili.com/jsonp/seasoninfo/%d.ver?callback=%s&jsonp=jsonp' % ( |
||||
season_id, name) |
||||
logger.info(url) |
||||
req = requests.get(url, timeout=10) |
||||
if req.status_code == 200: |
||||
child_json_obj = json.loads( |
||||
req.text.replace('seasonListCallback(', '').replace(');', '')) |
||||
if 'result' in child_json_obj and 'bangumi_id' in child_json_obj['result']: |
||||
bangumi_id = int(child_json_obj['result']['bangumi_id']) |
||||
if 'media' in child_json_obj['result']: |
||||
media_id = int(child_json_obj['result']['media']['media_id']) |
||||
ids.append((bangumi_id, season_id, media_id)) |
||||
seasonIdList.append(season_id) |
||||
if pages is None and 'count' in json_obj['result']: |
||||
pages = int(math.ceil(int(json_obj['result']['count']) / page_size)) |
||||
page = page + 1 |
||||
logger.info('获取id数量%d' % len(ids)) |
||||
queue.put(ids) |
||||
except BaseException as e: |
||||
logger.error(repr(e)) |
||||
continue |
||||
|
||||
#根据aid获取cid |
||||
def getCid(aid, type=None): |
||||
while True and aid > 0: |
||||
url = "https://api.bilibili.com/x/web-interface/archive/stat?aid=%d" % aid |
||||
print(url) |
||||
req = requests.get(url) |
||||
code = json.loads(req.text)["code"] |
||||
if code == 0: |
||||
req = requests.get("https://www.bilibili.com/video/av%d" % aid) |
||||
if req.status_code == 200: |
||||
json_obj = getJsonText(req, 9) |
||||
if "videoData" in json_obj and "pages" in json_obj['videoData'] and len( |
||||
json_obj['videoData']['pages']) > 0 and "cid" in json_obj['videoData']['pages'][0]: |
||||
cid = json_obj['videoData']['pages'][0]['cid'] |
||||
print('cid=%s' % cid) |
||||
return cid |
||||
if type is None: |
||||
break |
||||
else: |
||||
if type: |
||||
aid = aid - 1 |
||||
else: |
||||
aid = aid + 1 |
||||
|
||||
#根据aid获取cid |
||||
def getCids(aid): |
||||
s = {"min": getCid(aid, True), "max": getCid(aid, False)} |
||||
return s |
||||
|
||||
#获取专题所有cid |
||||
def episodeIdToCid(episode_id): |
||||
cids = [] |
||||
url = "https://www.bilibili.com/bangumi/play/ep%d" % episode_id |
||||
print("url=%s" % url) |
||||
req = requests.get(url) |
||||
json_obj = getJsonText(req, 8) |
||||
if "epList" in json_obj: |
||||
for i in json_obj["epList"]: |
||||
cids.append(i['cid']) |
||||
return cids |
||||
|
||||
|
||||
|
||||
if __name__ == '__main__': |
||||
# print(getCids(29416)) |
||||
|
||||
req=requests.post('https://api.bilibili.com/x/v2/history/shadow/set','') |
||||
# obj = loadData([34807341], []) |
||||
|
||||
@ -0,0 +1,33 @@ |
||||
[loggers] |
||||
keys=root,main |
||||
|
||||
[handlers] |
||||
keys=consoleHandler,fileHandler |
||||
|
||||
[formatters] |
||||
keys=fmt |
||||
|
||||
[logger_root] |
||||
level=DEBUG |
||||
handlers=consoleHandler |
||||
|
||||
[logger_main] |
||||
level=DEBUG |
||||
qualname=file |
||||
handlers=fileHandler |
||||
|
||||
[handler_consoleHandler] |
||||
class=StreamHandler |
||||
level=INFO |
||||
formatter=fmt |
||||
args=(sys.stdout,) |
||||
|
||||
[handler_fileHandler] |
||||
class=FileHandler |
||||
level=DEBUG |
||||
formatter=fmt |
||||
args=('pixiv.log','a','utf-8',False) |
||||
|
||||
[formatter_fmt] |
||||
format=%(asctime)s - %(name)s - %(levelname)s - %(module)s:line:%(lineno)d - %(message)s |
||||
datefmt=%Y-%m-%d %H:%M:%S |
||||
@ -0,0 +1,43 @@ |
||||
from django.db import models |
||||
|
||||
|
||||
class param(models.Model): |
||||
param_name = models.CharField(max_length=10, primary_key=True) |
||||
param_value = models.CharField(max_length=128) |
||||
|
||||
|
||||
class stat(models.Model): |
||||
id = models.IntegerField(primary_key=True) |
||||
danmakus = models.IntegerField() |
||||
favorites = models.IntegerField() |
||||
views = models.IntegerField() |
||||
|
||||
# def json(self): |
||||
# return bangumi_stat(self['danmakus'],self['favorites'],self['views']) |
||||
|
||||
|
||||
class mediaInfo(models.Model): |
||||
bangumi_id = models.IntegerField() |
||||
season_id = models.IntegerField(primary_key=True) |
||||
media_id = models.IntegerField() |
||||
chn_name = models.CharField(max_length=128) |
||||
|
||||
def __str__(self) -> str: |
||||
i = {'media_id': self.id, 'chn_name': self.chn_name} |
||||
return i |
||||
|
||||
|
||||
class bangumi_list(models.Model): |
||||
season_id = models.IntegerField(primary_key=True) |
||||
badge = models.CharField(max_length=128) |
||||
brief = models.CharField(max_length=128) |
||||
copyright = models.CharField(max_length=128) |
||||
cover = models.CharField(max_length=128) |
||||
favorites = models.IntegerField() |
||||
is_finish = models.IntegerField() |
||||
newest_ep_index = models.IntegerField() |
||||
pub_time = models.DateTimeField() |
||||
season_status = models.IntegerField() |
||||
title = models.CharField(max_length=128) |
||||
total_count = models.IntegerField() |
||||
trailer_aid = models.IntegerField() |
||||
@ -0,0 +1,57 @@ |
||||
# Generated by Django 2.0 on 2018-03-24 17:02 |
||||
|
||||
from django.db import migrations, models |
||||
|
||||
|
||||
class Migration(migrations.Migration): |
||||
|
||||
initial = True |
||||
|
||||
dependencies = [ |
||||
] |
||||
|
||||
operations = [ |
||||
migrations.CreateModel( |
||||
name='bangumi_list', |
||||
fields=[ |
||||
('season_id', models.IntegerField(primary_key=True, serialize=False)), |
||||
('badge', models.CharField(max_length=128)), |
||||
('brief', models.CharField(max_length=128)), |
||||
('copyright', models.CharField(max_length=128)), |
||||
('cover', models.CharField(max_length=128)), |
||||
('favorites', models.IntegerField()), |
||||
('is_finish', models.IntegerField()), |
||||
('newest_ep_index', models.IntegerField()), |
||||
('pub_time', models.DateTimeField()), |
||||
('season_status', models.IntegerField()), |
||||
('title', models.CharField(max_length=128)), |
||||
('total_count', models.IntegerField()), |
||||
('trailer_aid', models.IntegerField()), |
||||
], |
||||
), |
||||
migrations.CreateModel( |
||||
name='mediaInfo', |
||||
fields=[ |
||||
('bangumi_id', models.IntegerField(primary_key=True, serialize=False)), |
||||
('season_id', models.IntegerField()), |
||||
('media_id', models.IntegerField()), |
||||
('chn_name', models.CharField(max_length=128)), |
||||
], |
||||
), |
||||
migrations.CreateModel( |
||||
name='param', |
||||
fields=[ |
||||
('param_name', models.CharField(max_length=10, primary_key=True, serialize=False)), |
||||
('param_value', models.CharField(max_length=128)), |
||||
], |
||||
), |
||||
migrations.CreateModel( |
||||
name='stat', |
||||
fields=[ |
||||
('id', models.IntegerField(primary_key=True, serialize=False)), |
||||
('danmakus', models.IntegerField()), |
||||
('favorites', models.IntegerField()), |
||||
('views', models.IntegerField()), |
||||
], |
||||
), |
||||
] |
||||
@ -0,0 +1,138 @@ |
||||
#!/usr/bin/env python |
||||
# coding:utf-8 |
||||
import json |
||||
import os |
||||
import sys |
||||
import threading |
||||
from concurrent import futures |
||||
from datetime import datetime |
||||
|
||||
import requests |
||||
from bs4 import BeautifulSoup |
||||
|
||||
import root |
||||
from PixivSearch.model import config |
||||
from PixivSearch.settings import logger |
||||
from PixivSearch.util import Util |
||||
|
||||
|
||||
headers = { |
||||
'X-Requested-With': 'XMLHttpRequest', |
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) ' |
||||
'Chrome/56.0.2924.87 Safari/537.36' |
||||
} |
||||
|
||||
lock = threading.Lock() # 多线程全局资源锁 |
||||
total = 1 |
||||
|
||||
|
||||
def get_cookies(): |
||||
_cookies = {} |
||||
array = config.param.objects.get(param_name='pixiv_cookie').param_value.split(';') |
||||
for row in array: |
||||
k, v = row.strip().split('=', 1) |
||||
_cookies[k] = v |
||||
return _cookies |
||||
|
||||
|
||||
def crawl(url): |
||||
global total |
||||
req = requests.get(url, headers=headers, cookies=get_cookies()).text |
||||
tag = BeautifulSoup(req, 'lxml').select('#js-mount-point-search-result-list')[0].attrs['data-items'] |
||||
imageNodes = json.loads(tag) |
||||
for imageNode in imageNodes: |
||||
with lock: |
||||
nodes.append(imageNode) |
||||
|
||||
|
||||
def get_urls(search, page): |
||||
fmt = 'https://www.pixiv.net/search.php?word={}&order=date_d&p={}' |
||||
return [fmt.format(search, p) for p in range(1, page)] |
||||
|
||||
|
||||
def get_Img(params): |
||||
params[1]['imgUrl'] = 'https://i.pximg.net/img-original/img/' + params[1]['url'][-46:-15] |
||||
|
||||
headers['referer'] = 'https://www.pixiv.net/member_illust.php?mode=medium&illust_id=' + params[1]['illustId'] |
||||
|
||||
suffix = ".jpg" |
||||
logger.info('开始下载图片：%s%s' % (params[1]['imgUrl'], suffix)) |
||||
|
||||
s = requests.get(params[1]['imgUrl'] + suffix, headers=headers, cookies=get_cookies()) |
||||
if (s.status_code == 404): |
||||
suffix = '.png' |
||||
s = requests.get(params[1]['imgUrl'] + suffix, headers=headers, cookies=get_cookies()) |
||||
if (s.status_code == 404): |
||||
logger.error('无法下载图片：%s' % (params[1]['illustTitle'])) |
||||
return |
||||
|
||||
logger.info('下载图片："%s"到%s' % ( |
||||
params[1]['illustTitle'], os.getcwd().replace('\\', '/') + '/' + imgPath + params[1]['illustId'] + suffix)) |
||||
f = open(imgPath + params[1]['illustId'] + suffix, 'wb') # 写入多媒体文件要 b 这个参数 |
||||
f.write(s.content) # 多媒体文件要是用conctent |
||||
f.close() |
||||
|
||||
params[1]['localName'] = params[1]['illustId'] + suffix |
||||
logger.info('排行第%d名，收藏数%d，标题：%s，标签：%s，(%s)前投稿，链接：%s' % ( |
||||
params[0], params[1]['bookmarkCount'], params[1]['illustTitle'], ','.join(params[1]['tags']), '', |
||||
params[1]['imgUrl'])) |
||||
|
||||
fsize = '' |
||||
|
||||
|
||||
def get_nodes(param): |
||||
global nodes, fsize, imgPath |
||||
nodes = [] |
||||
start = datetime.now() |
||||
urls = get_urls(param[1], int(param[2]) + 1) |
||||
logger.info('开始从P站获取图片数据') |
||||
with futures.ThreadPoolExecutor(32) as executor: |
||||
executor.map(crawl, urls) |
||||
|
||||
# for url in urls: |
||||
# crawl(url) |
||||
|
||||
length = len(nodes) |
||||
logger.info('获取到%d张图片' % (length)) |
||||
logger.info('对图片收藏数进行排序') |
||||
nodes = sorted(nodes, key=lambda v: v.get('bookmarkCount'), reverse=True)[:int(param[3])] # 按star数降序排序 |
||||
if (param[4] != None and param[4] == 'img'): |
||||
imgPath = root.getConfigValue('imgPath') |
||||
if not os.path.exists(imgPath): |
||||
os.makedirs(imgPath) |
||||
for file in os.listdir(imgPath): |
||||
os.remove(imgPath + file) |
||||
nodes_tup = [] |
||||
start_d = datetime.now() |
||||
for index, img in enumerate(nodes): |
||||
nodes_tup.append((index + 1, img)) |
||||
# get_Img((index+1,img)) |
||||
|
||||
with futures.ThreadPoolExecutor(32) as executor: |
||||
executor.map(get_Img, nodes_tup) |
||||
print('下载图片花费时间：%s' % (datetime.now() - start_d)) |
||||
logger.info('%s张图片下载完毕' % (len(os.listdir(imgPath)))) |
||||
|
||||
baseZipPath=root.getConfigValue('baseZipPath') |
||||
if not os.path.exists(baseZipPath): |
||||
os.makedirs(baseZipPath) |
||||
zipPath = baseZipPath+ param[1] + '.zip' |
||||
logger.info('图片打包到：%s' % (zipPath)) |
||||
Util.zip(imgPath, zipPath) |
||||
fsize = str(round(os.path.getsize(zipPath) / float(1024 * 1024), 2)) + 'MB' |
||||
logger.info('图包大小：%s' % (fsize)) |
||||
|
||||
tip = '从%d张图片中筛选出收藏数前%s的图片,处理耗时：%s' % (length, param[3], datetime.now() - start) |
||||
logger.info(tip) |
||||
return [nodes, tip, fsize] |
||||
|
||||
|
||||
if __name__ == "__main__": |
||||
if (len(sys.argv)) == 5 and sys.argv[2].isdigit() and sys.argv[3].isdigit(): |
||||
try: |
||||
get_nodes(sys.argv) |
||||
except BaseException as e: |
||||
repr(e) |
||||
|
||||
else: |
||||
logger.error('参数不合法') |
||||
@ -0,0 +1,141 @@ |
||||
""" |
||||
Django settings for PixivSearch project. |
||||
|
||||
Generated by 'django-admin startproject' using Django 1.11.7. |
||||
|
||||
For more information on this file, see |
||||
https://docs.djangoproject.com/en/1.11/topics/settings/ |
||||
|
||||
For the full list of settings and their values, see |
||||
https://docs.djangoproject.com/en/1.11/ref/settings/ |
||||
""" |
||||
import logging.config |
||||
import os |
||||
|
||||
# Build paths inside the project like this: os.path.join(BASE_DIR, ...) |
||||
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) |
||||
|
||||
|
||||
# Quick-start development settings - unsuitable for production |
||||
# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/ |
||||
|
||||
# SECURITY WARNING: keep the secret key used in production secret! |
||||
SECRET_KEY = 'dh3^+=iugoo*+p_ea4u3dh&b!_zlgs8*m9kc+#*f2eozglsqjh' |
||||
|
||||
# SECURITY WARNING: don't run with debug turned on in production! |
||||
DEBUG = True |
||||
|
||||
ALLOWED_HOSTS = ['*'] |
||||
|
||||
|
||||
# Application definition |
||||
|
||||
INSTALLED_APPS = [ |
||||
'django.contrib.admin', |
||||
'django.contrib.auth', |
||||
'django.contrib.contenttypes', |
||||
'django.contrib.sessions', |
||||
'django.contrib.messages', |
||||
'django.contrib.staticfiles', |
||||
'PixivSearch.model' |
||||
] |
||||
|
||||
MIDDLEWARE = [ |
||||
'django.middleware.security.SecurityMiddleware', |
||||
'django.contrib.sessions.middleware.SessionMiddleware', |
||||
'django.middleware.common.CommonMiddleware', |
||||
'django.middleware.csrf.CsrfViewMiddleware', |
||||
'django.contrib.auth.middleware.AuthenticationMiddleware', |
||||
'django.contrib.messages.middleware.MessageMiddleware', |
||||
'django.middleware.clickjacking.XFrameOptionsMiddleware', |
||||
] |
||||
|
||||
ROOT_URLCONF = 'PixivSearch.urls' |
||||
|
||||
TEMPLATES = [ |
||||
{ |
||||
'BACKEND': 'django.template.backends.django.DjangoTemplates', |
||||
'DIRS': [os.path.join(os.path.dirname(__file__), 'templates').replace('\\','/')], |
||||
'APP_DIRS': True, |
||||
'OPTIONS': { |
||||
'context_processors': [ |
||||
'django.template.context_processors.debug', |
||||
'django.template.context_processors.request', |
||||
'django.contrib.auth.context_processors.auth', |
||||
'django.contrib.messages.context_processors.messages', |
||||
], |
||||
}, |
||||
}, |
||||
] |
||||
|
||||
WSGI_APPLICATION = 'PixivSearch.wsgi.application' |
||||
|
||||
|
||||
# Database |
||||
# https://docs.djangoproject.com/en/1.11/ref/settings/#databases |
||||
|
||||
# DATABASES = { |
||||
# 'default': { |
||||
# 'ENGINE': 'django.db.backends.mysql', |
||||
# 'NAME': 'bangumi', |
||||
# 'USER': 'root', |
||||
# 'PASSWORD': '', |
||||
# 'HOST': '184.170.212.72', |
||||
# # 'HOST': 'sukura.top', |
||||
# 'PORT': '3306', |
||||
# } |
||||
# } |
||||
|
||||
DATABASES = { |
||||
'default': { |
||||
'ENGINE': 'django.db.backends.sqlite3', |
||||
'NAME': 'mydatabase', |
||||
} |
||||
} |
||||
|
||||
|
||||
# Password validation |
||||
# https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators |
||||
|
||||
AUTH_PASSWORD_VALIDATORS = [ |
||||
{ |
||||
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', |
||||
}, |
||||
{ |
||||
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', |
||||
}, |
||||
{ |
||||
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', |
||||
}, |
||||
{ |
||||
'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', |
||||
}, |
||||
] |
||||
|
||||
|
||||
# Internationalization |
||||
# https://docs.djangoproject.com/en/1.11/topics/i18n/ |
||||
|
||||
LANGUAGE_CODE = 'en-us' |
||||
|
||||
TIME_ZONE = 'Asia/Shanghai' |
||||
|
||||
USE_I18N = True |
||||
|
||||
USE_L10N = True |
||||
|
||||
USE_TZ = False |
||||
|
||||
|
||||
# Static files (CSS, JavaScript, Images) |
||||
# https://docs.djangoproject.com/en/1.11/howto/static-files/ |
||||
|
||||
STATIC_URL = '/static/' |
||||
STATICFILES_DIRS = [ |
||||
os.path.join(os.path.dirname(__file__), 'static').replace('\\', '/'), |
||||
] |
||||
|
||||
configPath = '%s/logging.conf' % os.path.dirname(__file__).replace('\\', '/') |
||||
logging.config.fileConfig(configPath) |
||||
logger = logging.getLogger('file') |
||||
|
||||
File diff suppressed because one or more lines are too long
@ -0,0 +1,27 @@ |
||||
<html xmlns="http://www.w3.org/1999/html"> |
||||
<head> |
||||
<meta charset="UTF-8"> |
||||
<title>参数管理</title> |
||||
{% load staticfiles %} |
||||
<script src="{% static "js/jquery-3.2.1.min.js"%}"></script> |
||||
<script> |
||||
$(function () { |
||||
$("button").click(function(){ |
||||
$("form").attr("method",$(this).attr("id")).submit(); |
||||
}); |
||||
|
||||
}) |
||||
</script> |
||||
</head> |
||||
<body> |
||||
<form action="/tsdm"> |
||||
{% csrf_token %} |
||||
<label>参数名<input name="param_name" value="{{param_name}}"/></label> |
||||
<label>参数值<input name="param_value" value="{{param_value}}"/></label> |
||||
<div> |
||||
<button id="POST">设置</button> |
||||
<button id="GET">获取</button> |
||||
</div> |
||||
</form> |
||||
</body> |
||||
</html> |
||||
@ -0,0 +1,107 @@ |
||||
<html xmlns="http://www.w3.org/1999/html"> |
||||
<head> |
||||
<style> |
||||
div { |
||||
text-align: center; |
||||
} |
||||
|
||||
ol { |
||||
list-style-type: demical; |
||||
} |
||||
|
||||
ol li { |
||||
list-style-position: outside; |
||||
} |
||||
</style> |
||||
{% load staticfiles %} |
||||
<script src="{% static "js/jquery-3.2.1.min.js" %}"></script> |
||||
<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css" |
||||
integrity="sha384-MCw98/SFnGE8fJT3GXwEOngsV7Zt27NXFoaoApmYm81iuXoPkFOJwJ8ERdknLPMO" crossorigin="anonymous"> |
||||
<script src="https://code.jquery.com/jquery-3.3.1.slim.min.js" |
||||
integrity="sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo" |
||||
crossorigin="anonymous"></script> |
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.3/umd/popper.min.js" |
||||
integrity="sha384-ZMP7rVo3mIykV+2+9J3UJ46jBk0WLaUAdn689aCwoqbBJiSnjAK/l8WvCWPIPm49" |
||||
crossorigin="anonymous"></script> |
||||
<script src="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js" |
||||
integrity="sha384-ChfqqxuZUCnJSK3+MXmPNIyE6ZbWh2IMqE241rYiqJxyMiZ6OW/JmZQ5stwEULTy" |
||||
crossorigin="anonymous"></script> |
||||
<script> |
||||
|
||||
function check() { |
||||
$("[name=pageSize],[name=order]").each(function () { |
||||
if ($(this).val() != '' && $(this).val() <= 0) { |
||||
$(this).val(''); |
||||
alert($(this).parent().text() + '不能小于等于0'); |
||||
$(this).focus(); |
||||
} |
||||
}); |
||||
} |
||||
</script> |
||||
</head> |
||||
<body> |
||||
|
||||
|
||||
<form action="/pixiv/search" method="post" onsubmit="check()"> |
||||
{% csrf_token %} |
||||
|
||||
<div class="form-inline"> |
||||
<div class="form-group"> |
||||
<label for="word">关键字</label> |
||||
<input type="text" class="form-control" id="word" name="word" value="{{ word }}" placeholder="R-18"> |
||||
</div> |
||||
|
||||
<div class="form-group"> |
||||
<label for="word">页数</label> |
||||
<input type="number" class="form-control" id="pageSize" name="pageSize" value="{{ pageSize }}" placeholder="10"> |
||||
</div> |
||||
|
||||
<div class="form-group"> |
||||
<label for="word">统计数</label> |
||||
<input type="number" class="form-control" id="order" name="order" value="{{ order }}" placeholder="10"> |
||||
</div> |
||||
|
||||
<div class="custom-control custom-radio custom-control-inline"> |
||||
<input type="radio" id="customRadio1" name="type" value="text" class="custom-control-input" {% if type == 'text' or type is None %}checked{% endif %}> |
||||
<label class="custom-control-label" for="customRadio1">不显示大图</label> |
||||
</div> |
||||
<div class="custom-control custom-radio custom-control-inline"> |
||||
<input type="radio" id="customRadio2" name="type" value="img" class="custom-control-input" {% ifequal type 'img' %}checked{% endifequal %}> |
||||
<label class="custom-control-label" for="customRadio2">显示大图</label> |
||||
</div> |
||||
</div> |
||||
|
||||
<div class="form-inline"> |
||||
<button class="btn btn-primary offset-md-3" id="button" type="submit">查询</button> |
||||
|
||||
{% if download %} |
||||
<a href="{% static "/download/" %}{{ download }}">图包提取(文件大小:{{ size }})</a> |
||||
{% endif %} |
||||
</div> |
||||
|
||||
|
||||
</form> |
||||
|
||||
<div> |
||||
<h1>{{ msg }}</h1> |
||||
</div> |
||||
<ol> |
||||
{% for imageNode in imageNodes %} |
||||
<li> |
||||
<div> |
||||
<h1>标题：<a href="https://www.pixiv.net/member_illust.php?mode=medium&illust_id={{ imageNode.illustId }}" |
||||
target="_blank">{{ imageNode.illustTitle }}</a>，画师：<a |
||||
href="https://www.pixiv.net/member.php?id={{ imageNode.userId }}">{{ imageNode.userName }}</a>，收藏数：{{ imageNode.bookmarkCount }} |
||||
</h1> |
||||
{% if imageNode.localName %} |
||||
<img src="{% static "images/" %}{{ imageNode.localName }}"> |
||||
{% endif %} |
||||
</div> |
||||
</li> |
||||
{% empty %} |
||||
<p>{{ tip }}</p> |
||||
{% endfor %} |
||||
|
||||
</ol> |
||||
</body> |
||||
</html> |
||||
@ -0,0 +1,23 @@ |
||||
from thrift.protocol import TBinaryProtocol |
||||
from thrift.transport import TSocket, TTransport |
||||
|
||||
from PixivSearch.thrift.QueryComment.QueryComment import Client |
||||
|
||||
if __name__ == '__main__': |
||||
|
||||
socket = TSocket.TSocket('sukura.top', 2233) |
||||
|
||||
# Buffering is critical. Raw sockets are very slow |
||||
transport = TTransport.TFramedTransport(socket) |
||||
|
||||
if not transport.isOpen(): |
||||
transport.open() |
||||
|
||||
# Wrap in a protocol |
||||
protocol = TBinaryProtocol.TBinaryProtocol(transport) |
||||
|
||||
# Create a client to use the protocol encoder |
||||
client = Client(protocol) |
||||
|
||||
cids = [7636499, 7636501, 7636500, 7636503, 7636504, 7636502, 7636509, 7636508, 7636506, 7636507, 7636505] |
||||
print(client.commentSumMap(cids)) |
||||
@ -0,0 +1,131 @@ |
||||
#!/usr/bin/env python |
||||
# |
||||
# Autogenerated by Thrift Compiler (0.11.0) |
||||
# |
||||
# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING |
||||
# |
||||
# options string: py |
||||
# |
||||
|
||||
import sys |
||||
import pprint |
||||
if sys.version_info[0] > 2: |
||||
from urllib.parse import urlparse |
||||
else: |
||||
from urlparse import urlparse |
||||
from thrift.transport import TTransport, TSocket, TSSLSocket, THttpClient |
||||
from thrift.protocol.TBinaryProtocol import TBinaryProtocol |
||||
|
||||
from QueryComment import QueryComment |
||||
from QueryComment.ttypes import * |
||||
|
||||
if len(sys.argv) <= 1 or sys.argv[1] == '--help': |
||||
print('') |
||||
print('Usage: ' + sys.argv[0] + ' [-h host[:port]] [-u url] [-f[ramed]] [-s[sl]] [-novalidate] [-ca_certs certs] [-keyfile keyfile] [-certfile certfile] function [arg1 [arg2...]]') |
||||
print('') |
||||
print('Functions:') |
||||
print(' i32 commentSum(i32 cid)') |
||||
print(' commentSumList( cids)') |
||||
print(' string download( cids, string fileName)') |
||||
print('') |
||||
sys.exit(0) |
||||
|
||||
pp = pprint.PrettyPrinter(indent=2) |
||||
host = 'localhost' |
||||
port = 9090 |
||||
uri = '' |
||||
framed = False |
||||
ssl = False |
||||
validate = True |
||||
ca_certs = None |
||||
keyfile = None |
||||
certfile = None |
||||
http = False |
||||
argi = 1 |
||||
|
||||
if sys.argv[argi] == '-h': |
||||
parts = sys.argv[argi + 1].split(':') |
||||
host = parts[0] |
||||
if len(parts) > 1: |
||||
port = int(parts[1]) |
||||
argi += 2 |
||||
|
||||
if sys.argv[argi] == '-u': |
||||
url = urlparse(sys.argv[argi + 1]) |
||||
parts = url[1].split(':') |
||||
host = parts[0] |
||||
if len(parts) > 1: |
||||
port = int(parts[1]) |
||||
else: |
||||
port = 80 |
||||
uri = url[2] |
||||
if url[4]: |
||||
uri += '?%s' % url[4] |
||||
http = True |
||||
argi += 2 |
||||
|
||||
if sys.argv[argi] == '-f' or sys.argv[argi] == '-framed': |
||||
framed = True |
||||
argi += 1 |
||||
|
||||
if sys.argv[argi] == '-s' or sys.argv[argi] == '-ssl': |
||||
ssl = True |
||||
argi += 1 |
||||
|
||||
if sys.argv[argi] == '-novalidate': |
||||
validate = False |
||||
argi += 1 |
||||
|
||||
if sys.argv[argi] == '-ca_certs': |
||||
ca_certs = sys.argv[argi+1] |
||||
argi += 2 |
||||
|
||||
if sys.argv[argi] == '-keyfile': |
||||
keyfile = sys.argv[argi+1] |
||||
argi += 2 |
||||
|
||||
if sys.argv[argi] == '-certfile': |
||||
certfile = sys.argv[argi+1] |
||||
argi += 2 |
||||
|
||||
cmd = sys.argv[argi] |
||||
args = sys.argv[argi + 1:] |
||||
|
||||
if http: |
||||
transport = THttpClient.THttpClient(host, port, uri) |
||||
else: |
||||
if ssl: |
||||
socket = TSSLSocket.TSSLSocket(host, port, validate=validate, ca_certs=ca_certs, keyfile=keyfile, certfile=certfile) |
||||
else: |
||||
socket = TSocket.TSocket(host, port) |
||||
if framed: |
||||
transport = TTransport.TFramedTransport(socket) |
||||
else: |
||||
transport = TTransport.TBufferedTransport(socket) |
||||
protocol = TBinaryProtocol(transport) |
||||
client = QueryComment.Client(protocol) |
||||
transport.open() |
||||
|
||||
if cmd == 'commentSum': |
||||
if len(args) != 1: |
||||
print('commentSum requires 1 args') |
||||
sys.exit(1) |
||||
pp.pprint(client.commentSum(eval(args[0]),)) |
||||
|
||||
elif cmd == 'commentSumList': |
||||
if len(args) != 1: |
||||
print('commentSumList requires 1 args') |
||||
sys.exit(1) |
||||
pp.pprint(client.commentSumList(eval(args[0]),)) |
||||
|
||||
elif cmd == 'download': |
||||
if len(args) != 2: |
||||
print('download requires 2 args') |
||||
sys.exit(1) |
||||
pp.pprint(client.download(eval(args[0]), args[1],)) |
||||
|
||||
else: |
||||
print('Unrecognized method %s' % cmd) |
||||
sys.exit(1) |
||||
|
||||
transport.close() |
||||
@ -0,0 +1,660 @@ |
||||
# |
||||
# Autogenerated by Thrift Compiler (0.11.0) |
||||
# |
||||
# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING |
||||
# |
||||
# options string: py |
||||
# |
||||
|
||||
from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException |
||||
from thrift.protocol.TProtocol import TProtocolException |
||||
from thrift.TRecursive import fix_spec |
||||
|
||||
import sys |
||||
import logging |
||||
from .ttypes import * |
||||
from thrift.Thrift import TProcessor |
||||
from thrift.transport import TTransport |
||||
all_structs = [] |
||||
|
||||
|
||||
class Iface(object): |
||||
def commentSum(self, cid): |
||||
""" |
||||
查询单个cid弹幕数 |
||||
|
||||
|
||||
Parameters: |
||||
- cid |
||||
""" |
||||
pass |
||||
|
||||
def commentSumList(self, cids): |
||||
""" |
||||
* 批量查询多个cid弹幕数 |
||||
* |
||||
|
||||
Parameters: |
||||
- cids |
||||
""" |
||||
pass |
||||
|
||||
def download(self, cids, fileName): |
||||
""" |
||||
* 批量下载弹幕 |
||||
* |
||||
|
||||
Parameters: |
||||
- cids |
||||
- fileName |
||||
""" |
||||
pass |
||||
|
||||
|
||||
class Client(Iface): |
||||
def __init__(self, iprot, oprot=None): |
||||
self._iprot = self._oprot = iprot |
||||
if oprot is not None: |
||||
self._oprot = oprot |
||||
self._seqid = 0 |
||||
|
||||
def commentSum(self, cid): |
||||
""" |
||||
查询单个cid弹幕数 |
||||
|
||||
|
||||
Parameters: |
||||
- cid |
||||
""" |
||||
self.send_commentSum(cid) |
||||
return self.recv_commentSum() |
||||
|
||||
def send_commentSum(self, cid): |
||||
self._oprot.writeMessageBegin('commentSum', TMessageType.CALL, self._seqid) |
||||
args = commentSum_args() |
||||
args.cid = cid |
||||
args.write(self._oprot) |
||||
self._oprot.writeMessageEnd() |
||||
self._oprot.trans.flush() |
||||
|
||||
def recv_commentSum(self): |
||||
iprot = self._iprot |
||||
(fname, mtype, rseqid) = iprot.readMessageBegin() |
||||
if mtype == TMessageType.EXCEPTION: |
||||
x = TApplicationException() |
||||
x.read(iprot) |
||||
iprot.readMessageEnd() |
||||
raise x |
||||
result = commentSum_result() |
||||
result.read(iprot) |
||||
iprot.readMessageEnd() |
||||
if result.success is not None: |
||||
return result.success |
||||
raise TApplicationException(TApplicationException.MISSING_RESULT, "commentSum failed: unknown result") |
||||
|
||||
def commentSumList(self, cids): |
||||
""" |
||||
* 批量查询多个cid弹幕数 |
||||
* |
||||
|
||||
Parameters: |
||||
- cids |
||||
""" |
||||
self.send_commentSumList(cids) |
||||
return self.recv_commentSumList() |
||||
|
||||
def send_commentSumList(self, cids): |
||||
self._oprot.writeMessageBegin('commentSumList', TMessageType.CALL, self._seqid) |
||||
args = commentSumList_args() |
||||
args.cids = cids |
||||
args.write(self._oprot) |
||||
self._oprot.writeMessageEnd() |
||||
self._oprot.trans.flush() |
||||
|
||||
def recv_commentSumList(self): |
||||
iprot = self._iprot |
||||
(fname, mtype, rseqid) = iprot.readMessageBegin() |
||||
if mtype == TMessageType.EXCEPTION: |
||||
x = TApplicationException() |
||||
x.read(iprot) |
||||
iprot.readMessageEnd() |
||||
raise x |
||||
result = commentSumList_result() |
||||
result.read(iprot) |
||||
iprot.readMessageEnd() |
||||
if result.success is not None: |
||||
return result.success |
||||
raise TApplicationException(TApplicationException.MISSING_RESULT, "commentSumList failed: unknown result") |
||||
|
||||
def download(self, cids, fileName): |
||||
""" |
||||
* 批量下载弹幕 |
||||
* |
||||
|
||||
Parameters: |
||||
- cids |
||||
- fileName |
||||
""" |
||||
self.send_download(cids, fileName) |
||||
return self.recv_download() |
||||
|
||||
def send_download(self, cids, fileName): |
||||
self._oprot.writeMessageBegin('download', TMessageType.CALL, self._seqid) |
||||
args = download_args() |
||||
args.cids = cids |
||||
args.fileName = fileName |
||||
args.write(self._oprot) |
||||
self._oprot.writeMessageEnd() |
||||
self._oprot.trans.flush() |
||||
|
||||
def recv_download(self): |
||||
iprot = self._iprot |
||||
(fname, mtype, rseqid) = iprot.readMessageBegin() |
||||
if mtype == TMessageType.EXCEPTION: |
||||
x = TApplicationException() |
||||
x.read(iprot) |
||||
iprot.readMessageEnd() |
||||
raise x |
||||
result = download_result() |
||||
result.read(iprot) |
||||
iprot.readMessageEnd() |
||||
if result.success is not None: |
||||
return result.success |
||||
raise TApplicationException(TApplicationException.MISSING_RESULT, "download failed: unknown result") |
||||
|
||||
|
||||
class Processor(Iface, TProcessor): |
||||
def __init__(self, handler): |
||||
self._handler = handler |
||||
self._processMap = {} |
||||
self._processMap["commentSum"] = Processor.process_commentSum |
||||
self._processMap["commentSumList"] = Processor.process_commentSumList |
||||
self._processMap["download"] = Processor.process_download |
||||
|
||||
def process(self, iprot, oprot): |
||||
(name, type, seqid) = iprot.readMessageBegin() |
||||
if name not in self._processMap: |
||||
iprot.skip(TType.STRUCT) |
||||
iprot.readMessageEnd() |
||||
x = TApplicationException(TApplicationException.UNKNOWN_METHOD, 'Unknown function %s' % (name)) |
||||
oprot.writeMessageBegin(name, TMessageType.EXCEPTION, seqid) |
||||
x.write(oprot) |
||||
oprot.writeMessageEnd() |
||||
oprot.trans.flush() |
||||
return |
||||
else: |
||||
self._processMap[name](self, seqid, iprot, oprot) |
||||
return True |
||||
|
||||
def process_commentSum(self, seqid, iprot, oprot): |
||||
args = commentSum_args() |
||||
args.read(iprot) |
||||
iprot.readMessageEnd() |
||||
result = commentSum_result() |
||||
try: |
||||
result.success = self._handler.commentSum(args.cid) |
||||
msg_type = TMessageType.REPLY |
||||
except TTransport.TTransportException: |
||||
raise |
||||
except TApplicationException as ex: |
||||
logging.exception('TApplication exception in handler') |
||||
msg_type = TMessageType.EXCEPTION |
||||
result = ex |
||||
except Exception: |
||||
logging.exception('Unexpected exception in handler') |
||||
msg_type = TMessageType.EXCEPTION |
||||
result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') |
||||
oprot.writeMessageBegin("commentSum", msg_type, seqid) |
||||
result.write(oprot) |
||||
oprot.writeMessageEnd() |
||||
oprot.trans.flush() |
||||
|
||||
def process_commentSumList(self, seqid, iprot, oprot): |
||||
args = commentSumList_args() |
||||
args.read(iprot) |
||||
iprot.readMessageEnd() |
||||
result = commentSumList_result() |
||||
try: |
||||
result.success = self._handler.commentSumList(args.cids) |
||||
msg_type = TMessageType.REPLY |
||||
except TTransport.TTransportException: |
||||
raise |
||||
except TApplicationException as ex: |
||||
logging.exception('TApplication exception in handler') |
||||
msg_type = TMessageType.EXCEPTION |
||||
result = ex |
||||
except Exception: |
||||
logging.exception('Unexpected exception in handler') |
||||
msg_type = TMessageType.EXCEPTION |
||||
result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') |
||||
oprot.writeMessageBegin("commentSumList", msg_type, seqid) |
||||
result.write(oprot) |
||||
oprot.writeMessageEnd() |
||||
oprot.trans.flush() |
||||
|
||||
def process_download(self, seqid, iprot, oprot): |
||||
args = download_args() |
||||
args.read(iprot) |
||||
iprot.readMessageEnd() |
||||
result = download_result() |
||||
try: |
||||
result.success = self._handler.download(args.cids, args.fileName) |
||||
msg_type = TMessageType.REPLY |
||||
except TTransport.TTransportException: |
||||
raise |
||||
except TApplicationException as ex: |
||||
logging.exception('TApplication exception in handler') |
||||
msg_type = TMessageType.EXCEPTION |
||||
result = ex |
||||
except Exception: |
||||
logging.exception('Unexpected exception in handler') |
||||
msg_type = TMessageType.EXCEPTION |
||||
result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') |
||||
oprot.writeMessageBegin("download", msg_type, seqid) |
||||
result.write(oprot) |
||||
oprot.writeMessageEnd() |
||||
oprot.trans.flush() |
||||
|
||||
# HELPER FUNCTIONS AND STRUCTURES |
||||
|
||||
|
||||
class commentSum_args(object): |
||||
""" |
||||
Attributes: |
||||
- cid |
||||
""" |
||||
|
||||
|
||||
def __init__(self, cid=None,): |
||||
self.cid = cid |
||||
|
||||
def read(self, iprot): |
||||
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: |
||||
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) |
||||
return |
||||
iprot.readStructBegin() |
||||
while True: |
||||
(fname, ftype, fid) = iprot.readFieldBegin() |
||||
if ftype == TType.STOP: |
||||
break |
||||
if fid == 1: |
||||
if ftype == TType.I32: |
||||
self.cid = iprot.readI32() |
||||
else: |
||||
iprot.skip(ftype) |
||||
else: |
||||
iprot.skip(ftype) |
||||
iprot.readFieldEnd() |
||||
iprot.readStructEnd() |
||||
|
||||
def write(self, oprot): |
||||
if oprot._fast_encode is not None and self.thrift_spec is not None: |
||||
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) |
||||
return |
||||
oprot.writeStructBegin('commentSum_args') |
||||
if self.cid is not None: |
||||
oprot.writeFieldBegin('cid', TType.I32, 1) |
||||
oprot.writeI32(self.cid) |
||||
oprot.writeFieldEnd() |
||||
oprot.writeFieldStop() |
||||
oprot.writeStructEnd() |
||||
|
||||
def validate(self): |
||||
return |
||||
|
||||
def __repr__(self): |
||||
L = ['%s=%r' % (key, value) |
||||
for key, value in self.__dict__.items()] |
||||
return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) |
||||
|
||||
def __eq__(self, other): |
||||
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ |
||||
|
||||
def __ne__(self, other): |
||||
return not (self == other) |
||||
all_structs.append(commentSum_args) |
||||
commentSum_args.thrift_spec = ( |
||||
None, # 0 |
||||
(1, TType.I32, 'cid', None, None, ), # 1 |
||||
) |
||||
|
||||
|
||||
class commentSum_result(object): |
||||
""" |
||||
Attributes: |
||||
- success |
||||
""" |
||||
|
||||
|
||||
def __init__(self, success=None,): |
||||
self.success = success |
||||
|
||||
def read(self, iprot): |
||||
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: |
||||
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) |
||||
return |
||||
iprot.readStructBegin() |
||||
while True: |
||||
(fname, ftype, fid) = iprot.readFieldBegin() |
||||
if ftype == TType.STOP: |
||||
break |
||||
if fid == 0: |
||||
if ftype == TType.I32: |
||||
self.success = iprot.readI32() |
||||
else: |
||||
iprot.skip(ftype) |
||||
else: |
||||
iprot.skip(ftype) |
||||
iprot.readFieldEnd() |
||||
iprot.readStructEnd() |
||||
|
||||
def write(self, oprot): |
||||
if oprot._fast_encode is not None and self.thrift_spec is not None: |
||||
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) |
||||
return |
||||
oprot.writeStructBegin('commentSum_result') |
||||
if self.success is not None: |
||||
oprot.writeFieldBegin('success', TType.I32, 0) |
||||
oprot.writeI32(self.success) |
||||
oprot.writeFieldEnd() |
||||
oprot.writeFieldStop() |
||||
oprot.writeStructEnd() |
||||
|
||||
def validate(self): |
||||
return |
||||
|
||||
def __repr__(self): |
||||
L = ['%s=%r' % (key, value) |
||||
for key, value in self.__dict__.items()] |
||||
return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) |
||||
|
||||
def __eq__(self, other): |
||||
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ |
||||
|
||||
def __ne__(self, other): |
||||
return not (self == other) |
||||
all_structs.append(commentSum_result) |
||||
commentSum_result.thrift_spec = ( |
||||
(0, TType.I32, 'success', None, None, ), # 0 |
||||
) |
||||
|
||||
|
||||
class commentSumList_args(object): |
||||
""" |
||||
Attributes: |
||||
- cids |
||||
""" |
||||
|
||||
|
||||
def __init__(self, cids=None,): |
||||
self.cids = cids |
||||
|
||||
def read(self, iprot): |
||||
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: |
||||
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) |
||||
return |
||||
iprot.readStructBegin() |
||||
while True: |
||||
(fname, ftype, fid) = iprot.readFieldBegin() |
||||
if ftype == TType.STOP: |
||||
break |
||||
if fid == 1: |
||||
if ftype == TType.LIST: |
||||
self.cids = [] |
||||
(_etype3, _size0) = iprot.readListBegin() |
||||
for _i4 in range(_size0): |
||||
_elem5 = iprot.readI32() |
||||
self.cids.append(_elem5) |
||||
iprot.readListEnd() |
||||
else: |
||||
iprot.skip(ftype) |
||||
else: |
||||
iprot.skip(ftype) |
||||
iprot.readFieldEnd() |
||||
iprot.readStructEnd() |
||||
|
||||
def write(self, oprot): |
||||
if oprot._fast_encode is not None and self.thrift_spec is not None: |
||||
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) |
||||
return |
||||
oprot.writeStructBegin('commentSumList_args') |
||||
if self.cids is not None: |
||||
oprot.writeFieldBegin('cids', TType.LIST, 1) |
||||
oprot.writeListBegin(TType.I32, len(self.cids)) |
||||
for iter6 in self.cids: |
||||
oprot.writeI32(iter6) |
||||
oprot.writeListEnd() |
||||
oprot.writeFieldEnd() |
||||
oprot.writeFieldStop() |
||||
oprot.writeStructEnd() |
||||
|
||||
def validate(self): |
||||
return |
||||
|
||||
def __repr__(self): |
||||
L = ['%s=%r' % (key, value) |
||||
for key, value in self.__dict__.items()] |
||||
return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) |
||||
|
||||
def __eq__(self, other): |
||||
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ |
||||
|
||||
def __ne__(self, other): |
||||
return not (self == other) |
||||
all_structs.append(commentSumList_args) |
||||
commentSumList_args.thrift_spec = ( |
||||
None, # 0 |
||||
(1, TType.LIST, 'cids', (TType.I32, None, False), None, ), # 1 |
||||
) |
||||
|
||||
|
||||
class commentSumList_result(object): |
||||
""" |
||||
Attributes: |
||||
- success |
||||
""" |
||||
|
||||
|
||||
def __init__(self, success=None,): |
||||
self.success = success |
||||
|
||||
def read(self, iprot): |
||||
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: |
||||
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) |
||||
return |
||||
iprot.readStructBegin() |
||||
while True: |
||||
(fname, ftype, fid) = iprot.readFieldBegin() |
||||
if ftype == TType.STOP: |
||||
break |
||||
if fid == 0: |
||||
if ftype == TType.LIST: |
||||
self.success = [] |
||||
(_etype10, _size7) = iprot.readListBegin() |
||||
for _i11 in range(_size7): |
||||
_elem12 = iprot.readI32() |
||||
self.success.append(_elem12) |
||||
iprot.readListEnd() |
||||
else: |
||||
iprot.skip(ftype) |
||||
else: |
||||
iprot.skip(ftype) |
||||
iprot.readFieldEnd() |
||||
iprot.readStructEnd() |
||||
|
||||
def write(self, oprot): |
||||
if oprot._fast_encode is not None and self.thrift_spec is not None: |
||||
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) |
||||
return |
||||
oprot.writeStructBegin('commentSumList_result') |
||||
if self.success is not None: |
||||
oprot.writeFieldBegin('success', TType.LIST, 0) |
||||
oprot.writeListBegin(TType.I32, len(self.success)) |
||||
for iter13 in self.success: |
||||
oprot.writeI32(iter13) |
||||
oprot.writeListEnd() |
||||
oprot.writeFieldEnd() |
||||
oprot.writeFieldStop() |
||||
oprot.writeStructEnd() |
||||
|
||||
def validate(self): |
||||
return |
||||
|
||||
def __repr__(self): |
||||
L = ['%s=%r' % (key, value) |
||||
for key, value in self.__dict__.items()] |
||||
return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) |
||||
|
||||
def __eq__(self, other): |
||||
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ |
||||
|
||||
def __ne__(self, other): |
||||
return not (self == other) |
||||
all_structs.append(commentSumList_result) |
||||
commentSumList_result.thrift_spec = ( |
||||
(0, TType.LIST, 'success', (TType.I32, None, False), None, ), # 0 |
||||
) |
||||
|
||||
|
||||
class download_args(object): |
||||
""" |
||||
Attributes: |
||||
- cids |
||||
- fileName |
||||
""" |
||||
|
||||
|
||||
def __init__(self, cids=None, fileName=None,): |
||||
self.cids = cids |
||||
self.fileName = fileName |
||||
|
||||
def read(self, iprot): |
||||
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: |
||||
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) |
||||
return |
||||
iprot.readStructBegin() |
||||
while True: |
||||
(fname, ftype, fid) = iprot.readFieldBegin() |
||||
if ftype == TType.STOP: |
||||
break |
||||
if fid == 1: |
||||
if ftype == TType.LIST: |
||||
self.cids = [] |
||||
(_etype17, _size14) = iprot.readListBegin() |
||||
for _i18 in range(_size14): |
||||
_elem19 = iprot.readI32() |
||||
self.cids.append(_elem19) |
||||
iprot.readListEnd() |
||||
else: |
||||
iprot.skip(ftype) |
||||
elif fid == 2: |
||||
if ftype == TType.STRING: |
||||
self.fileName = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() |
||||
else: |
||||
iprot.skip(ftype) |
||||
else: |
||||
iprot.skip(ftype) |
||||
iprot.readFieldEnd() |
||||
iprot.readStructEnd() |
||||
|
||||
def write(self, oprot): |
||||
if oprot._fast_encode is not None and self.thrift_spec is not None: |
||||
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) |
||||
return |
||||
oprot.writeStructBegin('download_args') |
||||
if self.cids is not None: |
||||
oprot.writeFieldBegin('cids', TType.LIST, 1) |
||||
oprot.writeListBegin(TType.I32, len(self.cids)) |
||||
for iter20 in self.cids: |
||||
oprot.writeI32(iter20) |
||||
oprot.writeListEnd() |
||||
oprot.writeFieldEnd() |
||||
if self.fileName is not None: |
||||
oprot.writeFieldBegin('fileName', TType.STRING, 2) |
||||
oprot.writeString(self.fileName.encode('utf-8') if sys.version_info[0] == 2 else self.fileName) |
||||
oprot.writeFieldEnd() |
||||
oprot.writeFieldStop() |
||||
oprot.writeStructEnd() |
||||
|
||||
def validate(self): |
||||
return |
||||
|
||||
def __repr__(self): |
||||
L = ['%s=%r' % (key, value) |
||||
for key, value in self.__dict__.items()] |
||||
return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) |
||||
|
||||
def __eq__(self, other): |
||||
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ |
||||
|
||||
def __ne__(self, other): |
||||
return not (self == other) |
||||
all_structs.append(download_args) |
||||
download_args.thrift_spec = ( |
||||
None, # 0 |
||||
(1, TType.LIST, 'cids', (TType.I32, None, False), None, ), # 1 |
||||
(2, TType.STRING, 'fileName', 'UTF8', None, ), # 2 |
||||
) |
||||
|
||||
|
||||
class download_result(object): |
||||
""" |
||||
Attributes: |
||||
- success |
||||
""" |
||||
|
||||
|
||||
def __init__(self, success=None,): |
||||
self.success = success |
||||
|
||||
def read(self, iprot): |
||||
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: |
||||
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) |
||||
return |
||||
iprot.readStructBegin() |
||||
while True: |
||||
(fname, ftype, fid) = iprot.readFieldBegin() |
||||
if ftype == TType.STOP: |
||||
break |
||||
if fid == 0: |
||||
if ftype == TType.STRING: |
||||
self.success = iprot.readString().decode('utf-8') if sys.version_info[0] == 2 else iprot.readString() |
||||
else: |
||||
iprot.skip(ftype) |
||||
else: |
||||
iprot.skip(ftype) |
||||
iprot.readFieldEnd() |
||||
iprot.readStructEnd() |
||||
|
||||
def write(self, oprot): |
||||
if oprot._fast_encode is not None and self.thrift_spec is not None: |
||||
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) |
||||
return |
||||
oprot.writeStructBegin('download_result') |
||||
if self.success is not None: |
||||
oprot.writeFieldBegin('success', TType.STRING, 0) |
||||
oprot.writeString(self.success.encode('utf-8') if sys.version_info[0] == 2 else self.success) |
||||
oprot.writeFieldEnd() |
||||
oprot.writeFieldStop() |
||||
oprot.writeStructEnd() |
||||
|
||||
def validate(self): |
||||
return |
||||
|
||||
def __repr__(self): |
||||
L = ['%s=%r' % (key, value) |
||||
for key, value in self.__dict__.items()] |
||||
return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) |
||||
|
||||
def __eq__(self, other): |
||||
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ |
||||
|
||||
def __ne__(self, other): |
||||
return not (self == other) |
||||
all_structs.append(download_result) |
||||
download_result.thrift_spec = ( |
||||
(0, TType.STRING, 'success', 'UTF8', None, ), # 0 |
||||
) |
||||
fix_spec(all_structs) |
||||
del all_structs |
||||
|
||||
@ -0,0 +1,48 @@ |
||||
import os |
||||
import sys |
||||
|
||||
from thrift.server.TNonblockingServer import TNonblockingServer |
||||
from thrift.transport import TSocket |
||||
|
||||
sys.path.append('/root/PixivSearch') |
||||
from PixivSearch.util import Util |
||||
from PixivSearch.dao.Comment import xml, downloadXml |
||||
from PixivSearch.thrift.QueryComment import QueryComment |
||||
from PixivSearch.thrift.QueryComment.QueryComment import Iface |
||||
|
||||
commentPath = '/root/PixivSearch/PixivSearch/thrift/tmpFile/comment' |
||||
#commentPath='/home/hua/下载/comment' |
||||
|
||||
class Server(Iface): |
||||
def commentSumList(self, cids): |
||||
result = [] |
||||
for cid in cids: |
||||
comment_selector = xml('https://comment.bilibili.com/{cid}.xml'.format(cid=cid)) |
||||
length = len(comment_selector.xpath('//i//d/text()')) |
||||
print('cid:{cid},弹幕数：{length}'.format(cid=cid, length=length)) |
||||
result.append(length) |
||||
return result |
||||
|
||||
def commentSum(self, cid): |
||||
return self.commentSumList([cid])[0]; |
||||
|
||||
def download(self, cids, fileName): |
||||
path = '{commentPath}/{fileName}'.format(commentPath=commentPath, fileName=fileName) |
||||
for cid in cids: |
||||
downloadXml(path, cid, |
||||
histroy=False) |
||||
zipFile = '{commentPath}/{fileName}.zip'.format(commentPath=commentPath,fileName=fileName) |
||||
print(zipFile) |
||||
Util.zip(path, zipFile) |
||||
if os.path.isfile(zipFile): |
||||
print('压缩包成功生成到{zipFile}'.format(zipFile=zipFile)) |
||||
return zipFile |
||||
else: |
||||
return None |
||||
|
||||
|
||||
if __name__ == '__main__': |
||||
socket = TSocket.TServerSocket(port=2233) |
||||
processor = QueryComment.Processor(Server()) |
||||
server = TNonblockingServer(processor, socket) |
||||
server.serve() |
||||
@ -0,0 +1,14 @@ |
||||
# |
||||
# Autogenerated by Thrift Compiler (0.11.0) |
||||
# |
||||
# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING |
||||
# |
||||
# options string: py |
||||
# |
||||
|
||||
from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException |
||||
from thrift.protocol.TProtocol import TProtocolException |
||||
from thrift.TRecursive import fix_spec |
||||
|
||||
import sys |
||||
from .ttypes import * |
||||
@ -0,0 +1,18 @@ |
||||
# |
||||
# Autogenerated by Thrift Compiler (0.11.0) |
||||
# |
||||
# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING |
||||
# |
||||
# options string: py |
||||
# |
||||
|
||||
from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException |
||||
from thrift.protocol.TProtocol import TProtocolException |
||||
from thrift.TRecursive import fix_spec |
||||
|
||||
import sys |
||||
|
||||
from thrift.transport import TTransport |
||||
all_structs = [] |
||||
fix_spec(all_structs) |
||||
del all_structs |
||||
@ -0,0 +1,22 @@ |
||||
import sys |
||||
|
||||
from thrift.server.TNonblockingServer import TNonblockingServer |
||||
from thrift.transport import TSocket |
||||
|
||||
sys.path.append('/root/PixivSearch') |
||||
from PixivSearch.thrift.task import TSDM |
||||
from PixivSearch.thrift.task.TSDM import Iface |
||||
|
||||
|
||||
class Server(Iface): |
||||
def qiandao(self): |
||||
return False |
||||
|
||||
def word(self): |
||||
return True |
||||
|
||||
if __name__ == '__main__': |
||||
socket = TSocket.TServerSocket(port=2233) |
||||
processor = TSDM.Processor(Server()) |
||||
server = TNonblockingServer(processor, socket) |
||||
server.serve() |
||||
@ -0,0 +1,124 @@ |
||||
#!/usr/bin/env python |
||||
# |
||||
# Autogenerated by Thrift Compiler (0.11.0) |
||||
# |
||||
# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING |
||||
# |
||||
# options string: py |
||||
# |
||||
|
||||
import sys |
||||
import pprint |
||||
if sys.version_info[0] > 2: |
||||
from urllib.parse import urlparse |
||||
else: |
||||
from urlparse import urlparse |
||||
from thrift.transport import TTransport, TSocket, TSSLSocket, THttpClient |
||||
from thrift.protocol.TBinaryProtocol import TBinaryProtocol |
||||
|
||||
from core.thrift.task import TSDM |
||||
from core.thrift.task.ttypes import * |
||||
|
||||
if len(sys.argv) <= 1 or sys.argv[1] == '--help': |
||||
print('') |
||||
print('Usage: ' + sys.argv[0] + ' [-h host[:port]] [-u url] [-f[ramed]] [-s[sl]] [-novalidate] [-ca_certs certs] [-keyfile keyfile] [-certfile certfile] function [arg1 [arg2...]]') |
||||
print('') |
||||
print('Functions:') |
||||
print(' bool qiandao()') |
||||
print(' bool word()') |
||||
print('') |
||||
sys.exit(0) |
||||
|
||||
pp = pprint.PrettyPrinter(indent=2) |
||||
host = 'localhost' |
||||
port = 9090 |
||||
uri = '' |
||||
framed = False |
||||
ssl = False |
||||
validate = True |
||||
ca_certs = None |
||||
keyfile = None |
||||
certfile = None |
||||
http = False |
||||
argi = 1 |
||||
|
||||
if sys.argv[argi] == '-h': |
||||
parts = sys.argv[argi + 1].split(':') |
||||
host = parts[0] |
||||
if len(parts) > 1: |
||||
port = int(parts[1]) |
||||
argi += 2 |
||||
|
||||
if sys.argv[argi] == '-u': |
||||
url = urlparse(sys.argv[argi + 1]) |
||||
parts = url[1].split(':') |
||||
host = parts[0] |
||||
if len(parts) > 1: |
||||
port = int(parts[1]) |
||||
else: |
||||
port = 80 |
||||
uri = url[2] |
||||
if url[4]: |
||||
uri += '?%s' % url[4] |
||||
http = True |
||||
argi += 2 |
||||
|
||||
if sys.argv[argi] == '-f' or sys.argv[argi] == '-framed': |
||||
framed = True |
||||
argi += 1 |
||||
|
||||
if sys.argv[argi] == '-s' or sys.argv[argi] == '-ssl': |
||||
ssl = True |
||||
argi += 1 |
||||
|
||||
if sys.argv[argi] == '-novalidate': |
||||
validate = False |
||||
argi += 1 |
||||
|
||||
if sys.argv[argi] == '-ca_certs': |
||||
ca_certs = sys.argv[argi+1] |
||||
argi += 2 |
||||
|
||||
if sys.argv[argi] == '-keyfile': |
||||
keyfile = sys.argv[argi+1] |
||||
argi += 2 |
||||
|
||||
if sys.argv[argi] == '-certfile': |
||||
certfile = sys.argv[argi+1] |
||||
argi += 2 |
||||
|
||||
cmd = sys.argv[argi] |
||||
args = sys.argv[argi + 1:] |
||||
|
||||
if http: |
||||
transport = THttpClient.THttpClient(host, port, uri) |
||||
else: |
||||
if ssl: |
||||
socket = TSSLSocket.TSSLSocket(host, port, validate=validate, ca_certs=ca_certs, keyfile=keyfile, certfile=certfile) |
||||
else: |
||||
socket = TSocket.TSocket(host, port) |
||||
if framed: |
||||
transport = TTransport.TFramedTransport(socket) |
||||
else: |
||||
transport = TTransport.TBufferedTransport(socket) |
||||
protocol = TBinaryProtocol(transport) |
||||
client = TSDM.Client(protocol) |
||||
transport.open() |
||||
|
||||
if cmd == 'qiandao': |
||||
if len(args) != 0: |
||||
print('qiandao requires 0 args') |
||||
sys.exit(1) |
||||
pp.pprint(client.qiandao()) |
||||
|
||||
elif cmd == 'word': |
||||
if len(args) != 0: |
||||
print('word requires 0 args') |
||||
sys.exit(1) |
||||
pp.pprint(client.word()) |
||||
|
||||
else: |
||||
print('Unrecognized method %s' % cmd) |
||||
sys.exit(1) |
||||
|
||||
transport.close() |
||||
@ -0,0 +1,366 @@ |
||||
# |
||||
# Autogenerated by Thrift Compiler (0.11.0) |
||||
# |
||||
# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING |
||||
# |
||||
# options string: py |
||||
# |
||||
|
||||
from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException |
||||
from thrift.protocol.TProtocol import TProtocolException |
||||
from thrift.TRecursive import fix_spec |
||||
|
||||
import sys |
||||
import logging |
||||
from .ttypes import * |
||||
from thrift.Thrift import TProcessor |
||||
from thrift.transport import TTransport |
||||
all_structs = [] |
||||
|
||||
|
||||
class Iface(object): |
||||
def qiandao(self): |
||||
pass |
||||
|
||||
def word(self): |
||||
pass |
||||
|
||||
|
||||
class Client(Iface): |
||||
def __init__(self, iprot, oprot=None): |
||||
self._iprot = self._oprot = iprot |
||||
if oprot is not None: |
||||
self._oprot = oprot |
||||
self._seqid = 0 |
||||
|
||||
def qiandao(self): |
||||
self.send_qiandao() |
||||
return self.recv_qiandao() |
||||
|
||||
def send_qiandao(self): |
||||
self._oprot.writeMessageBegin('qiandao', TMessageType.CALL, self._seqid) |
||||
args = qiandao_args() |
||||
args.write(self._oprot) |
||||
self._oprot.writeMessageEnd() |
||||
self._oprot.trans.flush() |
||||
|
||||
def recv_qiandao(self): |
||||
iprot = self._iprot |
||||
(fname, mtype, rseqid) = iprot.readMessageBegin() |
||||
if mtype == TMessageType.EXCEPTION: |
||||
x = TApplicationException() |
||||
x.read(iprot) |
||||
iprot.readMessageEnd() |
||||
raise x |
||||
result = qiandao_result() |
||||
result.read(iprot) |
||||
iprot.readMessageEnd() |
||||
if result.success is not None: |
||||
return result.success |
||||
raise TApplicationException(TApplicationException.MISSING_RESULT, "qiandao failed: unknown result") |
||||
|
||||
def word(self): |
||||
self.send_word() |
||||
return self.recv_word() |
||||
|
||||
def send_word(self): |
||||
self._oprot.writeMessageBegin('word', TMessageType.CALL, self._seqid) |
||||
args = word_args() |
||||
args.write(self._oprot) |
||||
self._oprot.writeMessageEnd() |
||||
self._oprot.trans.flush() |
||||
|
||||
def recv_word(self): |
||||
iprot = self._iprot |
||||
(fname, mtype, rseqid) = iprot.readMessageBegin() |
||||
if mtype == TMessageType.EXCEPTION: |
||||
x = TApplicationException() |
||||
x.read(iprot) |
||||
iprot.readMessageEnd() |
||||
raise x |
||||
result = word_result() |
||||
result.read(iprot) |
||||
iprot.readMessageEnd() |
||||
if result.success is not None: |
||||
return result.success |
||||
raise TApplicationException(TApplicationException.MISSING_RESULT, "word failed: unknown result") |
||||
|
||||
|
||||
class Processor(Iface, TProcessor): |
||||
def __init__(self, handler): |
||||
self._handler = handler |
||||
self._processMap = {} |
||||
self._processMap["qiandao"] = Processor.process_qiandao |
||||
self._processMap["word"] = Processor.process_word |
||||
|
||||
def process(self, iprot, oprot): |
||||
(name, type, seqid) = iprot.readMessageBegin() |
||||
if name not in self._processMap: |
||||
iprot.skip(TType.STRUCT) |
||||
iprot.readMessageEnd() |
||||
x = TApplicationException(TApplicationException.UNKNOWN_METHOD, 'Unknown function %s' % (name)) |
||||
oprot.writeMessageBegin(name, TMessageType.EXCEPTION, seqid) |
||||
x.write(oprot) |
||||
oprot.writeMessageEnd() |
||||
oprot.trans.flush() |
||||
return |
||||
else: |
||||
self._processMap[name](self, seqid, iprot, oprot) |
||||
return True |
||||
|
||||
def process_qiandao(self, seqid, iprot, oprot): |
||||
args = qiandao_args() |
||||
args.read(iprot) |
||||
iprot.readMessageEnd() |
||||
result = qiandao_result() |
||||
try: |
||||
result.success = self._handler.qiandao() |
||||
msg_type = TMessageType.REPLY |
||||
except TTransport.TTransportException: |
||||
raise |
||||
except TApplicationException as ex: |
||||
logging.exception('TApplication exception in handler') |
||||
msg_type = TMessageType.EXCEPTION |
||||
result = ex |
||||
except Exception: |
||||
logging.exception('Unexpected exception in handler') |
||||
msg_type = TMessageType.EXCEPTION |
||||
result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') |
||||
oprot.writeMessageBegin("qiandao", msg_type, seqid) |
||||
result.write(oprot) |
||||
oprot.writeMessageEnd() |
||||
oprot.trans.flush() |
||||
|
||||
def process_word(self, seqid, iprot, oprot): |
||||
args = word_args() |
||||
args.read(iprot) |
||||
iprot.readMessageEnd() |
||||
result = word_result() |
||||
try: |
||||
result.success = self._handler.word() |
||||
msg_type = TMessageType.REPLY |
||||
except TTransport.TTransportException: |
||||
raise |
||||
except TApplicationException as ex: |
||||
logging.exception('TApplication exception in handler') |
||||
msg_type = TMessageType.EXCEPTION |
||||
result = ex |
||||
except Exception: |
||||
logging.exception('Unexpected exception in handler') |
||||
msg_type = TMessageType.EXCEPTION |
||||
result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') |
||||
oprot.writeMessageBegin("word", msg_type, seqid) |
||||
result.write(oprot) |
||||
oprot.writeMessageEnd() |
||||
oprot.trans.flush() |
||||
|
||||
# HELPER FUNCTIONS AND STRUCTURES |
||||
|
||||
|
||||
class qiandao_args(object): |
||||
|
||||
|
||||
def read(self, iprot): |
||||
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: |
||||
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) |
||||
return |
||||
iprot.readStructBegin() |
||||
while True: |
||||
(fname, ftype, fid) = iprot.readFieldBegin() |
||||
if ftype == TType.STOP: |
||||
break |
||||
else: |
||||
iprot.skip(ftype) |
||||
iprot.readFieldEnd() |
||||
iprot.readStructEnd() |
||||
|
||||
def write(self, oprot): |
||||
if oprot._fast_encode is not None and self.thrift_spec is not None: |
||||
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) |
||||
return |
||||
oprot.writeStructBegin('qiandao_args') |
||||
oprot.writeFieldStop() |
||||
oprot.writeStructEnd() |
||||
|
||||
def validate(self): |
||||
return |
||||
|
||||
def __repr__(self): |
||||
L = ['%s=%r' % (key, value) |
||||
for key, value in self.__dict__.items()] |
||||
return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) |
||||
|
||||
def __eq__(self, other): |
||||
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ |
||||
|
||||
def __ne__(self, other): |
||||
return not (self == other) |
||||
all_structs.append(qiandao_args) |
||||
qiandao_args.thrift_spec = ( |
||||
) |
||||
|
||||
|
||||
class qiandao_result(object): |
||||
""" |
||||
Attributes: |
||||
- success |
||||
""" |
||||
|
||||
|
||||
def __init__(self, success=None,): |
||||
self.success = success |
||||
|
||||
def read(self, iprot): |
||||
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: |
||||
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) |
||||
return |
||||
iprot.readStructBegin() |
||||
while True: |
||||
(fname, ftype, fid) = iprot.readFieldBegin() |
||||
if ftype == TType.STOP: |
||||
break |
||||
if fid == 0: |
||||
if ftype == TType.BOOL: |
||||
self.success = iprot.readBool() |
||||
else: |
||||
iprot.skip(ftype) |
||||
else: |
||||
iprot.skip(ftype) |
||||
iprot.readFieldEnd() |
||||
iprot.readStructEnd() |
||||
|
||||
def write(self, oprot): |
||||
if oprot._fast_encode is not None and self.thrift_spec is not None: |
||||
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) |
||||
return |
||||
oprot.writeStructBegin('qiandao_result') |
||||
if self.success is not None: |
||||
oprot.writeFieldBegin('success', TType.BOOL, 0) |
||||
oprot.writeBool(self.success) |
||||
oprot.writeFieldEnd() |
||||
oprot.writeFieldStop() |
||||
oprot.writeStructEnd() |
||||
|
||||
def validate(self): |
||||
return |
||||
|
||||
def __repr__(self): |
||||
L = ['%s=%r' % (key, value) |
||||
for key, value in self.__dict__.items()] |
||||
return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) |
||||
|
||||
def __eq__(self, other): |
||||
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ |
||||
|
||||
def __ne__(self, other): |
||||
return not (self == other) |
||||
all_structs.append(qiandao_result) |
||||
qiandao_result.thrift_spec = ( |
||||
(0, TType.BOOL, 'success', None, None, ), # 0 |
||||
) |
||||
|
||||
|
||||
class word_args(object): |
||||
|
||||
|
||||
def read(self, iprot): |
||||
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: |
||||
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) |
||||
return |
||||
iprot.readStructBegin() |
||||
while True: |
||||
(fname, ftype, fid) = iprot.readFieldBegin() |
||||
if ftype == TType.STOP: |
||||
break |
||||
else: |
||||
iprot.skip(ftype) |
||||
iprot.readFieldEnd() |
||||
iprot.readStructEnd() |
||||
|
||||
def write(self, oprot): |
||||
if oprot._fast_encode is not None and self.thrift_spec is not None: |
||||
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) |
||||
return |
||||
oprot.writeStructBegin('word_args') |
||||
oprot.writeFieldStop() |
||||
oprot.writeStructEnd() |
||||
|
||||
def validate(self): |
||||
return |
||||
|
||||
def __repr__(self): |
||||
L = ['%s=%r' % (key, value) |
||||
for key, value in self.__dict__.items()] |
||||
return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) |
||||
|
||||
def __eq__(self, other): |
||||
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ |
||||
|
||||
def __ne__(self, other): |
||||
return not (self == other) |
||||
all_structs.append(word_args) |
||||
word_args.thrift_spec = ( |
||||
) |
||||
|
||||
|
||||
class word_result(object): |
||||
""" |
||||
Attributes: |
||||
- success |
||||
""" |
||||
|
||||
|
||||
def __init__(self, success=None,): |
||||
self.success = success |
||||
|
||||
def read(self, iprot): |
||||
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: |
||||
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) |
||||
return |
||||
iprot.readStructBegin() |
||||
while True: |
||||
(fname, ftype, fid) = iprot.readFieldBegin() |
||||
if ftype == TType.STOP: |
||||
break |
||||
if fid == 0: |
||||
if ftype == TType.BOOL: |
||||
self.success = iprot.readBool() |
||||
else: |
||||
iprot.skip(ftype) |
||||
else: |
||||
iprot.skip(ftype) |
||||
iprot.readFieldEnd() |
||||
iprot.readStructEnd() |
||||
|
||||
def write(self, oprot): |
||||
if oprot._fast_encode is not None and self.thrift_spec is not None: |
||||
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) |
||||
return |
||||
oprot.writeStructBegin('word_result') |
||||
if self.success is not None: |
||||
oprot.writeFieldBegin('success', TType.BOOL, 0) |
||||
oprot.writeBool(self.success) |
||||
oprot.writeFieldEnd() |
||||
oprot.writeFieldStop() |
||||
oprot.writeStructEnd() |
||||
|
||||
def validate(self): |
||||
return |
||||
|
||||
def __repr__(self): |
||||
L = ['%s=%r' % (key, value) |
||||
for key, value in self.__dict__.items()] |
||||
return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) |
||||
|
||||
def __eq__(self, other): |
||||
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ |
||||
|
||||
def __ne__(self, other): |
||||
return not (self == other) |
||||
all_structs.append(word_result) |
||||
word_result.thrift_spec = ( |
||||
(0, TType.BOOL, 'success', None, None, ), # 0 |
||||
) |
||||
fix_spec(all_structs) |
||||
del all_structs |
||||
|
||||
@ -0,0 +1,14 @@ |
||||
# |
||||
# Autogenerated by Thrift Compiler (0.11.0) |
||||
# |
||||
# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING |
||||
# |
||||
# options string: py |
||||
# |
||||
|
||||
from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException |
||||
from thrift.protocol.TProtocol import TProtocolException |
||||
from thrift.TRecursive import fix_spec |
||||
|
||||
import sys |
||||
from .ttypes import * |
||||
@ -0,0 +1,18 @@ |
||||
# |
||||
# Autogenerated by Thrift Compiler (0.11.0) |
||||
# |
||||
# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING |
||||
# |
||||
# options string: py |
||||
# |
||||
|
||||
from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException |
||||
from thrift.protocol.TProtocol import TProtocolException |
||||
from thrift.TRecursive import fix_spec |
||||
|
||||
import sys |
||||
|
||||
from thrift.transport import TTransport |
||||
all_structs = [] |
||||
fix_spec(all_structs) |
||||
del all_structs |
||||
@ -0,0 +1,27 @@ |
||||
"""PixivSearch URL Configuration |
||||
|
||||
The `urlpatterns` list routes URLs to views. For more information please see: |
||||
https://docs.djangoproject.com/en/1.11/topics/http/urls/ |
||||
Examples: |
||||
Function views |
||||
1. Add an import: from my_app import views |
||||
2. Add a URL to urlpatterns: url(r'^$', views.home, name='home') |
||||
Class-based views |
||||
1. Add an import: from other_app.views import Home |
||||
2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home') |
||||
Including another URLconf |
||||
1. Import the include() function: from django.conf.urls import url, include |
||||
2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) |
||||
""" |
||||
from django.conf.urls import url |
||||
from PixivSearch.view import search, index, download, saveConfig, get, start, stop |
||||
|
||||
urlpatterns = [ |
||||
url(r'^$', index), |
||||
url(r'^pixiv/search', search), |
||||
url(r'^pixiv/download', download), |
||||
url(r'^tsdm', saveConfig), |
||||
url(r'^bangumi/get', get), |
||||
url(r'^bangumi/start', start), |
||||
url(r'^bangumi/stop', stop) |
||||
] |
||||
@ -0,0 +1,10 @@ |
||||
import os |
||||
import zipfile |
||||
|
||||
|
||||
def zip(inputFile, outFile): |
||||
f = zipfile.ZipFile(outFile, 'w', zipfile.ZIP_DEFLATED) |
||||
for dirpath, dirnames, filenames in os.walk(inputFile): |
||||
for filename in filenames: |
||||
f.write(os.path.join(dirpath, filename), filename) |
||||
f.close() |
||||
@ -0,0 +1,101 @@ |
||||
# coding=utf-8 |
||||
import _thread |
||||
import os |
||||
|
||||
import django |
||||
from django.http import Http404, StreamingHttpResponse, HttpResponse |
||||
from django.shortcuts import render |
||||
|
||||
import root |
||||
from PixivSearch.dao.bangumi import get_, stop_, getIds |
||||
from PixivSearch.pixiv.pixiv import get_nodes |
||||
from PixivSearch.settings import logger |
||||
|
||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "PixivSearch.settings") |
||||
django.setup() # 添加的代码 |
||||
|
||||
from PixivSearch.model import config |
||||
|
||||
|
||||
def search(request): |
||||
word = 'R-18' |
||||
pageSize = 10 |
||||
order = 10 |
||||
type = 'text' |
||||
|
||||
if request.POST.get('word') != '' and request.POST.get('word') is not None: |
||||
word = request.POST.get('word') |
||||
if request.POST.get('pageSize') != '' and request.POST.get('pageSize') is not None: |
||||
pageSize = int(request.POST.get('pageSize')) |
||||
if request.POST.get('order') != '' and request.POST.get('order') is not None: |
||||
order = int(request.POST.get('order')) |
||||
if request.POST.get('type') != '' and request.POST.get('type') is not None: |
||||
type = request.POST.get('type') |
||||
|
||||
try: |
||||
logger.info("word:%s,pageSize:%d,order:%d,type:%s", word, pageSize, order, type) |
||||
array = get_nodes([0, word, pageSize, order, type]) |
||||
if len(array[0]) > 0: |
||||
if request.POST.get('type') == 'img': |
||||
nodes = {'imageNodes': array[0], 'msg': array[1], 'download': word + '.zip', 'size': array[2], |
||||
'word': word, 'pageSize': pageSize, 'order': order, 'type': type, |
||||
'imgPath': root.getConfigValue('imgPath'), |
||||
'baseZipPath': root.getConfigValue('baseZipPath')} |
||||
else: |
||||
nodes = {'imageNodes': array[0], 'msg': array[1], 'word': word, 'pageSize': pageSize, 'order': order, |
||||
'type': type} |
||||
else: |
||||
nodes = {'tip': '没有返回结果'} |
||||
return render(request, 'index.html', nodes) |
||||
except ValueError: |
||||
raise Http404() |
||||
|
||||
|
||||
def index(request): |
||||
return render(request, 'index.html', {'tip': '输入参数进行搜索'}) |
||||
|
||||
|
||||
def download(request): |
||||
# do something... |
||||
def file_iterator(file_name, chunk_size=512): |
||||
with open(file_name) as f: |
||||
while True: |
||||
c = f.read(chunk_size) |
||||
if c: |
||||
yield c |
||||
else: |
||||
break |
||||
|
||||
fileName = os.path.join(os.path.dirname(__file__), 'static/download/').replace('\\', '/') + request.GET.get( |
||||
'fileName').encode('utf-8') |
||||
response = StreamingHttpResponse(file_iterator(fileName)) |
||||
response['Content-Type'] = 'application/octet-stream' |
||||
response['Content-Disposition'] = 'attachment;filename="{0}"'.format(request.GET.get('fileName').encode('utf-8')) |
||||
return response |
||||
|
||||
|
||||
def saveConfig(request): |
||||
if 'GET' == request.method and request.GET.get('param_name'): |
||||
p = config.param.objects.get(param_name=request.GET.get('param_name')) |
||||
return render(request, 'addConfig.html', p.__dict__) |
||||
elif 'POST' == request.method and request.POST.get('param_name') and request.POST.get('param_value'): |
||||
p = config.param(param_name=request.POST.get('param_name'), param_value=request.POST.get('param_value')) |
||||
p.save() |
||||
return render(request, 'addConfig.html', p.__dict__) |
||||
else: |
||||
return render(request, 'addConfig.html') |
||||
|
||||
|
||||
def get(request): |
||||
return HttpResponse(str(get_().__str__())) |
||||
|
||||
|
||||
# 测试方法 |
||||
def start(request): |
||||
_thread.start_new_thread(getIds, ()) |
||||
return HttpResponse("start success") |
||||
|
||||
|
||||
def stop(request): |
||||
stop_() |
||||
return HttpResponse("stop success") |
||||
@ -0,0 +1,16 @@ |
||||
""" |
||||
WSGI config for PixivSearch project. |
||||
|
||||
It exposes the WSGI callable as a model-level variable named ``application``. |
||||
|
||||
For more information on this file, see |
||||
https://docs.djangoproject.com/en/1.11/howto/deployment/wsgi/ |
||||
""" |
||||
|
||||
import os |
||||
|
||||
from django.core.wsgi import get_wsgi_application |
||||
|
||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "PixivSearch.settings") |
||||
|
||||
application = get_wsgi_application() |
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,18 @@ |
||||
aliyun-python-sdk-cloudphoto==1.1.18 |
||||
aliyun-python-sdk-core-v3==2.8.6 |
||||
beautifulsoup4==4.6.0 |
||||
bs4==0.0.1 |
||||
certifi==2018.4.16 |
||||
chardet==3.0.4 |
||||
crcmod==1.7 |
||||
Django==2.0.5 |
||||
idna==2.6 |
||||
lxml==4.2.1 |
||||
mysqlclient==1.3.12 |
||||
oss2==2.4.0 |
||||
param==1.6.1 |
||||
pytz==2018.4 |
||||
requests==2.18.4 |
||||
six==1.11.0 |
||||
thrift==0.11.0 |
||||
urllib3==1.22 |
||||
@ -0,0 +1,22 @@ |
||||
#!/usr/bin/env python |
||||
import os |
||||
import sys |
||||
|
||||
if __name__ == "__main__": |
||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "PixivSearch.settings") |
||||
try: |
||||
from django.core.management import execute_from_command_line |
||||
except ImportError: |
||||
# The above import may fail for some other reason. Ensure that the |
||||
# issue is really that Django is missing to avoid masking other |
||||
# exceptions on Python 2. |
||||
try: |
||||
import django |
||||
except ImportError: |
||||
raise ImportError( |
||||
"Couldn't import Django. Are you sure it's installed and " |
||||
"available on your PYTHONPATH environment variable? Did you " |
||||
"forget to activate a virtual environment?" |
||||
) |
||||
raise |
||||
execute_from_command_line(sys.argv) |
||||
Binary file not shown.
@ -0,0 +1,5 @@ |
||||
[config] |
||||
#图片保存目录 |
||||
imgPath = PixivSearch/static/images/ |
||||
#压缩包保存目录 |
||||
baseZipPath=PixivSearch/static/zip/ |
||||
@ -0,0 +1,13 @@ |
||||
import configparser |
||||
|
||||
cf = configparser.ConfigParser() |
||||
|
||||
|
||||
def getConfigValue(name): |
||||
cf.read('root.ini', encoding='utf-8') |
||||
value = cf.get("config", name) |
||||
return value |
||||
|
||||
if __name__ == '__main__': |
||||
print(getConfigValue('imgPath')) |
||||
print(getConfigValue('baseZipPath')) |
||||
Loading…
Reference in new issue