diff --git a/app.py b/app.py index 1fb8d3eab45073623f3d958cf600a0553b640cb6..af7e9be3839b95cf0489815b960396dc0c44cc88 100644 --- a/app.py +++ b/app.py @@ -35,6 +35,7 @@ def create_flask_app(): app = create_flask_app() db.init_app(app) +db.app = app db.create_all(app=app) migrate = Migrate(app, db) diff --git a/base/rules.db b/base/rules.db index c0f3c7be2819b322d39fc85c2285a5f6eb4f8f46..8a503d3657320f471a0335c39c461d2716c4b710 100644 Binary files a/base/rules.db and b/base/rules.db differ diff --git a/controllers/cms.py b/controllers/cms.py index 6f76ffff7f2ad3c441e7e4654a8daf48632eab6e..fa6fb4161bf23c8e78c3d90f1c46c48aa34659de 100644 --- a/controllers/cms.py +++ b/controllers/cms.py @@ -152,8 +152,8 @@ class CMS: headers['User-Agent'] = UA if not 'referer' in lower_keys: headers['Referer'] = host - # print(headers) self.headers = headers + # print(headers) self.host = host self.homeUrl = urljoin(host,homeUrl) if host and homeUrl else homeUrl or host if url.find('[') >-1 and url.find(']') > -1: @@ -190,7 +190,6 @@ class CMS: 'jsp':jsoup(self.url), 'getParse':self.getParse, 'saveParse':self.saveParse, - 'headers':self.headers, 'oheaders':self.oheaders, 'encoding':self.encoding, 'name':self.title, @@ -506,7 +505,7 @@ class CMS: 'HOST': self.host, 'TYPE': 'home', # 海阔js环境标志 'oheaders':self.d.oheaders, - 'fetch_params': {'headers': self.d.headers, 'timeout': self.d.timeout, 'encoding': self.d.encoding}, + 'fetch_params': {'headers': self.headers, 'timeout': self.d.timeout, 'encoding': self.d.encoding}, 'd': self.d, 'getParse': self.d.getParse, 'saveParse': self.d.saveParse, @@ -684,7 +683,7 @@ class CMS: 'input': url, 'TYPE': 'cate', # 海阔js环境标志 'oheaders': self.d.oheaders, - 'fetch_params': {'headers': self.d.headers, 'timeout': self.d.timeout, 'encoding': self.d.encoding}, + 'fetch_params': {'headers': self.headers, 'timeout': self.d.timeout, 'encoding': self.d.encoding}, 'd': self.d, 'cateID':fyclass, # 分类id 'MY_CATE':fyclass, # 分类id @@ -804,7 +803,7 @@ class CMS: 'TYPE': 'detail', # 海阔js环境标志 'cateID': fyclass, # 当前分类 'oheaders': self.d.oheaders, - 'fetch_params': {'headers': self.d.headers, 'timeout': self.d.timeout, 'encoding': self.d.encoding}, + 'fetch_params': {'headers': self.headers, 'timeout': self.d.timeout, 'encoding': self.d.encoding}, 'd': self.d, 'getParse': self.d.getParse, 'saveParse': self.d.saveParse, @@ -889,7 +888,7 @@ class CMS: 'TYPE': 'detail', # 海阔js环境标志 'cateID': fyclass, # 当前分类 'oheaders': self.d.oheaders, - 'fetch_params': {'headers': self.d.headers, 'timeout': self.d.timeout, 'encoding': self.d.encoding}, + 'fetch_params': {'headers': self.headers, 'timeout': self.d.timeout, 'encoding': self.d.encoding}, 'd': self.d, 'getParse': self.d.getParse, 'saveParse': self.d.saveParse, @@ -995,7 +994,7 @@ class CMS: # print(result) return result - def searchContent(self, key, fypage=1): + def searchContent(self, key, fypage=1,show_name=False): pg = str(fypage) if not self.searchUrl: return self.blank() @@ -1013,11 +1012,12 @@ class CMS: py_ctx.update({ 'input': url, 'oheaders': self.d.oheaders, - 'fetch_params': {'headers': self.d.headers, 'timeout': self.d.timeout, 'encoding': self.d.encoding}, + 'fetch_params': {'headers': self.headers, 'timeout': self.d.timeout, 'encoding': self.d.encoding}, 'd': self.d, 'KEY': key, # 搜索关键字 'TYPE': 'search', # 海阔js环境标志 - 'detailUrl': self.detailUrl or '', # 详情页链接 + 'detailUrl': self.detailUrl or '', + # 详情页链接 'getParse': self.d.getParse, 'saveParse': self.d.saveParse, 'jsp': jsp, 'setDetail': setDetail, @@ -1100,6 +1100,11 @@ class CMS: # print(videos) except Exception as e: logger.info(f'搜索{self.getName()}发生错误:{e}') + if show_name and len(videos) > 0: + for video in videos: + video['vod_name'] = self.id + ' '+video['vod_name'] + video['vod_rule'] = self.id + video['vod_id'] = video['vod_id'] +'#' + self.id result = { 'list': videos } @@ -1148,7 +1153,7 @@ class CMS: py_ctx.update({ 'input': play_url, 'oheaders': self.d.oheaders, - 'fetch_params':{'headers':self.d.headers,'timeout':self.d.timeout,'encoding':self.d.encoding}, + 'fetch_params':{'headers':self.headers,'timeout':self.d.timeout,'encoding':self.d.encoding}, 'd': self.d, 'jxs':jxs, 'getParse':self.d.getParse, diff --git a/controllers/vod.py b/controllers/vod.py index 8016335adca989b7b1d3288764e460970b08e8a5..71283ad2edf2e79090af07ea4131c3c3f7ac74cd 100644 --- a/controllers/vod.py +++ b/controllers/vod.py @@ -17,12 +17,72 @@ from controllers.cms import CMS from base.database import db from models.ruleclass import RuleClass from models.playparse import PlayParse +from js.rules import getRules +from concurrent.futures import ThreadPoolExecutor,as_completed,thread # 引入线程池 vod = Blueprint("vod", __name__) + @vod.route('/vod') def vod_home(): + def search_one(rule, wd, before: str = ''): + t1 = time() + if not before: + with open('js/模板.js', encoding='utf-8') as f: + before = f.read() + js_path = f'js/{rule}.js' + ctx, js_code = parser.runJs(js_path, before=before) + if not js_code: + return None + ruleDict = ctx.rule.to_dict() + ruleDict['id'] = rule # 把路由请求的id装到字典里,后面播放嗅探才能用 + logger.info(f'规则{rule}装载耗时:{get_interval(t1)}毫秒') + try: + cms = CMS(ruleDict, db, RuleClass, PlayParse, cfg) + data = cms.searchContent(wd,show_name=True) + return data + except Exception as e: + print(f'{rule}发生错误:{e}') + return None + + def multi_search(wd): + rules = getRules('js')['list'] + rules_exclude = ['drpy'] + new_rules = list(filter(lambda x: x.get('searchable', 0) and x.get('name', '') not in rules_exclude, rules)) + search_sites = [new_rule['name'] for new_rule in new_rules] + logger.info(f'开始聚搜{wd},共计{len(search_sites)}个规则') + timeout = 5 + res = [] + with open('js/模板.js', encoding='utf-8') as f: + before = f.read() + with ThreadPoolExecutor(max_workers=len(search_sites)) as executor: + to_do = [] + for site in search_sites: + future = executor.submit(search_one, site, wd, before) + to_do.append(future) + try: + for future in as_completed(to_do, timeout=timeout): # 并发执行 + ret = future.result() + print(ret) + if ret: + res.extend(ret['list']) + except Exception as e: + print(f'发生错误:{e}') + import atexit + atexit.unregister(thread._python_exit) + executor.shutdown = lambda wait: None + return jsonify({ + "list": res + }) + t0 = time() rule = getParmas('rule') + ac = getParmas('ac') + ids = getParmas('ids') + if ac and ids and ids.find('#') > -1: # 聚搜的二级 + id_list = ids.split(',') + rule = id_list[0].split('#')[1] + print(rule) + ext = getParmas('ext') filters = getParmas('f') tp = getParmas('type') @@ -58,15 +118,13 @@ def vod_home(): # print(rule) cms = CMS(ruleDict,db,RuleClass,PlayParse,cfg) wd = getParmas('wd') - ac = getParmas('ac') quick = getParmas('quick') play = getParmas('play') # 类型为4的时候点击播放会带上来 flag = getParmas('flag') # 类型为4的时候点击播放会带上来 - filter = getParmas('filter') + # myfilter = getParmas('filter') t = getParmas('t') pg = getParmas('pg','1') pg = int(pg) - ids = getParmas('ids') q = getParmas('q') play_url = getParmas('play_url') @@ -105,15 +163,21 @@ def vod_home(): return jsonify(data) if ac and ids: # 二级 id_list = ids.split(',') + if ids.find('#') > -1: + id_list = list(map(lambda x:x.split('#')[0],id_list)) # print('app:377',len(id_list)) # print(id_list) data = cms.detailContent(pg,id_list) # print(data) return jsonify(data) if wd: # 搜索 - data = cms.searchContent(wd) - # print(data) - return jsonify(data) + if rule == 'drpy': + # print(f'准备单独处理聚合搜索:{wd}') + return multi_search(wd) + else: + data = cms.searchContent(wd) + # print(data) + return jsonify(data) # return jsonify({'rule':rule,'js_code':js_code}) home_data = cms.homeContent(pg) return jsonify(home_data) \ No newline at end of file diff --git "a/js/\350\217\234\347\213\227.js" "b/js/\350\217\234\347\213\227.js" index c5664c7e8f3328bc3c259a06dfb2bf5f581efe04..790681b0c9f9b9ee1a842e1143cdd4ea0d2075d8 100644 --- "a/js/\350\217\234\347\213\227.js" +++ "b/js/\350\217\234\347\213\227.js" @@ -10,7 +10,7 @@ var rule = { url:'/napi/video/classlist?abtest=0&iploc=CN1304&spver=&listTab=fyclass&filter=&start=((fypage-1)*15)&len=15&fr=filter', filter_url:'style={{fl.style}}&zone={{fl.zone}}&year={{fl.year}}&fee={{fl.fee}}&order={{fl.order}}', headers:{ - 'User-Agent':'MOBILE_UA' + 'User-Agent':'PC_UA' }, timeout:5000, class_name:'电视剧&电影&动漫&综艺&纪录片', @@ -26,5 +26,5 @@ var rule = { 二级:'js:var vod={vod_id:input};let html=request(input);function adhead(url){let hd="https://v.sogou.com";if(!url.startsWith(hd)){url=hd+url}return urlencode(url)}try{let json=JSON.parse(html.match(/INITIAL_STATE.*?({.*});/)[1]).detail.itemData;let key=json.dockey;let name=json.name;let zone=json.zone;let score=json.score?json.score:"暂无";let style=json.style;let emcee=json.emcee?"主持："+json.emcee:json.name;let director=json.director?"导演："+json.director:name;director=director.replace(/;/g,"\\t");let starring=json.starring?"演员："+json.starring:"声优："+json.shengyou;starring=starring.replace(/.*undefined/,"").replace(/;/g,"\\t");let update=json.update_wordstr?json.update_wordstr:"";let tv_station=json.tv_station?json.tv_station:zone;let introduction=json.introduction;let shengyou=json.shengyou;let shows=json.play_from_open_index;let plays=json.play.item_list;if(shows){vod.vod_name=name;vod.vod_area=emcee+","+tv_station;vod.vod_director=director;vod.vod_actor=starring;vod.vod_pic=jsp.pd(html,"#thumb_img&&img&&src");vod.vod_remarks=style+" 评分:"+score+","+update;vod.vod_content=introduction}else{vod.vod_name=name;vod.vod_director=director;vod.vod_actor=starring;vod.vod_pic=jsp.pd(html,"#thumb_img&&img&&src");vod.vod_content=introduction}let tp="&type=json";try{let tabs=[];let lists=[];plays.forEach(function(it){lists.push(it.info);let tbn=it.sitename[0]||it.site.replace(".com","");tbn=tbn.split("").join(" ");tabs.push(tbn)});vod.vod_play_from=tabs.join("$$$");vod_lists=[];play_url=play_url.replace("&play_url=","&type=json&play_url=");lists.forEach(function(item,idex){if(item||shows){if(item&&Array.isArray(item)&&item.length>1){let tmp=item.slice(1).map(function(its){return its.index+"$"+play_url+base64Encode(adhead(its.url))});vod_lists.push(tmp.join("#"))}if(shows){let arr=[];let tmp=[];let zy=shows.item_list[idex];zy.date.forEach(function(date){let day=date.day;for(let j=0;j=10?day[j][0]:"0"+day[j][0];let Tdate=date.year+date.month+dayy;arr.push(Tdate)}});for(let k=0;k0?img[0]:""}}catch(e){}', // 二级:'js:var vod={vod_id:input};let html=request(input);function adhead(url){let hd="https://v.sogou.com";if(!url.startsWith(hd)){url=hd+url}return url}try{let json=JSON.parse(html.match(/INITIAL_STATE.*?({.*});/)[1]).detail.itemData;let key=json.dockey;let name=json.name;let zone=json.zone;let score=json.score?json.score:"暂无";let style=json.style;let emcee=json.emcee?"主持："+json.emcee:json.name;let director=json.director?"导演："+json.director:name;director=director.replace(/;/g,"\\t");let starring=json.starring?"演员："+json.starring:"声优："+json.shengyou;starring=starring.replace(/.*undefined/,"").replace(/;/g,"\\t");let update=json.update_wordstr?json.update_wordstr:"";let tv_station=json.tv_station?json.tv_station:zone;let introduction=json.introduction;let shengyou=json.shengyou;let shows=json.play_from_open_index;let plays=json.play.item_list;if(shows){vod.vod_name=name;vod.vod_area=emcee+","+tv_station;vod.vod_director=director;vod.vod_actor=starring;vod.vod_pic=jsp.pd(html,"#thumb_img&&img&&src");vod.vod_remarks=style+" 评分:"+score+","+update;vod.vod_content=introduction}else{vod.vod_name=name;vod.vod_director=director;vod.vod_actor=starring;vod.vod_pic=jsp.pd(html,"#thumb_img&&img&&src");vod.vod_content=introduction}let tp="&type=json";try{let tabs=[];let lists=[];plays.forEach(function(it){lists.push(it.info);let tbn=it.sitename[0]||it.site.replace(".com","");tbn=tbn.split("").join(" ");tabs.push(tbn)});vod.vod_play_from=tabs.join("$$$");vod_lists=[];play_url=play_url.replace("&play_url=","&type=json&play_url=");lists.forEach(function(item,idex){if(item||shows){if(item&&Array.isArray(item)&&item.length>1){let tmp=item.slice(1).map(function(its){return its.index+"$"+play_url+base64Encode(adhead(its.url))});vod_lists.push(tmp.join("#"))}if(shows){let arr=[];let tmp=[];let zy=shows.item_list[idex];zy.date.forEach(function(date){let day=date.day;for(let j=0;j=10?day[j][0]:"0"+day[j][0];let Tdate=date.year+date.month+dayy;arr.push(Tdate)}});for(let k=0;k0?img[0]:""}}catch(e){}', 搜索:'', - 搜索:'js:let d=[];let html=request(input);let jsonA=JSON.parse(html.match(/INITIAL_STATE.*?({.*});/)[1]).result.longVideo.results;jsonA.forEach(function(it){let name=it.name;let introduction=it.introduction;let pic=it.v_picurl;let url=it.tiny_url;let zone=it.zone;let score=it.score||"暂无";let style=it.style;if(it.play.item_list){let r={};r.title=name.replace(//,"").replace(//,"");r.url="https://v.sogou.com"+url;r.desc=it.list_category.join(",");r.content=introduction;r.pic_url=pic;d.push(r)}});setResult(d);', + 搜索:'js:let d=[];let html=request(input);let jsonA=JSON.parse(html.match(/INITIAL_STATE.*?({.*});/)[1]);print(jsonA);jsonA=jsonA.result.longVideo.results;jsonA.forEach(function(it){let name=it.name;let introduction=it.introduction;let pic=it.v_picurl;let url=it.tiny_url;let zone=it.zone;let score=it.score||"暂无";let style=it.style;if(it.play.item_list){let r={};r.title=name.replace(//,"").replace(//,"");r.url="https://v.sogou.com"+url;r.desc=it.list_category.join(",");r.content=introduction;r.pic_url=pic;d.push(r)}});setResult(d);', } \ No newline at end of file diff --git a/libs/pre.js b/libs/pre.js index ed8729f5ac4fe56c479f58392f9fb14cd917ca89..14e1f00a0dcb039ccd36c8c0de1bf0ad23b78338 100644 --- a/libs/pre.js +++ b/libs/pre.js @@ -227,7 +227,7 @@ function request(url,obj){ }else{ new_obj = obj||{} } - if(!obj||!obj.headers||(!obj.headers['User-Agent']&&!obj.headers['user-agent'])){ + if(!new_obj||!new_obj.headers||(!new_obj.headers['User-Agent']&&!new_obj.headers['user-agent'])){ new_obj.headers['User-Agent'] = MOBILE_UA; } // delete new_obj.headers['Referer']; diff --git "a/py/\346\220\234\347\213\227\346\220\234\347\264\242.js" "b/py/\346\220\234\347\213\227\346\220\234\347\264\242.js" index e09c8ab29b5555d91577a70bcb946304e7bb4d3e..caa4665c317d002337672d4d2466ef191092def3 100644 --- "a/py/\346\220\234\347\213\227\346\220\234\347\264\242.js" +++ "b/py/\346\220\234\347\213\227\346\220\234\347\264\242.js" @@ -1,7 +1,10 @@ js: let d=[]; let html = request(input); -let jsonA = JSON.parse(html.match(/INITIAL_STATE.*?({.*});/)[1]).result.longVideo.results; +// print(html); +let jsonA = JSON.parse(html.match(/INITIAL_STATE.*?({.*});/)[1]); +print(jsonA); +jsonA = jsonA.result.longVideo.results; jsonA.forEach(function (it){ let name=it.name; let introduction=it.introduction; diff --git a/utils/web.py b/utils/web.py index 33c4777a0057fb3d19dbb4f342354df2bf772ced..22d4ebc67be117359d563ace246784c7aa3f9bfd 100644 --- a/utils/web.py +++ b/utils/web.py @@ -83,8 +83,8 @@ def get_interval(t): def getHeaders(url): headers = {} if url: - headers.setdefault("Referer", url) - headers.setdefault("User-Agent", PC_UA) - headers.setdefault("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9") - headers.setdefault("Accept-Language", "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2") + headers["Referer"] = url + headers["User-Agent"] = UA + # headers.setdefault("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9") + # headers.setdefault("Accept-Language", "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2") return headers \ No newline at end of file