diff --git a/api/proxyApi.py b/api/proxyApi.py index bd2de57e2..d1dff1f03 100644 --- a/api/proxyApi.py +++ b/api/proxyApi.py @@ -42,7 +42,7 @@ def force_type(cls, response, environ=None): app.response_class = JsonResponse api_list = [ - {"url": "/get", "params": "type: ''https'|''", "desc": "get a proxy"}, + {"url": "/get", "params": "type: 'https'|'', region: '代理地区, 中文'", "desc": "get a proxy"}, {"url": "/pop", "params": "", "desc": "get and delete a proxy"}, {"url": "/delete", "params": "proxy: 'e.g. 127.0.0.1:8080'", "desc": "delete an unable proxy"}, {"url": "/all", "params": "type: ''https'|''", "desc": "get all proxy from proxy pool"}, @@ -59,7 +59,8 @@ def index(): @app.route('/get/') def get(): https = request.args.get("type", "").lower() == 'https' - proxy = proxy_handler.get(https) + region = request.args.get("region", "").lower() + proxy = proxy_handler.get(https, region) return proxy.to_dict if proxy else {"code": 0, "src": "no proxy"} diff --git a/db/dbClient.py b/db/dbClient.py index 4d9554b18..0ef5f2720 100644 --- a/db/dbClient.py +++ b/db/dbClient.py @@ -86,8 +86,8 @@ def __initDbClient(self): password=self.db_pwd, db=self.db_name) - def get(self, https, **kwargs): - return self.client.get(https, **kwargs) + def get(self, https, region, **kwargs): + return self.client.get(https, region, **kwargs) def put(self, key, **kwargs): return self.client.put(key, **kwargs) diff --git a/db/redisClient.py b/db/redisClient.py index e66614d7e..f8b9406a9 100644 --- a/db/redisClient.py +++ b/db/redisClient.py @@ -47,7 +47,7 @@ def __init__(self, **kwargs): socket_timeout=5, **kwargs)) - def get(self, https): + def get(self, https, region): """ 返回一个代理 :return: @@ -55,6 +55,12 @@ def get(self, https): if https: items = self.__conn.hvals(self.name) proxies = list(filter(lambda x: json.loads(x).get("https"), items)) + if region != '': + proxies = list(filter(lambda x: region in json.loads(x).get("region"), iter(proxies))) + return choice(proxies) if proxies else None + elif region != '': + items = self.__conn.hvals(self.name) + proxies = list(filter(lambda x: region in json.loads(x).get("region"), items)) return choice(proxies) if proxies else None else: proxies = self.__conn.hkeys(self.name) diff --git a/db/ssdbClient.py b/db/ssdbClient.py index 0f5c00054..c785e656f 100644 --- a/db/ssdbClient.py +++ b/db/ssdbClient.py @@ -47,19 +47,20 @@ def __init__(self, **kwargs): socket_timeout=5, **kwargs)) - def get(self, https): + def get(self, https, region): """ 从hash中随机返回一个代理 :return: """ + items = self.__conn.hgetall(self.name).values() + proxies = list(items) if https: - items_dict = self.__conn.hgetall(self.name) - proxies = list(filter(lambda x: json.loads(x).get("https"), items_dict.values())) - return choice(proxies) if proxies else None - else: - proxies = self.__conn.hkeys(self.name) - proxy = choice(proxies) if proxies else None - return self.__conn.hget(self.name, proxy) if proxy else None + proxies = list(filter(lambda x: json.loads(x).get("https"), items)) + if region != '': + proxies = list(filter(lambda x: region in json.loads(x).get("region"), iter(proxies))) + # if anonymous != -1: + # proxies = list(filter(lambda x: json.loads(x).get("anonymous") == anonymous, iter(proxies))) + return choice(proxies) if proxies else None def put(self, proxy_obj): """ diff --git a/handler/proxyHandler.py b/handler/proxyHandler.py index 32e215e5d..f1a6832ad 100644 --- a/handler/proxyHandler.py +++ b/handler/proxyHandler.py @@ -26,14 +26,15 @@ def __init__(self): self.db = DbClient(self.conf.dbConn) self.db.changeTable(self.conf.tableName) - def get(self, https=False): + def get(self, https=False, region=''): """ return a proxy Args: https: True/False + region: Returns: """ - proxy = self.db.get(https) + proxy = self.db.get(https, region) return Proxy.createFromJson(proxy) if proxy else None def pop(self, https): diff --git a/helper/check.py b/helper/check.py index 39a481feb..4863b6048 100644 --- a/helper/check.py +++ b/helper/check.py @@ -13,6 +13,9 @@ """ __author__ = 'JHao' +import requests +import json + from util.six import Empty from threading import Thread from datetime import datetime @@ -40,6 +43,7 @@ def validator(cls, proxy): proxy.check_count += 1 proxy.last_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") proxy.last_status = True if http_r else False + proxy._region = cls.region_get(proxy.proxy) if http_r: if proxy.fail_count > 0: proxy.fail_count -= 1 @@ -69,6 +73,14 @@ def preValidator(cls, proxy): return False return True + @classmethod + def region_get(cls, proxy): + try: + r = requests.get(url='https://searchplugin.csdn.net/api/v1/ip/get', params={'ip': proxy.split(':')[0]}) + return json.loads(r.text)['data']['address'] + except: + return '未知或请求失败' + class _ThreadChecker(Thread): """ 多线程检测 """