(?:^|[\s,{])(?:server|ip|ipv6)\s*:\s*)" r'(?P(?![\'"])\S*:\S*)' r"(?P\s*(?:,|\}|$|\n|#))"
+)
+
+
+def quote_unquoted_ipv6(document: str) -> str:
+ if not document:
+ return ""
+
+ def replacer(match: re.Match[str]) -> str:
+ return f'{match.group("pre")}"{match.group("val")}"{match.group("post")}'
+
+ return _UNQUOTED_IPV6.sub(replacer, document)
+
def lookup(name: str) -> Category:
name = utils.trim(name)
@@ -110,7 +128,7 @@ class RegisterRequire:
class NoRedirHandler(urllib.request.HTTPRedirectHandler):
- def http_error_302(self, req, fp, code, msg, headers):
+ def http_error_302(self, req: Request, fp: HTTPResponse, code: int, msg: str, headers: HTTPMessage) -> HTTPResponse:
return fp
http_error_301 = http_error_302
@@ -144,32 +162,33 @@ def __init__(
self,
name: str,
site: str,
- sub: str,
+ nodes: NodeInput | None = None,
rename: str = "",
exclude: str = "",
include: str = "",
- liveness: bool = True,
+ check_alive: bool = True,
coupon: str = "",
api_prefix: str = "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/api/v1/",
- ):
+ ) -> None:
if site.endswith("/"):
site = site[: len(site) - 1]
self.api_prefix = utils.get_subpath(api_prefix)
- if sub.strip() != "":
+ self.nodes = nodes if isinstance(nodes, NodeInput) else NodeInput()
+ subs = self.nodes.subscribe_list()
+ sub = subs[0] if subs else ""
+ if sub:
if sub.startswith(utils.FILEPATH_PROTOCAL):
ref = sub[8:]
else:
ref = utils.extract_domain(sub, include_protocal=True)
- self.sub = sub
self.fetch = ""
self.ref = ref
self.reg = ""
self.registed = True
self.send_email = ""
else:
- self.sub = ""
self.fetch = f"{site}{self.api_prefix}user/server/fetch"
self.registed = False
self.send_email = f"{site}{self.api_prefix}passport/comm/sendEmailVerify"
@@ -179,7 +198,7 @@ def __init__(
self.rename = rename
self.exclude = exclude
self.include = include
- self.liveness = liveness
+ self.check_alive = check_alive
self.coupon = "" if utils.isblank(coupon) else coupon
self.headers = {
"User-Agent": utils.USER_AGENT,
@@ -327,10 +346,10 @@ def register(
api_prefix=self.api_prefix,
)
if subscribe_info:
- self.sub = subscribe_info.sub_url
- if not self.sub:
+ self.nodes.subscribe = subscribe_info.sub_url
+ if not self.nodes.subscribe_list():
if token:
- self.sub = f"{self.ref}/api/v1/client/subscribe?token={token}"
+ self.nodes.subscribe = f"{self.ref}/api/v1/client/subscribe?token={token}"
else:
logger.error(f"[RegisterError] cannot get token when register, domain: {self.ref}")
@@ -395,7 +414,7 @@ def order_plan(
return success
- def fetch_unused(self, cookies: str, auth: str = "", rate: float = 3.0) -> list:
+ def fetch_unused(self, cookies: str, auth: str = "", rate: float = 3.0) -> list[dict[str, object]]:
if (not cookies and not auth) or "" == self.fetch.strip():
return []
@@ -420,7 +439,12 @@ def fetch_unused(self, cookies: str, auth: str = "", rate: float = 3.0) -> list:
return []
def get_subscribe(
- self, retry: int, rr: RegisterRequire = None, rigid: bool = True, chuck: bool = False, invite_code: str = None
+ self,
+ retry: int,
+ rr: RegisterRequire = None,
+ allow_gmail_alias: bool = False,
+ skip_captcha: bool = False,
+ invite_code: str = None,
) -> tuple[str, str]:
if self.registed:
return "", ""
@@ -439,8 +463,8 @@ def get_subscribe(
# 需要邀请码或者强制验证
if (
(rr.invite and not invite_code)
- or (chuck and rr.recaptcha)
- or (rr.whitelist and rr.verify and (rigid or "gmail.com" not in rr.whitelist))
+ or (skip_captcha and rr.recaptcha)
+ or (rr.whitelist and rr.verify and (not allow_gmail_alias or "gmail.com" not in rr.whitelist))
):
self.available = False
return "", ""
@@ -507,6 +531,31 @@ def get_subscribe(
except:
return "", ""
+ def _fetch_text(self, url: str, retry: int) -> str:
+ url = utils.trim(url)
+ if not url:
+ return ""
+ if url.startswith(utils.FILEPATH_PROTOCAL):
+ file = url[len(utils.FILEPATH_PROTOCAL) :]
+ if not os.path.exists(file) or not os.path.isfile(file):
+ logger.error(f"[ParseError] file: {file} not found")
+ return ""
+ with open(file, "r", encoding="utf8") as reader:
+ return reader.read()
+
+ client = f"{utils.USER_AGENT}; Clash.Meta; Mihomo; Shadowrocket;"
+ headers = {"User-Agent": client}
+ trace = os.environ.get("TRACE_ENABLE", "false").lower() in ["true", "1"]
+ return utils.http_get(
+ url=url,
+ headers=headers,
+ retry=retry,
+ timeout=120,
+ trace=trace,
+ interval=1,
+ max_size=15 * 1024 * 1024,
+ ).strip()
+
def parse(
self,
cookie: str,
@@ -514,192 +563,153 @@ def parse(
retry: int,
rate: float,
bin_name: str,
- disable_insecure: bool = False,
+ require_tls: bool = False,
udp: bool = True,
ignore_exclude: bool = False,
- chatgpt: dict = None,
special_protocols: bool = False,
- ) -> list:
- if "" == self.sub:
- logger.error(f"[ParseError] cannot found any proxies because subscribe url is empty, domain: {self.ref}")
- return []
+ nodes: NodeInput | None = None,
+ ) -> list[dict[str, object]]:
+ source = nodes if isinstance(nodes, NodeInput) else self.nodes
+ collected = []
+ chars = utils.random_chars(length=3, punctuation=False)
+ artifact = f"{self.name}-{chars}"
+
+ for url in source.subscribe_list():
+ text = self._fetch_text(url, retry)
+ if not text or (
+ text.startswith("{") and text.endswith("}") and not re.search(r'"outbounds":', text, flags=re.I)
+ ):
+ logger.error(f"[ParseError] cannot found any proxies, subscribe: {utils.mask(url=url)}")
+ continue
+ collected.extend(
+ self.decode(
+ text=text,
+ artifact=f"{artifact}-sub",
+ program=bin_name,
+ ignore=ignore_exclude,
+ special=special_protocols,
+ do_verify=False,
+ )
+ )
- if self.sub.startswith(utils.FILEPATH_PROTOCAL):
- self.sub = self.sub[len(utils.FILEPATH_PROTOCAL) :]
- if not os.path.exists(self.sub) or not os.path.isfile(self.sub):
- logger.error(f"[ParseError] file: {self.sub} not found")
- return []
+ if source.uris:
+ collected.extend(
+ self.decode(
+ text="\n".join(source.uris),
+ artifact=f"{artifact}-uri",
+ program=bin_name,
+ ignore=ignore_exclude,
+ special=special_protocols,
+ do_verify=False,
+ )
+ )
- with open(self.sub, "r", encoding="UTF8") as f:
- text = f.read()
- else:
- headers = {"User-Agent": "Clash.Meta; Mihomo"}
- trace = os.environ.get("TRACE_ENABLE", "false").lower() in ["true", "1"]
-
- text = utils.http_get(
- url=self.sub,
- headers=headers,
- retry=retry,
- timeout=30,
- trace=trace,
- interval=1,
- max_size=15 * 1024 * 1024,
- ).strip()
+ collected.extend([item for item in source.proxies if isinstance(item, dict)])
+ if not collected:
+ if source.empty():
+ logger.error(f"[ParseError] cannot found any proxies because node input is empty, domain: {self.ref}")
+ return []
- if "" == text or (
- text.startswith("{") and text.endswith("}") and not re.search(r'"outbounds":', text, flags=re.I)
- ):
- logger.error(f"[ParseError] cannot found any proxies, subscribe: {utils.mask(url=self.sub)}")
+ parsed = [item for item in collected if verify(item, special_protocols)]
+ if not parsed:
+ logger.info(f"cannot found any proxy, domain: {self.ref}")
return []
- chatgpt = chatgpt if chatgpt and type(chatgpt) == dict else None
- enable, operate, pattern = False, "IN", ""
- if chatgpt:
- enable = chatgpt.get("enable", False)
- operate = utils.trim(chatgpt.get("operate", "IN")).upper()
- pattern = utils.trim(chatgpt.get("regex", ""))
+ proxies = []
+ unused_nodes = self.fetch_unused(cookie, auth, rate) if cookie or auth else []
+ subscribe_url = source.subscribe_list()[0] if source.subscribe_list() else ""
+ for item in parsed:
+ name = item.get("name", "")
+ if utils.isblank(name) or name in unused_nodes:
+ continue
- try:
- chars = utils.random_chars(length=3, punctuation=False)
- artifact = f"{self.name}-{chars}"
-
- nodes = self.decode(
- text=text,
- artifact=artifact,
- program=bin_name,
- ignore=ignore_exclude,
- special=special_protocols,
- )
-
- if not nodes:
- logger.info(f"cannot found any proxy, domain: {self.ref}")
- return []
+ if re.match(r"^JMS-\d+@[a-zA-Z0-9.]+:\d+$", name, flags=re.I):
+ server = name.split("@", maxsplit=1)[1]
+ hostname = utils.trim(server.split(":", maxsplit=1)[0]).lower()
+ if re.match(r"^(\d+\.){3}\d+$", item.get("server", ""), flags=re.I):
+ item["server"] = hostname
- proxies = []
- unused_nodes = self.fetch_unused(cookie, auth, rate)
- for item in nodes:
- name = item.get("name", "")
- if utils.isblank(name) or name in unused_nodes:
+ try:
+ if self.include and not re.search(self.include, name, re.I):
continue
+ if self.exclude and re.search(self.exclude, name, re.I):
+ continue
+ except Exception:
+ logger.error(
+ f"filter proxies error, maybe include or exclude regex exists problems, include: {self.include}\texclude: {self.exclude}"
+ )
- # JustMySocks节点，用主机名代替 IP 地址
- if re.match(r"^JMS-\d+@[a-zA-Z0-9.]+:\d+$", name, flags=re.I):
- server = name.split("@", maxsplit=1)[1]
- hostname = utils.trim(server.split(":", maxsplit=1)[0]).lower()
- if re.match(r"^(\d+\.){3}\d+$", item.get("server", ""), flags=re.I):
- item["server"] = hostname
-
- try:
- if self.include and not re.search(self.include, name, re.I):
- continue
- else:
- if self.exclude and re.search(self.exclude, name, re.I):
+ try:
+ if self.rename:
+ pattern_groups = self.rename.split(RENAME_GROUP_SEPARATOR)
+ for group in pattern_groups:
+ rename_regex = utils.trim(group)
+ if not rename_regex:
continue
- except:
- logger.error(
- f"filter proxies error, maybe include or exclude regex exists problems, include: {self.include}\texclude: {self.exclude}"
- )
-
- try:
- if self.rename:
- pattern_groups = self.rename.split(RENAME_GROUP_SEPARATOR)
- for group in pattern_groups:
- rename_regex = utils.trim(group)
- if not rename_regex:
- continue
-
- # re对group的引用方法: https://stackoverflow.com/questions/7191209/re-sub-replace-with-matched-content
- if RENAME_SEPARATOR in rename_regex:
- words = rename_regex.split(RENAME_SEPARATOR, maxsplit=1)
- old = words[0].strip()
- new = words[1].strip()
- if old:
- name = re.sub(old, new, name, flags=re.I)
- else:
- name = re.sub(rename_regex, "", name, flags=re.I)
-
- # 标记需要进行ChatGPT连通性测试的节点
- flag, detect = (
- enable or re.search(f"{utils.CHATGPT_FLAG}|(Chat)?GPT", name, flags=re.I),
- True,
- )
- if flag and pattern:
- match = re.search(pattern, name, flags=re.I)
- detect = match is None if operate != "IN" else match is not None
-
- if flag:
- name = re.sub(
- r"((\s+)?([\-\|_]+)?(\s+)?)?(Chat)?GPT",
- " ",
- name,
- flags=re.I,
- )
- item["chatgpt"] = detect
-
- # 重命名带网址的节点
- regex = r"(?:https?://)?(?:[a-zA-Z0-9\u4e00-\u9fa5\-]+\.)+[a-zA-Z\u4e00-\u9fa5]{2,}"
- name = re.sub(regex, "", name, flags=re.I)
- except:
- logger.error(
- f"rename error, name: {name},\trename: {self.rename}\tseparator: {RENAME_SEPARATOR}\tchatgpt: {pattern}\tdomain: {self.ref}"
- )
-
- name = re.sub(
- r"\[[^\[]*\]|[（\(][^（\(]*[\)）]|{[^{]*}|<[^<]*>|【[^【]*】|「[^「]*」|[^a-zA-Z0-9\u4e00-\u9fa5_×\.\-|\s]",
- " ",
- name,
- flags=re.I,
- ).strip()
-
- name = (
- re.sub(r"\s+|\r|\n|\\r|\\n", " ", name, flags=re.I)
- .replace("_", "-")
- .replace("+", "-")
- .strip(r"""!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~ """)
+ if RENAME_SEPARATOR in rename_regex:
+ words = rename_regex.split(RENAME_SEPARATOR, maxsplit=1)
+ old = words[0].strip()
+ new = words[1].strip()
+ if old:
+ name = re.sub(old, new, name, flags=re.I)
+ else:
+ name = re.sub(rename_regex, "", name, flags=re.I)
+
+ regex = r"(?:https?://)?(?:[a-zA-Z0-9\u4e00-\u9fa5\-]+\.)+[a-zA-Z\u4e00-\u9fa5]{2,}"
+ name = re.sub(regex, "", name, flags=re.I)
+ except Exception:
+ logger.error(
+ f"rename error, name: {name},\trename: {self.rename}\tseparator: {RENAME_SEPARATOR}\tdomain: {self.ref}"
)
- name = re.sub(r"((\s+)?-(\s+)?)+", "-", name)
- if not name:
- name = f"{self.name[0]}{self.name[-1]}-{''.join(random.sample(string.ascii_uppercase, 3))}"
-
- if len(name) > 30:
- i, j, k, n = 10, 4, 4, len(name)
- alphabets = [x for x in name[i : n - j] if x in LETTERS]
- if len(alphabets) > k:
- abbreviation = "".join(random.sample(alphabets, k)).strip()
- else:
- abbreviation = "".join(alphabets)
- name = f"{name[:i].strip()}-{abbreviation}-{name[-j:].strip()}"
+ name = re.sub(
+ r"\[[^\[]*\]|[（\(][^（\(]*[\)）]|{[^{]*}|<[^<]*>|【[^【]*】|「[^「]*」|[^a-zA-Z0-9\u4e00-\u9fa5_×\.\-|\s]",
+ " ",
+ name,
+ flags=re.I,
+ ).strip()
- name = re.sub(r"\s+(\d+)[\s_\-\|]+([A-Za-z])\b", r"-\1\2", name)
- item["name"] = re.sub(r"(-\d+[A-Za-z])+$", "", name).upper()
+ name = (
+ re.sub(r"\s+|\r|\n|\\r|\\n", " ", name, flags=re.I)
+ .replace("_", "-")
+ .replace("+", "-")
+ .strip(r"""!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~ """)
+ )
+ name = re.sub(r"((\s+)?-(\s+)?)+", "-", name)
+ if not name:
+ name = f"{self.name[0]}{self.name[-1]}-{''.join(random.sample(string.ascii_uppercase, 3))}"
+
+ if len(name) > 30:
+ i, j, k, n = 10, 4, 4, len(name)
+ alphabets = [x for x in name[i : n - j] if x in LETTERS]
+ if len(alphabets) > k:
+ abbreviation = "".join(random.sample(alphabets, k)).strip()
+ else:
+ abbreviation = "".join(alphabets)
+ name = f"{name[:i].strip()}-{abbreviation}-{name[-j:].strip()}"
- # 方便过滤无效订阅
- item["sub"] = self.sub
- item["liveness"] = self.liveness
+ name = re.sub(r"\s+(\d+)[\s_\-\|]+([A-Za-z])\b", r"-\1\2", name)
+ item["name"] = re.sub(r"(-\d+[A-Za-z])+$", "", name).upper()
+ item["sub"] = subscribe_url
+ item["liveness"] = self.check_alive
- if disable_insecure:
- if "skip-cert-verify" in item:
- item["skip-cert-verify"] = False
- if "tls" in item:
- item["tls"] = True
+ if require_tls:
+ if "skip-cert-verify" in item:
+ item["skip-cert-verify"] = False
+ if "tls" in item:
+ item["tls"] = True
- if udp and "udp" not in item and (item.get("type", "") != "snell" or int(item.get("version", 1)) == 3):
- item["udp"] = True
+ if udp and "udp" not in item and (item.get("type", "") != "snell" or int(item.get("version", 1)) == 3):
+ item["udp"] = True
- proxies.append(item)
+ proxies.append(item)
- return proxies
- except:
- logger.error(
- f"[ParseError] occur error when parse data, domain: {self.ref}, message:\n{traceback.format_exc()}"
- )
- return []
+ return proxies
@staticmethod
def check_protocol(link: str) -> bool:
return re.match(
- r"^(vmess|trojan|ss|ssr|vless|hysteria|hysteria2|tuic|snell|anytls)://[a-zA-Z0-9:.?+=@%&#_\-/]{10,}",
+ r"^(vmess|trojan|ss|ssr|vless|hysteria|hysteria2|tuic|snell|anytls|socks5|https?)://[a-zA-Z0-9:.?+=@%&#_\-/]{10,}",
utils.trim(link).replace("\r", ""),
flags=re.I,
)
@@ -713,7 +723,8 @@ def decode(
special: bool = False,
throw: bool = False,
use_subconverter: bool = True,
- ) -> list:
+ do_verify: bool = True,
+ ) -> list[dict[str, object]]:
def clean_text(document: str) -> str:
document = utils.trim(text=document)
if not document:
@@ -731,6 +742,38 @@ def clean_text(document: str) -> str:
return document
+ def load_nodes(document: str) -> list[dict[str, object]]:
+ document = quote_unquoted_ipv6(clean_text(document))
+ if not document:
+ return []
+
+ loaded = None
+ try:
+ loaded = yaml.load(document, Loader=yaml.SafeLoader)
+ except yaml.YAMLError:
+ try:
+ yaml.add_multi_constructor(
+ "str",
+ lambda loader, suffix, node: str(node.value),
+ Loader=yaml.SafeLoader,
+ )
+ loaded = yaml.load(document, Loader=yaml.FullLoader)
+ except yaml.YAMLError as e:
+ if throw:
+ raise e
+ logger.error(f"cannot load yaml file, artifact: {artifact}, message:\n{traceback.format_exc()}")
+ return []
+ except Exception as e:
+ if throw:
+ raise e
+ logger.error(f"cannot load yaml file, artifact: {artifact}, message:\n{traceback.format_exc()}")
+ return []
+
+ if not isinstance(loaded, dict):
+ return []
+ proxies = loaded.get("proxies", [])
+ return proxies if isinstance(proxies, list) else []
+
text, nodes = utils.trim(text=text), []
if not text:
return []
@@ -754,7 +797,7 @@ def clean_text(document: str) -> str:
not is_b64encode
and not is_json
and not is_yaml
- and all(AirPort.check_protocol(x) for x in text.split("\n") if x)
+ and all(AirPort.check_protocol(x) for x in text.split("\n") if x and not x.startswith("#"))
):
text = base64.b64encode(text.encode(encoding="UTF8")).decode(encoding="UTF8")
@@ -766,7 +809,7 @@ def clean_text(document: str) -> str:
if os.path.exists(v2ray_file):
os.remove(v2ray_file)
- logger.error(f"save file fialed, artifact: {artifact}")
+ logger.error(f"save file failed, artifact: {artifact}")
traceback.print_exc()
generate_conf = os.path.join(PATH, "subconverter", "generate.ini")
@@ -777,6 +820,7 @@ def clean_text(document: str) -> str:
f"{artifact}.yaml",
"clash",
True,
+ True,
ignore,
)
if not success:
@@ -792,47 +836,26 @@ def clean_text(document: str) -> str:
if not success:
return []
- with open(clash_file, "r", encoding="utf8", errors="ignore") as reader:
- config = None
- try:
- config = yaml.load(reader, Loader=yaml.SafeLoader)
- except (yaml.constructor.ConstructorError, yaml.parser.ParserError):
- reader.seek(0, 0)
- yaml.add_multi_constructor(
- "str",
- lambda loader, suffix, node: str(node.value),
- Loader=yaml.SafeLoader,
- )
- config = yaml.load(reader, Loader=yaml.SafeLoader)
- except Exception as e:
- if throw:
- raise e
- else:
- logger.error(f"cannot load yaml file, artifact: {artifact}, message:\n{traceback.format_exc()}")
-
- nodes = [] if not config else config.get("proxies", [])
-
- # 已经读取，可以删除
- os.remove(clash_file)
- else:
- nodes = None
+ document = ""
try:
- nodes = yaml.load(text, Loader=yaml.SafeLoader).get("proxies", [])
- except yaml.scanner.ScannerError:
- text = clean_text(document=text)
- nodes = yaml.load(text, Loader=yaml.SafeLoader).get("proxies", [])
- except (yaml.constructor.ConstructorError, yaml.parser.ParserError):
- yaml.add_multi_constructor("str", lambda loader, suffix, node: str(node.value), Loader=yaml.SafeLoader)
- nodes = yaml.load(text, Loader=yaml.FullLoader).get("proxies", [])
- except Exception as e:
- if throw:
- raise e
- else:
- logger.error(f"cannot load yaml file, artifact: {artifact}, message:\n{traceback.format_exc()}")
+ with open(clash_file, "r", encoding="utf8", errors="ignore") as reader:
+ document = reader.read()
+ finally:
+ if os.path.exists(clash_file):
+ os.remove(clash_file)
+ nodes = load_nodes(document)
+ else:
+ nodes = load_nodes(text)
- return [] if not nodes else [x for x in nodes if verify(x, special)]
+ if not nodes:
+ return []
+ if do_verify:
+ return [x for x in nodes if verify(x, special)]
+ return nodes
@staticmethod
- def enable_special_protocols() -> bool:
- flag = utils.trim(os.environ.get("ENABLE_SPECIAL_PROTOCOLS", "true")).lower()
- return (flag == "" or flag in ["true", "1"]) and is_mihomo()
+ def enable_special_protocols(enabled: bool | None = None) -> bool:
+ if enabled is None:
+ flag = utils.trim(os.environ.get("ENABLE_SPECIAL_PROTOCOLS", "true")).lower()
+ enabled = flag == "" or flag in ["true", "1"]
+ return bool(enabled) and is_mihomo()
diff --git a/subscribe/clash.py b/subscribe/clash.py
index 193559f585..78ac3546ec 100644
--- a/subscribe/clash.py
+++ b/subscribe/clash.py
@@ -3,8 +3,6 @@
# @Author : wzdnzd
# @Time : 2022-07-15
-import base64
-import ipaddress
import itertools
import json
import os
@@ -14,13 +12,13 @@
import string
import urllib
import urllib.parse
-import urllib.request
from collections import defaultdict
import executable
import utils
import yaml
from logger import logger
+from outbound import QuotedStr, endpoint_key, proxy_exists, quoted_scalar
CTX = ssl.create_default_context()
CTX.check_hostname = False
@@ -35,15 +33,7 @@
EXTERNAL_CONTROLLER = "127.0.0.1:9090"
-class QuotedStr(str):
- pass
-
-
-def quoted_scalar(dumper, data):
- return dumper.represent_scalar("tag:yaml.org,2002:str", data, style='"')
-
-
-def generate_config(path: str, proxies: list, filename: str) -> list:
+def generate_config(path: str, proxies: list[dict[str, object]], filename: str) -> list[dict[str, object]]:
os.makedirs(path, exist_ok=True)
external_config = filter_proxies(proxies)
config = {
@@ -64,7 +54,7 @@ def generate_config(path: str, proxies: list, filename: str) -> list:
return config.get("proxies", [])
-def filter_proxies(proxies: list) -> dict:
+def filter_proxies(proxies: list[dict[str, object]]) -> dict[str, object]:
config = {
"proxies": [],
"proxy-groups": [
@@ -85,9 +75,9 @@ def filter_proxies(proxies: list) -> dict:
unique_proxies, hosts = [], defaultdict(list)
for item in proxies:
- if not proxies_exists(item, hosts):
+ if not proxy_exists(item, hosts):
unique_proxies.append(item)
- key = f"{item.get('server')}:{item.get('port')}"
+ key = endpoint_key(item)
hosts[key].append(item)
# 防止多个代理节点名字相同导致clash配置错误
@@ -134,579 +124,9 @@ def filter_proxies(proxies: list) -> dict:
return config
-def proxies_exists(proxy: dict, hosts: dict) -> bool:
- if not proxy:
- return True
- if not hosts:
- return False
-
- key = f"{proxy.get('server')}:{proxy.get('port')}"
- proxies = hosts.get(key, [])
-
- if not proxies:
- return False
-
- protocol = proxy.get("type", "")
- if protocol == "http" or protocol == "socks5":
- return True
- elif protocol in ["ss", "trojan", "anytls", "hysteria2"]:
- return any(p.get("password", "") == proxy.get("password", "") for p in proxies)
- elif protocol == "ssr":
- return any(
- str(p.get("protocol-param", "")).lower() == str(proxy.get("protocol-param", "")).lower() for p in proxies
- )
- elif protocol == "vmess" or protocol == "vless":
- return any(p.get("uuid", "") == proxy.get("uuid", "") for p in proxies)
- elif protocol == "snell":
- return any(p.get("psk", "") == proxy.get("psk", "") for p in proxies)
- elif protocol == "tuic":
- if proxy.get("token", ""):
- return any(p.get("token", "") == proxy.get("token", "") for p in proxies)
- return any(p.get("uuid", "") == proxy.get("uuid", "") for p in proxies)
- elif protocol == "hysteria":
- key = "auth-str" if "auth-str" in proxy else "auth_str"
- value = proxy.get(key, "")
- for p in proxies:
- if "auth-str" in p and p.get("auth-str", "") == value:
- return True
- if "auth_str" in p and p.get("auth_str", "") == value:
- return True
-
- return False
-
-
-COMMON_SS_SUPPORTED_CIPHERS = [
- "aes-128-gcm",
- "aes-192-gcm",
- "aes-256-gcm",
- "aes-128-cfb",
- "aes-192-cfb",
- "aes-256-cfb",
- "aes-128-ctr",
- "aes-192-ctr",
- "aes-256-ctr",
- "rc4-md5",
- "chacha20-ietf",
- "xchacha20",
- "chacha20-ietf-poly1305",
- "xchacha20-ietf-poly1305",
-]
-
-# reference: https://github.com/MetaCubeX/sing-shadowsocks2/blob/dev/shadowaead_2022/method.go#L73-L86
-MIHOMO_SS_SUPPORTED_CIPHERS_SALT_LEN = {
- "2022-blake3-aes-128-gcm": 16,
- "2022-blake3-aes-256-gcm": 32,
- "2022-blake3-chacha20-poly1305": 32,
-}
-
-MIHOMO_SS_SUPPORTED_CIPHERS = (
- COMMON_SS_SUPPORTED_CIPHERS
- + list(MIHOMO_SS_SUPPORTED_CIPHERS_SALT_LEN.keys())
- + [
- "aes-128-ccm",
- "aes-192-ccm",
- "aes-256-ccm",
- "aes-128-gcm-siv",
- "aes-256-gcm-siv",
- "chacha20",
- "chacha8-ietf-poly1305",
- "xchacha8-ietf-poly1305",
- "lea-128-gcm",
- "lea-192-gcm",
- "lea-256-gcm",
- "rabbit128-poly1305",
- "aegis-128l",
- "aegis-256",
- "aez-384",
- "deoxys-ii-256-128",
- "none",
- ]
-)
-
-SSR_SUPPORTED_CIPHERS = COMMON_SS_SUPPORTED_CIPHERS + ["dummy", "none"]
-
-SSR_SUPPORTED_OBFS = [
- "plain",
- "http_simple",
- "http_post",
- "random_head",
- "tls1.2_ticket_auth",
- "tls1.2_ticket_fastauth",
-]
-
-SSR_SUPPORTED_PROTOCOL = [
- "origin",
- "auth_sha1_v4",
- "auth_aes128_md5",
- "auth_aes128_sha1",
- "auth_chain_a",
- "auth_chain_b",
-]
-
-VMESS_SUPPORTED_CIPHERS = ["auto", "aes-128-gcm", "chacha20-poly1305", "none"]
-
-SPECIAL_PROTOCOLS = set(["vless", "tuic", "hysteria", "hysteria2", "anytls"])
-
-# xtls-rprx-direct and xtls-rprx-origin are deprecated and no longer supported
-# XTLS_FLOWS = set(["xtls-rprx-direct", "xtls-rprx-origin", "xtls-rprx-vision"])
-
-
-def is_hex(word: str) -> bool:
- digits = set("0123456789abcdef")
- word = word.lower().strip()
- for c in word:
- if not (c in digits):
- return False
-
- return True
-
-
-def check_ports(port: str, ranges: str, protocol: str) -> bool:
- protocol = utils.trim(protocol).lower()
-
- try:
- flag = 0 < int(port) <= 65535
- if not flag or protocol not in ["hysteria", "hysteria2"] or not ranges:
- return flag
- except:
- return False
-
- nums = re.split(r"/|,", utils.trim(ranges))
- if not nums:
- return False
-
- for num in nums:
- start, end = num, num
- if "-" in num:
- start, end = num.split("-", maxsplit=1)
-
- try:
- start, end = int(start), int(end)
- if start <= 0 or start > 65535 or end <= 0 or end > 65535 or start > end:
- return False
- except:
- return False
-
- return True
-
-
-def verify(item: dict, mihomo: bool = True) -> bool:
- if not item or type(item) != dict or "type" not in item:
- return False
-
- try:
- # name must be string
- name = str(item.get("name", "")).strip()
- if not name:
- return False
- item["name"] = name
-
- # server must be string
- server = str(item.get("server", "")).strip().lower()
- if not server:
- return False
-
- if server.startswith("::"):
- # ipv6 addresses starting with ":::" cause yaml loading errors, need to expand to full format
- server = ipaddress.IPv6Address(server).exploded
-
- item["server"] = server
-
- # port must be valid port number
- if not check_ports(item.get("port", ""), item.get("ports", None), item.get("type", "")):
- return False
-
- # check uuid
- if "uuid" in item and not utils.verify_uuid(item.get("uuid")):
- return False
-
- # check servername and sni
- for attribute in ["servername", "sni"]:
- if attribute in item and type(item[attribute]) != str:
- return False
-
- for attribute in ["udp", "tls", "skip-cert-verify", "tfo"]:
- if attribute in item and type(item[attribute]) != bool:
- return False
-
- authentication = "password"
-
- if item["type"] == "ss":
- ciphers = COMMON_SS_SUPPORTED_CIPHERS if not mihomo else MIHOMO_SS_SUPPORTED_CIPHERS
- if item["cipher"] not in ciphers:
- return False
-
- if item["cipher"] in MIHOMO_SS_SUPPORTED_CIPHERS_SALT_LEN:
- # will throw bad key length error
- # see: https://github.com/MetaCubeX/sing-shadowsocks2/blob/dev/shadowaead_2022/method.go#L59-L108
- password = str(item.get(authentication, ""))
- words = password.split(":")
- for word in words:
- try:
- text = base64.b64decode(word)
- if len(text) != MIHOMO_SS_SUPPORTED_CIPHERS_SALT_LEN.get(item["cipher"]):
- return False
- except:
- return False
-
- plugin = item.get("plugin", "")
-
- # clash: https://clash.wiki/configuration/outbound.html#shadowsocks
- # mihomo: https://wiki.metacubex.one/config/proxies/ss/#plugin
- all_plugins, meta_plugins = ["", "obfs", "v2ray-plugin"], ["shadow-tls", "restls"]
- if mihomo:
- all_plugins.extend(meta_plugins)
-
- if plugin not in all_plugins:
- return False
- if plugin:
- option = item.get("plugin-opts", {}).get("mode", "")
- if plugin not in meta_plugins and (
- not option
- or (plugin == "v2ray-plugin" and option != "websocket")
- or (plugin == "obfs" and option not in ["tls", "http"])
- ):
- return False
- elif item["type"] == "ssr":
- if item["cipher"] not in SSR_SUPPORTED_CIPHERS:
- return False
- if item["obfs"] not in SSR_SUPPORTED_OBFS:
- return False
- if item["protocol"] not in SSR_SUPPORTED_PROTOCOL:
- return False
- elif item["type"] == "vmess":
- authentication = "uuid"
-
- # clash: https://clash.wiki/configuration/outbound.html#vmess
- # mihomo: https://wiki.metacubex.one/config/proxies/vmess/#network
- network, network_opts = item.get("network", "ws"), ["ws", "h2", "http", "grpc"]
- if mihomo:
- network_opts.append("httpupgrade")
-
- if network not in network_opts:
- return False
- if item.get("network", "ws") in ["h2", "grpc"] and not item.get("tls", False):
- return False
-
- # mihomo: https://wiki.metacubex.one/config/proxies/vmess/#cipher
- ciphers = VMESS_SUPPORTED_CIPHERS + ["zero"] if mihomo else VMESS_SUPPORTED_CIPHERS
- if item["cipher"] not in ciphers:
- return False
- if "alterId" not in item or not utils.is_number(item["alterId"]):
- return False
-
- if "h2-opts" in item:
- if network != "h2":
- return False
-
- h2_opts = item.get("h2-opts", {})
- if not h2_opts or type(h2_opts) != dict:
- return False
- if "host" in h2_opts and type(h2_opts["host"]) != list:
- return False
- elif "http-opts" in item:
- if network != "http":
- return False
-
- http_opts = item.get("http-opts", {})
- if not http_opts or type(http_opts) != dict:
- return False
- if "path" in http_opts and type(http_opts["path"]) != list:
- return False
- if "headers" in http_opts:
- headers = http_opts.get("headers", {})
- if not isinstance(headers, dict):
- return False
-
- for key, value in headers.items():
- if not isinstance(key, str):
- return False
- if key.lower() == "host" and not isinstance(value, list):
- return False
- elif "ws-opts" in item:
- if network != "ws" and network != "httpupgrade":
- return False
-
- ws_opts = item.get("ws-opts", {})
- if not ws_opts or type(ws_opts) != dict:
- return False
- if "path" in ws_opts and type(ws_opts["path"]) != str:
- return False
- if "headers" in ws_opts and type(ws_opts["headers"]) != dict:
- return False
- elif "grpc-opts" in item:
- if network != "grpc":
- return False
- if not mihomo:
- return False
-
- grpc_opts = item.get("grpc-opts", {})
- if not grpc_opts or type(grpc_opts) != dict:
- return False
- if "grpc-service-name" not in grpc_opts or type(grpc_opts["grpc-service-name"]) != str:
- return False
- elif item["type"] == "trojan":
- network = utils.trim(item.get("network", ""))
-
- if "alpn" in item and type(item["alpn"]) != list:
- return False
- if "ws-opts" in item:
- if network != "ws":
- return False
-
- ws_opts = item.get("ws-opts", {})
- if not ws_opts or type(ws_opts) != dict:
- return False
- if "path" in ws_opts and type(ws_opts["path"]) != str:
- return False
- if "headers" in ws_opts and type(ws_opts["headers"]) != dict:
- return False
- if "grpc-opts" in item:
- if network != "grpc":
- return False
-
- grpc_opts = item.get("grpc-opts", {})
- if not grpc_opts or type(grpc_opts) != dict:
- return False
- if "grpc-service-name" not in grpc_opts or type(grpc_opts["grpc-service-name"]) != str:
- return False
- if "flow" in item and (not mihomo or item["flow"] not in ["xtls-rprx-origin", "xtls-rprx-direct"]):
- return False
- elif item["type"] == "snell":
- authentication = "psk"
- if "version" in item and not utils.is_number(item["version"]):
- return False
-
- version = int(item.get("version", 1))
- if version < 1 or version > 3:
- return False
- if version != 3:
- # only version 3 supports UDP
- item.pop("udp", None)
-
- if "obfs-opts" in item:
- obfs_opts = item.get("obfs-opts", {})
- if not obfs_opts or type(obfs_opts) != dict:
- return False
- if "mode" in obfs_opts:
- mode = utils.trim(obfs_opts.get("mode", ""))
- if mode not in ["http", "tls"]:
- return False
- elif item["type"] == "http" or item["type"] == "socks5":
- for field in ["username", "password"]:
- value = item.get(field, None)
- if not value:
- continue
-
- if not isinstance(value, str) and not utils.is_number(value):
- return False
-
- if utils.is_number(value):
- value = QuotedStr(value)
- else:
- value = utils.trim(value)
-
- item[field] = value
-
- return True
-
- elif mihomo and item["type"] in SPECIAL_PROTOCOLS:
- if item["type"] == "anytls":
- if "alpn" in item and type(item["alpn"]) != list:
- return False
-
- for property in ["idle-session-check-interval", "idle-session-timeout", "min-idle-session"]:
- if property in item and (not utils.is_number(item[property]) or int(item[property]) < 0):
- return False
-
- elif item["type"] == "vless":
- authentication = "uuid"
- network = utils.trim(item.get("network", "tcp"))
-
- # mihomo: https://wiki.metacubex.one/config/proxies/vless/#network
- network_opts = ["ws", "tcp", "grpc", "http", "h2"] if mihomo else ["ws", "tcp", "grpc"]
-
- if network not in network_opts:
- return False
- if "flow" in item:
- flow = utils.trim(item.get("flow", ""))
- if flow and flow != "xtls-rprx-vision":
- return False
- if "ws-opts" in item:
- if network != "ws":
- return False
-
- ws_opts = item.get("ws-opts", {})
- if not ws_opts or type(ws_opts) != dict:
- return False
- if "path" in ws_opts and type(ws_opts["path"]) != str:
- return False
- if "headers" in ws_opts and type(ws_opts["headers"]) != dict:
- return False
- if "grpc-opts" in item:
- if network != "grpc":
- return False
-
- grpc_opts = item.get("grpc-opts", {})
- if not grpc_opts or type(grpc_opts) != dict:
- return False
- if "grpc-service-name" not in grpc_opts or type(grpc_opts["grpc-service-name"]) != str:
- return False
- if "reality-opts" in item:
- reality_opts = item.get("reality-opts", {})
- if (
- not reality_opts
- or type(reality_opts) != dict
- or "public-key" not in reality_opts
- or "short-id" not in reality_opts
- or type(reality_opts["public-key"]) != str
- ):
- return False
-
- content = utils.trim(reality_opts["public-key"])
- content += "=" * (4 - len(content) % 4)
- public_key = base64.urlsafe_b64decode(content)
- if len(public_key) != 32:
- return False
-
- short_id = reality_opts["short-id"]
- if type(short_id) != str:
- if utils.is_number(short_id):
- short_id = str(short_id)
- else:
- return False
-
- if len(short_id) != 8 or not is_hex(short_id) or re.match(r"\d+e\d+", short_id, flags=re.I):
- return False
-
- reality_opts["short-id"] = QuotedStr(short_id)
- elif item["type"] == "tuic":
- # mihomo: https://wiki.metacubex.one/config/proxies/tuic
- token = wrap(item.get("token", ""))
- uuid = wrap(item.get("uuid", ""))
- password = wrap(item.get("password", ""))
- if not token and not uuid and not password:
- return False
- if token and uuid and password:
- return False
- if token:
- authentication = "token"
- item["token"] = token
- else:
- if not uuid:
- return False
-
- authentication = "uuid"
- if password:
- item["password"] = password
-
- for property in ["disable-sni", "reduce-rtt", "fast-open"]:
- if property in item and type(item[property]) != bool:
- return False
- for property in [
- "heartbeat-interval",
- "request-timeout",
- "max-udp-relay-packet-size",
- "max-open-streams",
- ]:
- if property in item and not utils.is_number(item[property]):
- return False
- if "udp-relay-mode" in item and item["udp-relay-mode"] not in ["native", "quic"]:
- return False
- if "congestion-controller" in item and item["congestion-controller"] not in [
- "cubic",
- "bbr",
- "new_reno",
- ]:
- return False
- if "alpn" in item and type(item["alpn"]) != list:
- return False
- if "ip" in item:
- ip = utils.trim(item.get("ip", ""))
-
- # ip must be valid ipv4 or ipv6 address
- if not re.match(r"^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$", ip) and not re.match(
- r"^(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}$", ip
- ):
- return False
- else:
- for property in ["up", "down"]:
- if property not in item:
- continue
-
- traffic = item.get(property, "")
- if traffic == "null":
- item.pop(property)
- continue
-
- if traffic and utils.is_number(traffic):
- traffic = str(traffic)
-
- if not re.match(r"^\d+(\.\d+)?(\s+)?([kmgt]?bps)?$", utils.trim(traffic), flags=re.I):
- return False
-
- if "alpn" in item and type(item["alpn"]) != list:
- return False
- for property in ["ca", "ca-str"]:
- if property in item and type(item[property]) != str:
- return False
- if item["type"] == "hysteria2":
- # mihomo: https://wiki.metacubex.one/config/proxies/hysteria2
- authentication = "password"
- if "obfs" in item:
- obfs = utils.trim(item.get("obfs", ""))
- if obfs:
- if obfs != "salamander":
- return False
- obfs_password = utils.trim(item.get("obfs-password", ""))
- if not obfs_password:
- return False
-
- if "obfs-password" in item and type(item["obfs-password"]) != str:
- return False
- else:
- # mihomo: https://wiki.metacubex.one/config/proxies/hysteria
- authentication = "auth-str" if "auth-str" in item else "auth_str"
-
- for property in ["auth-str", "auth_str", "obfs"]:
- if property in item and type(item[property]) != str:
- return False
- for property in ["disable_mtu_discovery", "fast-open"]:
- if property in item and type(item[property]) != bool:
- return False
- if "protocol" in item:
- protocol = utils.trim(item.get("protocol", ""))
- if protocol not in ["udp", "wechat-video", "faketcp"]:
- return False
- if "ports" in item:
- ports = utils.trim(item.get("ports", [])).split(",")
- if not ports:
- return False
- for port in ports:
- # port must be valid port number
- if not utils.is_number(port) or int(port) <= 0 or int(port) > 65535:
- return False
- for property in ["recv_window_conn", "recv-window-conn", "recv_window", "recv-window"]:
- if property not in item:
- continue
- window = item.get(property, "")
- if not utils.is_number(window):
- return False
- else:
- return False
-
- if not item.get(authentication, ""):
- return False
-
- if utils.is_number(item[authentication]):
- item[authentication] = QuotedStr(item[authentication])
-
- return True
- except:
- return False
-
-
-def check(proxy: dict, api_url: str, timeout: int, test_url: str, delay: int, strict: bool = False) -> bool:
+def check(
+ proxy: dict[str, object], api_url: str, timeout: int, test_url: str, delay: int, strict: bool = False
+) -> bool:
proxy_name = ""
try:
proxy_name = urllib.parse.quote(proxy.get("name", ""), safe="")
@@ -745,34 +165,6 @@ def check(proxy: dict, api_url: str, timeout: int, test_url: str, delay: int, st
alive = False
break
- if alive:
- # filter and check US(for speed) proxies as candidates for ChatGPT/OpenAI/New Bing/Google Bard
- proxy_name = proxy.get("name", "")
- if proxy.pop("chatgpt", False) and not proxy_name.endswith(utils.CHATGPT_FLAG):
- try:
- # check for ChatGPT Web: https://chat.openai.com
- request = urllib.request.Request(
- url=f"{base_url}https://chat.openai.com/favicon.ico&expected=200",
- headers=utils.DEFAULT_HTTP_HEADERS,
- )
- response = urllib.request.urlopen(request, timeout=5, context=CTX)
- if response.getcode() == 200:
- content = str(response.read(), encoding="utf-8")
- data = json.loads(content)
- allowed = data.get("delay", -1) > 0
-
- # check for ChatGPT API: https://api.openai.com
- if allowed:
- content = utils.http_get(
- url=f"{base_url}https://api.openai.com/v1/engines&expected=401",
- retry=1,
- )
- data = json.loads(content)
- if data.get("delay", -1) > 0:
- proxy["name"] = f"{proxy_name}{utils.CHATGPT_FLAG}"
- except Exception:
- logger.debug(f"check for OpenAI failed, proxy: {proxy.get('name')}, message: {str(e)}")
-
return alive
except Exception as e:
logger.debug(f"check failed, proxy: {proxy.get('name')}, message: {str(e)}")
@@ -790,10 +182,3 @@ def is_mihomo() -> bool:
return re.search("Mihomo Meta", output, flags=re.I) is not None
except:
return False
-
-
-def wrap(text: str) -> str:
- if utils.is_number(text):
- text = str(text)
-
- return utils.trim(text)
diff --git a/subscribe/collect.py b/subscribe/collect.py
index b835903e6e..0eab3c1731 100644
--- a/subscribe/collect.py
+++ b/subscribe/collect.py
@@ -13,13 +13,16 @@
import sys
import time
-import crawl
import executable
+import pipeline
import push
import utils
import workflow
import yaml
from airport import AirPort
+from config.models import NodeInput, SiteConfig, StorageItem
+from crawl.helpers import check_status, naming_task
+from discovery import AirportRecord, collect_airport, parse_records, save_records
from logger import logger
from urlvalidator import isurl
from workflow import TaskConfig
@@ -37,10 +40,10 @@ def assign(
domains_file: str = "",
overwrite: bool = False,
pages: int = sys.maxsize,
- rigid: bool = True,
+ allow_gmail_alias: bool = False,
display: bool = True,
num_threads: int = 0,
- **kwargs,
+ **kwargs: object,
) -> list[TaskConfig]:
def load_exist(username: str, gist_id: str, access_token: str, filename: str) -> list[str]:
if not filename:
@@ -58,7 +61,7 @@ def load_exist(username: str, gist_id: str, access_token: str, filename: str) ->
if username and gist_id and access_token:
push_tool = push.PushToGist(token=access_token)
- url = push_tool.raw_url(config={"username": username, "gistid": gist_id, "filename": filename})
+ url = push_tool.raw_url(item=StorageItem(username=username, gist_id=gist_id, filename=filename))
content = utils.http_get(url=url, timeout=30)
items = re.findall(pattern, content, flags=re.M)
@@ -70,7 +73,7 @@ def load_exist(username: str, gist_id: str, access_token: str, filename: str) ->
# 过滤已过期订阅并返回
links = list(subscriptions)
results = utils.multi_thread_run(
- func=crawl.check_status,
+ func=check_status,
tasks=links,
num_threads=num_threads,
show_progress=display,
@@ -78,32 +81,11 @@ def load_exist(username: str, gist_id: str, access_token: str, filename: str) ->
return [links[i] for i in range(len(links)) if results[i][0] and not results[i][1]]
- def parse_domains(content: str) -> dict:
- if not content or not isinstance(content, str):
- logger.warning("cannot found any domain due to content is empty or not string")
- return {}
-
- records = {}
- for line in content.split("\n"):
- line = utils.trim(line)
- if not line or line.startswith("#"):
- continue
-
- words = line.rsplit(delimiter, maxsplit=3)
- address = utils.trim(words[0])
- coupon = utils.trim(words[1]) if len(words) > 1 else ""
- invite_code = utils.trim(words[2]) if len(words) > 2 else ""
- api_prefix = utils.trim(words[3]) if len(words) > 3 else ""
-
- records[address] = {"coupon": coupon, "invite_code": invite_code, "api_prefix": api_prefix}
-
- return records
-
subscribes_file = utils.trim(kwargs.get("subscribes_file", ""))
access_token = utils.trim(kwargs.get("access_token", ""))
gist_id = utils.trim(kwargs.get("gist_id", ""))
username = utils.trim(kwargs.get("username", ""))
- chuck = kwargs.get("chuck", False)
+ skip_captcha = kwargs.get("skip_captcha", False)
# 加载已有订阅
subscriptions = load_exist(username, gist_id, access_token, subscribes_file)
@@ -114,7 +96,12 @@ def parse_domains(content: str) -> dict:
tasks = (
[
- TaskConfig(name=utils.random_chars(length=8), sub=x, bin_name=bin_name, special_protocols=special_protocols)
+ TaskConfig(
+ name=utils.random_chars(length=8),
+ nodes=NodeInput(subscribe=x),
+ bin_name=bin_name,
+ special_protocols=special_protocols,
+ )
for x in subscriptions
if x
]
@@ -127,7 +114,8 @@ def parse_domains(content: str) -> dict:
logger.info("skip registering new accounts, will use existing subscriptions for refreshing")
return tasks
- domains, delimiter = {}, "@#@#"
+ records: dict[str, AirportRecord] = {}
+ delimiter = "@#@#"
domains_file = utils.trim(domains_file)
if not domains_file:
domains_file = "domains.txt"
@@ -135,65 +123,72 @@ def parse_domains(content: str) -> dict:
# 加载已有站点列表
fullpath = os.path.join(DATA_BASE, domains_file)
if os.path.exists(fullpath) and os.path.isfile(fullpath):
- with open(fullpath, "r", encoding="UTF8") as f:
- domains.update(parse_domains(content=str(f.read())))
+ with open(fullpath, "r", encoding="UTF8") as handle:
+ records.update(parse_records(content=str(handle.read()), delimiter=delimiter))
# 爬取新站点列表
- if not domains or overwrite:
- candidates = crawl.collect_airport(
+ if not records or overwrite:
+ crawled = collect_airport(
channel="jichang_list",
page_num=pages,
- num_thread=num_threads,
- rigid=rigid,
+ num_threads=num_threads,
+ allow_gmail_alias=allow_gmail_alias,
display=display,
filepath=os.path.join(DATA_BASE, "coupons.txt"),
delimiter=delimiter,
- chuck=chuck,
+ skip_captcha=skip_captcha,
)
- if candidates:
- for k, v in candidates.items():
- item = domains.get(k, {})
- item.update(v)
-
- domains[k] = item
-
+ if crawled:
+ for domain, incoming in crawled.items():
+ current = records.get(domain)
+ if current is None:
+ records[domain] = incoming
+ else:
+ current.coupon = incoming.coupon
+ current.api_prefix = incoming.api_prefix
overwrite = True
# 加载自定义机场列表
customize_link = utils.trim(kwargs.get("customize_link", ""))
if customize_link:
if isurl(customize_link):
- domains.update(parse_domains(content=utils.http_get(url=customize_link)))
+ records.update(parse_records(content=utils.http_get(url=customize_link), delimiter=delimiter))
else:
local_file = os.path.join(DATA_BASE, customize_link)
if local_file != fullpath and os.path.exists(local_file) and os.path.isfile(local_file):
- with open(local_file, "r", encoding="UTF8") as f:
- domains.update(parse_domains(content=str(f.read())))
+ with open(local_file, "r", encoding="UTF8") as handle:
+ records.update(parse_records(content=str(handle.read()), delimiter=delimiter))
- if not domains:
+ if not records:
logger.error("cannot collect any new airport for free use")
return tasks
if overwrite:
- crawl.save_candidates(candidates=domains, filepath=fullpath, delimiter=delimiter)
+ save_records(records=records, filepath=fullpath, delimiter=delimiter, full=True)
- for domain, param in domains.items():
- name = crawl.naming_task(url=domain)
- tasks.append(
- TaskConfig(
+ sites = []
+ for domain, record in records.items():
+ name = naming_task(url=domain)
+ sites.append(
+ SiteConfig(
name=name,
domain=domain,
- coupon=param.get("coupon", ""),
- invite_code=param.get("invite_code", ""),
- api_prefix=param.get("api_prefix", ""),
- bin_name=bin_name,
- rigid=rigid,
- chuck=chuck,
- special_protocols=special_protocols,
+ coupon=record.coupon,
+ invite_code=record.invite_code,
+ api_prefix=record.api_prefix or "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/api/v1/",
+ skip_captcha=skip_captcha,
)
)
+ assigned, _ = pipeline.assign_sites(
+ sites=sites,
+ groups={},
+ retry=3,
+ bin_name=bin_name,
+ allow_gmail_alias=allow_gmail_alias,
+ )
+ tasks.extend(assigned)
return tasks
@@ -219,16 +214,16 @@ def parse_gist_link(link: str) -> tuple[str, str]:
domains_file="domains.txt",
overwrite=args.overwrite,
pages=args.pages,
- rigid=not args.easygoing,
+ allow_gmail_alias=args.easygoing,
display=display,
num_threads=args.num,
refresh=args.refresh,
- chuck=args.chuck,
+ skip_captcha=args.skip_captcha,
username=username,
gist_id=gist_id,
access_token=access_token,
subscribes_file=subscribes_file,
- customize_link=args.yourself,
+ customize_link=args.custom_sites,
)
if not tasks:
@@ -236,14 +231,16 @@ def parse_gist_link(link: str) -> tuple[str, str]:
sys.exit(0)
# 已有订阅已经做过过期检查，无需再测
- old_subscriptions = set([t.sub for t in tasks if t.sub])
+ old_subscriptions = set()
+ for task in tasks:
+ old_subscriptions.update(task.nodes.subscribe_list())
logger.info(f"start generate subscribes information, tasks: {len(tasks)}")
generate_conf = os.path.join(PATH, "subconverter", "generate.ini")
if os.path.exists(generate_conf) and os.path.isfile(generate_conf):
os.remove(generate_conf)
- results = utils.multi_thread_run(func=workflow.executewrapper, tasks=tasks, num_threads=args.num)
+ results = pipeline.execute_tasks(tasks)
proxies = list(itertools.chain.from_iterable([x[1] for x in results if x]))
if len(proxies) == 0:
@@ -300,7 +297,6 @@ def parse_gist_link(link: str) -> tuple[str, str]:
subscriptions = set()
for p in proxies:
# 移除无用的标记
- p.pop("chatgpt", False)
p.pop("liveness", True)
sub = p.pop("sub", "")
@@ -332,7 +328,7 @@ def parse_gist_link(link: str) -> tuple[str, str]:
filename = subconverter.get_filename(target=target)
list_only = False if target == "v2ray" or target == "mixed" or "ss" in target else not args.all
- targets.append((convert_name, filename, target, list_only, args.vitiate))
+ targets.append((convert_name, filename, target, list_only, args.ignore_default_filters))
for t in targets:
success = subconverter.generate_conf(generate_conf, t[0], source, t[1], t[2], True, t[3], t[4])
@@ -361,7 +357,7 @@ def parse_gist_link(link: str) -> tuple[str, str]:
tasks = [[x, 2, traffic, life, 0, True] for x in new_subscriptions]
results = utils.multi_thread_run(
- func=crawl.check_status,
+ func=check_status,
tasks=tasks,
num_threads=args.num,
show_progress=display,
@@ -386,7 +382,7 @@ def parse_gist_link(link: str) -> tuple[str, str]:
# 如有必要，上传至 Gist
if gist_id and access_token:
- files, config = {}, {"gistid": gist_id, "filename": list(records.keys())[0]}
+ files, item = {}, StorageItem(gist_id=gist_id, filename=list(records.keys())[0])
for k, v in records.items():
if os.path.exists(v) and os.path.isfile(v):
@@ -402,7 +398,7 @@ def parse_gist_link(link: str) -> tuple[str, str]:
push_client = push.PushToGist(token=access_token)
# 上传
- success = push_client.push_to(content="", config=config, payload={"files": files}, group="collect")
+ success = push_client.push_to(content="", item=item, payload={"files": files}, group="collect")
if success:
logger.info(f"upload proxies and subscriptions to gist successed")
else:
@@ -413,7 +409,7 @@ def parse_gist_link(link: str) -> tuple[str, str]:
class CustomHelpFormatter(argparse.HelpFormatter):
- def _format_action_invocation(self, action):
+ def _format_action_invocation(self, action: argparse.Action) -> str:
if action.choices:
parts = []
if action.option_strings:
@@ -433,7 +429,17 @@ def _format_action_invocation(self, action):
if __name__ == "__main__":
- parser = argparse.ArgumentParser(formatter_class=CustomHelpFormatter)
+ env_parser = argparse.ArgumentParser(add_help=False)
+ env_parser.add_argument(
+ "--environment",
+ type=str,
+ default=".env",
+ help="environment file name",
+ )
+ env_args, _ = env_parser.parse_known_args()
+ utils.load_dotenv(env_args.environment)
+
+ parser = argparse.ArgumentParser(formatter_class=CustomHelpFormatter, parents=[env_parser])
parser.add_argument(
"-a",
"--all",
@@ -445,8 +451,8 @@ def _format_action_invocation(self, action):
parser.add_argument(
"-c",
- "--chuck",
- dest="chuck",
+ "--skip-captcha",
+ dest="skip_captcha",
action="store_true",
default=False,
help="discard candidate sites that may require human-authentication",
@@ -555,8 +561,8 @@ def _format_action_invocation(self, action):
"-s",
"--skip",
dest="skip",
- action="store_true",
- default=False,
+ action=argparse.BooleanOptionalAction,
+ default=utils.env_bool("SKIP_ALIVE_CHECK", False),
help="skip usability checks",
)
@@ -580,8 +586,8 @@ def _format_action_invocation(self, action):
parser.add_argument(
"-v",
- "--vitiate",
- dest="vitiate",
+ "--ignore-default-filters",
+ dest="ignore_default_filters",
action="store_true",
default=False,
help="ignoring default proxies filter rules",
@@ -589,7 +595,7 @@ def _format_action_invocation(self, action):
parser.add_argument(
"-y",
- "--yourself",
+ "--custom-sites",
type=str,
required=False,
default=os.environ.get("CUSTOMIZE_LINK", ""),
diff --git a/subscribe/config/.gitkeep b/subscribe/config/.gitkeep
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/subscribe/config/__init__.py b/subscribe/config/__init__.py
new file mode 100644
index 0000000000..274046a38b
--- /dev/null
+++ b/subscribe/config/__init__.py
@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+
+from config.models import (
+ CONVERT_TARGETS,
+ CrawlConfig,
+ GithubConfig,
+ GoogleConfig,
+ GroupConfig,
+ NodeInput,
+ PageJob,
+ ProcessConfig,
+ RegularizeConfig,
+ RenewConfig,
+ RenewJob,
+ RepoConfig,
+ ScriptJob,
+ SiteConfig,
+ SourceRule,
+ StorageConfig,
+ StorageItem,
+ TaskParams,
+ TelegramConfig,
+ TwitterConfig,
+ UpdateConfig,
+ YandexConfig,
+)
+
+__all__ = [
+ "CONVERT_TARGETS",
+ "CrawlConfig",
+ "GithubConfig",
+ "GoogleConfig",
+ "GroupConfig",
+ "NodeInput",
+ "PageJob",
+ "ProcessConfig",
+ "RegularizeConfig",
+ "RenewConfig",
+ "RenewJob",
+ "RepoConfig",
+ "ScriptJob",
+ "SiteConfig",
+ "SourceRule",
+ "StorageConfig",
+ "StorageItem",
+ "TaskParams",
+ "TelegramConfig",
+ "TwitterConfig",
+ "UpdateConfig",
+ "YandexConfig",
+]
diff --git a/subscribe/config/models.py b/subscribe/config/models.py
new file mode 100644
index 0000000000..c7b6d31119
--- /dev/null
+++ b/subscribe/config/models.py
@@ -0,0 +1,1056 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+import re
+from copy import deepcopy
+from dataclasses import dataclass, field, fields
+from typing import TYPE_CHECKING, TypeVar
+
+import utils
+from config.parse import Node, ObjectNode
+from origin import Origin
+
+if TYPE_CHECKING:
+ from push import PushTo
+
+TValue = TypeVar("TValue")
+
+CONVERT_TARGETS = [
+ "clash",
+ "v2ray",
+ "singbox",
+ "mixed",
+ "clashr",
+ "quan",
+ "quanx",
+ "loon",
+ "ss",
+ "sssub",
+ "ssd",
+ "ssr",
+ "surfboard",
+ "surge",
+]
+
+
+def _parse_source_rule(obj: ObjectNode) -> dict[str, object]:
+ push_to = obj.string_list("push_to")
+ task = TaskParams.parse(obj.field("task"))
+ if task.push_to is None:
+ task.push_to = list(push_to)
+ return {
+ "include": obj.regex("include", default=""),
+ "exclude": obj.regex("exclude", default=""),
+ "push_to": push_to,
+ "task": task,
+ }
+
+
+@dataclass
+class NodeInput:
+ subscribe: str | list[str] = ""
+ uris: list[str] = field(default_factory=list)
+ proxies: list[dict[str, object]] = field(default_factory=list)
+
+ def empty(self) -> bool:
+ subs = self.subscribe if isinstance(self.subscribe, list) else [self.subscribe]
+ return (
+ not any(utils.trim(str(item)) for item in subs if item is not None) and not self.uris and not self.proxies
+ )
+
+ def subscribe_list(self) -> list[str]:
+ if isinstance(self.subscribe, list):
+ return [utils.trim(item) for item in self.subscribe if utils.trim(item)]
+ text = utils.trim(self.subscribe)
+ return [text] if text else []
+
+
+@dataclass
+class TaskParams:
+ name: str | None = None
+ rename: str | None = None
+ include: str | None = None
+ exclude: str | None = None
+ push_to: list[str] | None = None
+ check_alive: bool | None = None
+ ignore_default_exclude: bool | None = None
+ require_tls: bool | None = None
+ max_rate: float | None = None
+ coupon: str | None = None
+ invite_code: str | None = None
+ api_prefix: str | None = None
+ enable: bool | None = None
+
+ @classmethod
+ def parse(cls, node: Node) -> TaskParams:
+ if node.absent:
+ return cls()
+ obj = node.object()
+ return cls(
+ name=obj.string("name") if obj.has("name") else None,
+ rename=obj.string("rename") if obj.has("rename") else None,
+ include=obj.regex("include") if obj.has("include") else None,
+ exclude=obj.regex("exclude") if obj.has("exclude") else None,
+ push_to=obj.string_list("push_to") if obj.has("push_to") else None,
+ check_alive=obj.boolean("check_alive"),
+ ignore_default_exclude=obj.boolean("ignore_default_exclude"),
+ require_tls=obj.boolean("require_tls"),
+ max_rate=obj.number("max_rate", minimum=0),
+ coupon=obj.string("coupon") if obj.has("coupon") else None,
+ invite_code=obj.string("invite_code") if obj.has("invite_code") else None,
+ api_prefix=obj.string("api_prefix") if obj.has("api_prefix") else None,
+ enable=obj.boolean("enable"),
+ )
+
+ def merge(self, override: TaskParams) -> TaskParams:
+ def pick(current: TValue, incoming: TValue) -> TValue:
+ return incoming if incoming is not None else current
+
+ return TaskParams(
+ name=pick(self.name, override.name),
+ rename=pick(self.rename, override.rename),
+ include=pick(self.include, override.include),
+ exclude=pick(self.exclude, override.exclude),
+ push_to=pick(self.push_to, override.push_to),
+ check_alive=pick(self.check_alive, override.check_alive),
+ ignore_default_exclude=pick(self.ignore_default_exclude, override.ignore_default_exclude),
+ require_tls=pick(self.require_tls, override.require_tls),
+ max_rate=pick(self.max_rate, override.max_rate),
+ coupon=pick(self.coupon, override.coupon),
+ invite_code=pick(self.invite_code, override.invite_code),
+ api_prefix=pick(self.api_prefix, override.api_prefix),
+ enable=pick(self.enable, override.enable),
+ )
+
+ def apply_to_site(self, site: SiteConfig) -> None:
+ if self.name is not None and not site.name:
+ site.name = self.name
+ if self.rename is not None and not site.rename:
+ site.rename = self.rename
+ if self.include is not None and not site.include:
+ site.include = self.include
+ if self.exclude is not None and not site.exclude:
+ site.exclude = self.exclude
+ if self.push_to is not None and not site.push_to:
+ site.push_to = list(self.push_to)
+ if self.check_alive is not None and site.check_alive is True:
+ site.check_alive = self.check_alive
+ if self.ignore_default_exclude is not None:
+ site.ignore_default_exclude = self.ignore_default_exclude
+ if self.require_tls is not None:
+ site.require_tls = self.require_tls
+ if self.max_rate is not None:
+ site.max_rate = self.max_rate
+ if self.coupon is not None and not site.coupon:
+ site.coupon = self.coupon
+ if self.invite_code is not None and not site.invite_code:
+ site.invite_code = self.invite_code
+ if self.api_prefix is not None and site.api_prefix == "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/api/v1/":
+ site.api_prefix = self.api_prefix
+
+ def to_dict(self) -> dict[str, object]:
+ payload = {}
+ names = (
+ "name",
+ "rename",
+ "include",
+ "exclude",
+ "push_to",
+ "check_alive",
+ "ignore_default_exclude",
+ "require_tls",
+ "max_rate",
+ "coupon",
+ "invite_code",
+ "api_prefix",
+ "enable",
+ )
+ for key in names:
+ value = getattr(self, key)
+ if value is not None:
+ payload[key] = list(value) if key == "push_to" else value
+ return payload
+
+
+@dataclass
+class TicketConfig:
+ enable: bool = True
+ auto_reset: bool = False
+ subject: str = ""
+ message: str = ""
+ level: int = 1
+
+ @classmethod
+ def parse(cls, node: Node) -> TicketConfig | None:
+ if node.absent:
+ return None
+ obj = node.object()
+ return cls(
+ enable=obj.boolean("enable", default=True),
+ auto_reset=obj.boolean("auto_reset", default=False),
+ subject=obj.string("subject", default="") or "",
+ message=obj.string("message", default="") or "",
+ level=obj.integer("level", default=1, minimum=1, maximum=3) or 1,
+ )
+
+ def to_dict(self) -> dict[str, object]:
+ return {
+ "enable": self.enable,
+ "auto_reset": self.auto_reset,
+ "subject": self.subject,
+ "message": self.message,
+ "level": self.level,
+ }
+
+
+@dataclass
+class RenewAccount:
+ email: str = ""
+ password: str = ""
+ ticket: TicketConfig | None = None
+
+ @classmethod
+ def parse(cls, node: Node) -> RenewAccount:
+ obj = node.object()
+ return cls(
+ email=obj.string("email", default="") or "",
+ password=obj.string("password", default="") or "",
+ ticket=TicketConfig.parse(obj.field("ticket")),
+ )
+
+ def to_dict(self) -> dict[str, object]:
+ payload = {"email": self.email, "password": self.password}
+ if self.ticket is not None:
+ payload["ticket"] = self.ticket.to_dict()
+ return payload
+
+
+@dataclass
+class RenewJob:
+ account: RenewAccount
+ plan_id: int | None = None
+ package: str = ""
+ method: int | None = None
+ coupon_code: str = ""
+ api_prefix: str = "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/api/v1/"
+ enable: bool = True
+
+
+@dataclass
+class RenewConfig:
+ accounts: list[RenewAccount] = field(default_factory=list)
+ plan_id: int | None = None
+ package: str = ""
+ method: int | None = None
+ coupon_code: str = ""
+
+ @classmethod
+ def parse(cls, node: Node) -> RenewConfig | None:
+ if node.absent:
+ return None
+ obj = node.object()
+ accounts = []
+ if obj.has("accounts"):
+ accounts = [RenewAccount.parse(item) for item in obj.field("accounts").array()]
+ return cls(
+ accounts=accounts,
+ plan_id=obj.integer("plan_id"),
+ package=obj.string("package", default="") or "",
+ method=obj.integer("method"),
+ coupon_code=obj.string("coupon_code", default="") or "",
+ )
+
+ def jobs(self, coupon: str = "", api_prefix: str = "") -> list[RenewJob]:
+ coupon_code = coupon or self.coupon_code
+ prefix = api_prefix or "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/api/v1/"
+ return [
+ RenewJob(
+ account=account,
+ plan_id=self.plan_id,
+ package=self.package,
+ method=self.method,
+ coupon_code=coupon_code,
+ api_prefix=prefix,
+ )
+ for account in self.accounts
+ ]
+
+ def to_dict(self) -> dict[str, object]:
+ payload = {
+ "accounts": [item.to_dict() for item in self.accounts],
+ "package": self.package,
+ "coupon_code": self.coupon_code,
+ }
+ if self.plan_id is not None:
+ payload["plan_id"] = self.plan_id
+ if self.method is not None:
+ payload["method"] = self.method
+ return payload
+
+
+@dataclass
+class SiteConfig:
+ name: str = ""
+ enable: bool = True
+ domain: str = ""
+ nodes: NodeInput = field(default_factory=NodeInput)
+ push_to: list[str] = field(default_factory=list)
+ rename: str = ""
+ include: str = ""
+ exclude: str = ""
+ ignore_default_exclude: bool = False
+ check_alive: bool = True
+ max_rate: float = 3.0
+ require_tls: bool = False
+ skip_captcha: bool = False
+ count: int = 1
+ coupon: str = ""
+ invite_code: str = ""
+ api_prefix: str = "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/api/v1/"
+ origin: str = ""
+ errors: int = 0
+ debut: bool = False
+ renew: RenewConfig | None = None
+ skip_cache: bool = False
+ allow_nonstandard: bool = False
+ persist_only: bool = False
+ discovered: bool = False
+
+ @classmethod
+ def parse(cls, node: Node) -> SiteConfig:
+ obj = node.object()
+ return cls(
+ name=obj.string("name", default="") or "",
+ enable=obj.boolean("enable", default=True),
+ domain=obj.string("domain", default="") or "",
+ nodes=NodeInput(subscribe=obj.string_or_list("subscribe", default="")),
+ push_to=obj.string_list("push_to"),
+ rename=obj.string("rename", default="") or "",
+ include=obj.regex("include", default=""),
+ exclude=obj.regex("exclude", default=""),
+ ignore_default_exclude=obj.boolean("ignore_default_exclude", default=False),
+ check_alive=obj.boolean("check_alive", default=True),
+ max_rate=obj.number("max_rate", default=3.0, minimum=0) or 0,
+ require_tls=obj.boolean("require_tls", default=False),
+ skip_captcha=obj.boolean("skip_captcha", default=False),
+ count=obj.integer("count", default=1, minimum=1, maximum=10) or 1,
+ coupon=obj.string("coupon", default="") or "",
+ invite_code=obj.string("invite_code", default="") or "",
+ api_prefix=obj.string("api_prefix", default="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/api/v1/") or "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/api/v1/",
+ origin=obj.string("origin", default="") or "",
+ errors=obj.integer("errors", default=0, minimum=0) or 0,
+ debut=obj.boolean("debut", default=False),
+ renew=RenewConfig.parse(obj.field("renew")),
+ )
+
+ def to_dict(self) -> dict[str, object]:
+ payload = {
+ "name": self.name,
+ "enable": self.enable,
+ "domain": self.domain,
+ "subscribe": deepcopy(self.nodes.subscribe),
+ "push_to": list(self.push_to),
+ "rename": self.rename,
+ "include": self.include,
+ "exclude": self.exclude,
+ "ignore_default_exclude": self.ignore_default_exclude,
+ "check_alive": self.check_alive,
+ "max_rate": self.max_rate,
+ "require_tls": self.require_tls,
+ "skip_captcha": self.skip_captcha,
+ "count": self.count,
+ "coupon": self.coupon,
+ "invite_code": self.invite_code,
+ "api_prefix": self.api_prefix,
+ "origin": self.origin,
+ "errors": self.errors,
+ "debut": self.debut,
+ }
+ if self.renew is not None:
+ payload["renew"] = self.renew.to_dict()
+ return payload
+
+
+@dataclass
+class SourceRule:
+ include: str = ""
+ exclude: str = ""
+ push_to: list[str] = field(default_factory=list)
+ task: TaskParams = field(default_factory=TaskParams)
+
+
+@dataclass
+class TelegramChannelConfig(SourceRule):
+ @classmethod
+ def parse(cls, node: Node) -> TelegramChannelConfig:
+ obj = node.object()
+ return cls(**_parse_source_rule(obj))
+
+ def to_dict(self) -> dict[str, object]:
+ payload = {
+ "include": self.include,
+ "exclude": self.exclude,
+ "push_to": list(self.push_to),
+ }
+ task = self.task.to_dict()
+ if task:
+ payload["task"] = task
+ return payload
+
+
+@dataclass
+class TwitterUserConfig(SourceRule):
+ tweets: int = 10
+ enable: bool = True
+
+ @classmethod
+ def parse(cls, node: Node) -> TwitterUserConfig:
+ obj = node.object()
+ payload = _parse_source_rule(obj)
+ payload["tweets"] = obj.integer("tweets", default=10, minimum=1, maximum=100) or 10
+ payload["enable"] = obj.boolean("enable", default=True)
+ return cls(**payload)
+
+ def to_dict(self) -> dict[str, object]:
+ payload = {
+ "enable": self.enable,
+ "tweets": self.tweets,
+ "include": self.include,
+ "exclude": self.exclude,
+ "push_to": list(self.push_to),
+ }
+ task = self.task.to_dict()
+ if task:
+ payload["task"] = task
+ return payload
+
+
+@dataclass
+class PageJob(SourceRule):
+ url: str | list[str] = ""
+ enable: bool = True
+ paged: bool = False
+ placeholder: str = ""
+ start: int = 1
+ end: int = 1
+ headers: dict[str, str] | None = None
+ origin: str = Origin.PAGE.name
+ skip_cache: bool = False
+
+ @classmethod
+ def parse(cls, node: Node) -> PageJob:
+ obj = node.object()
+ payload = _parse_source_rule(obj)
+ headers = obj.field("headers").value
+ payload.update(
+ {
+ "url": obj.string_or_list("url", default=""),
+ "enable": obj.boolean("enable", default=True),
+ "paged": obj.boolean("paged", default=False),
+ "placeholder": obj.string("placeholder", default="") or "",
+ "start": obj.integer("start", default=1, minimum=0) or 0,
+ "end": obj.integer("end", default=1, minimum=0) or 0,
+ "headers": headers if isinstance(headers, dict) else None,
+ "skip_cache": obj.boolean("skip_cache", default=False),
+ }
+ )
+ return cls(**payload)
+
+ def expand_urls(self) -> list[str]:
+ if self.paged:
+ if not self.placeholder or not isinstance(self.url, str) or self.placeholder not in self.url:
+ return []
+ if self.end < self.start:
+ return []
+ return [self.url.replace(self.placeholder, str(page)) for page in range(self.start, self.end + 1)]
+ urls = self.url if isinstance(self.url, list) else [self.url]
+ return [utils.trim(str(link)) for link in urls if utils.trim(str(link))]
+
+ def to_dict(self) -> dict[str, object]:
+ payload = {
+ "enable": self.enable,
+ "url": deepcopy(self.url),
+ "include": self.include,
+ "exclude": self.exclude,
+ "push_to": list(self.push_to),
+ }
+ if self.paged:
+ payload["paged"] = True
+ payload["placeholder"] = self.placeholder
+ payload["start"] = self.start
+ payload["end"] = self.end
+ if self.skip_cache:
+ payload["skip_cache"] = True
+ if self.headers:
+ payload["headers"] = dict(self.headers)
+ task = self.task.to_dict()
+ if task:
+ payload["task"] = task
+ return payload
+
+
+@dataclass
+class GoogleConfig:
+ enable: bool = True
+ push_to: list[str] = field(default_factory=list)
+ exclude: str = ""
+ limit: int = 100
+ days: int = 7
+ exclude_sites: list[str] = field(default_factory=list)
+
+ @classmethod
+ def parse(cls, node: Node) -> GoogleConfig | None:
+ if node.absent:
+ return None
+ obj = node.object()
+ return cls(
+ enable=obj.boolean("enable", default=True),
+ push_to=obj.string_list("push_to"),
+ exclude=obj.regex("exclude", default=""),
+ limit=obj.integer("limit", default=100, minimum=1, maximum=1000) or 100,
+ days=obj.integer("days", default=7, minimum=1) or 7,
+ exclude_sites=obj.string_list("exclude_sites"),
+ )
+
+ def to_dict(self) -> dict[str, object]:
+ return {
+ "enable": self.enable,
+ "push_to": list(self.push_to),
+ "exclude": self.exclude,
+ "limit": self.limit,
+ "days": self.days,
+ "exclude_sites": list(self.exclude_sites),
+ }
+
+
+@dataclass
+class YandexConfig:
+ enable: bool = True
+ push_to: list[str] = field(default_factory=list)
+ exclude: str = ""
+ days: int = 2
+ pages: int = 5
+ exclude_sites: list[str] = field(default_factory=list)
+
+ @classmethod
+ def parse(cls, node: Node) -> YandexConfig | None:
+ if node.absent:
+ return None
+ obj = node.object()
+ return cls(
+ enable=obj.boolean("enable", default=True),
+ push_to=obj.string_list("push_to"),
+ exclude=obj.regex("exclude", default=""),
+ days=obj.integer("days", default=2, minimum=0) or 0,
+ pages=obj.integer("pages", default=5, minimum=1) or 5,
+ exclude_sites=obj.string_list("exclude_sites"),
+ )
+
+ def to_dict(self) -> dict[str, object]:
+ return {
+ "enable": self.enable,
+ "push_to": list(self.push_to),
+ "exclude": self.exclude,
+ "days": self.days,
+ "pages": self.pages,
+ "exclude_sites": list(self.exclude_sites),
+ }
+
+
+@dataclass
+class GithubConfig:
+ enable: bool = True
+ pages: int = 1
+ push_to: list[str] = field(default_factory=list)
+ exclude: str = ""
+ exclude_repos: list[str] = field(default_factory=list)
+
+ @classmethod
+ def parse(cls, node: Node) -> GithubConfig | None:
+ if node.absent:
+ return None
+ obj = node.object()
+ repos = obj.string_list("exclude_repos")
+ for index, pattern in enumerate(repos):
+ try:
+ re.compile(pattern)
+ except re.error as exc:
+ obj.field("exclude_repos").fail(f"[{index}] is not a valid regex: {exc}")
+ return cls(
+ enable=obj.boolean("enable", default=True),
+ pages=obj.integer("pages", default=1, minimum=1) or 1,
+ push_to=obj.string_list("push_to"),
+ exclude=obj.regex("exclude", default=""),
+ exclude_repos=repos,
+ )
+
+ def to_dict(self) -> dict[str, object]:
+ return {
+ "enable": self.enable,
+ "pages": self.pages,
+ "push_to": list(self.push_to),
+ "exclude": self.exclude,
+ "exclude_repos": list(self.exclude_repos),
+ }
+
+
+@dataclass
+class RepoConfig:
+ username: str = ""
+ repo: str = ""
+ commits: int = 3
+ push_to: list[str] = field(default_factory=list)
+ exclude: str = ""
+ enable: bool = True
+
+ @classmethod
+ def parse(cls, node: Node) -> RepoConfig:
+ obj = node.object()
+ return cls(
+ username=obj.string("username", default="") or "",
+ repo=obj.string("repo", default="") or "",
+ commits=obj.integer("commits", default=3, minimum=1) or 3,
+ push_to=obj.string_list("push_to"),
+ exclude=obj.regex("exclude", default=""),
+ enable=obj.boolean("enable", default=True),
+ )
+
+ def to_dict(self) -> dict[str, object]:
+ return {
+ "enable": self.enable,
+ "username": self.username,
+ "repo": self.repo,
+ "commits": self.commits,
+ "push_to": list(self.push_to),
+ "exclude": self.exclude,
+ }
+
+
+@dataclass
+class TelegramConfig:
+ enable: bool = True
+ pages: int = 1
+ exclude: str = ""
+ channels: dict[str, TelegramChannelConfig] = field(default_factory=dict)
+
+ @classmethod
+ def parse(cls, node: Node) -> TelegramConfig | None:
+ if node.absent:
+ return None
+ obj = node.object()
+ channels = {}
+ if obj.has("channels"):
+ mapping = obj.field("channels").object()
+ for name in mapping.keys():
+ key = utils.trim(str(name))
+ if not key:
+ continue
+ channels[key] = TelegramChannelConfig.parse(mapping.field(name))
+ return cls(
+ enable=obj.boolean("enable", default=True),
+ pages=obj.integer("pages", default=1, minimum=1) or 1,
+ exclude=obj.regex("exclude", default=""),
+ channels=channels,
+ )
+
+ def to_dict(self) -> dict[str, object]:
+ return {
+ "enable": self.enable,
+ "pages": self.pages,
+ "exclude": self.exclude,
+ "channels": {name: item.to_dict() for name, item in self.channels.items()},
+ }
+
+
+@dataclass
+class TwitterConfig:
+ enable: bool = True
+ users: dict[str, TwitterUserConfig] = field(default_factory=dict)
+
+ @classmethod
+ def parse(cls, node: Node) -> TwitterConfig | None:
+ if node.absent:
+ return None
+ obj = node.object()
+ users = {}
+ if obj.has("users"):
+ mapping = obj.field("users").object()
+ for name in mapping.keys():
+ key = utils.trim(str(name))
+ if not key:
+ continue
+ users[key] = TwitterUserConfig.parse(mapping.field(name))
+ return cls(enable=obj.boolean("enable", default=True), users=users)
+
+ def to_dict(self) -> dict[str, object]:
+ return {
+ "enable": self.enable,
+ "users": {name: item.to_dict() for name, item in self.users.items()},
+ }
+
+
+@dataclass
+class ScriptJob:
+ plugin: str
+ enable: bool = True
+ persist: str | dict[str, str] | None = None
+ task: TaskParams = field(default_factory=TaskParams)
+ options: dict[str, object] = field(default_factory=dict)
+
+ @classmethod
+ def parse(cls, node: Node) -> ScriptJob:
+ obj = node.object()
+ persist_node = obj.field("persist")
+ persist = None
+ if not persist_node.absent:
+ if isinstance(persist_node.value, str):
+ persist = utils.trim(persist_node.value)
+ elif isinstance(persist_node.value, dict):
+ persist = {}
+ for key, value in persist_node.value.items():
+ name = utils.trim(str(key))
+ if not name:
+ continue
+ persist[name] = utils.trim(str(value)) if isinstance(value, str) else value
+ else:
+ persist_node.fail("must be a string or object")
+ return cls(
+ plugin=obj.string("plugin", required=True, allow_empty=False),
+ enable=obj.boolean("enable", default=True),
+ persist=persist,
+ task=TaskParams.parse(obj.field("task")),
+ options=obj.rest({"plugin", "enable", "persist", "task"}),
+ )
+
+ def to_dict(self) -> dict[str, object]:
+ payload = {"plugin": self.plugin, "enable": self.enable}
+ if self.persist is not None:
+ payload["persist"] = deepcopy(self.persist)
+ task = self.task.to_dict()
+ if task:
+ payload["task"] = task
+ payload.update(self.options)
+ return payload
+
+
+@dataclass
+class RegularizeConfig:
+ enable: bool = False
+ locate: bool = True
+ residential: bool = True
+ digits: int = 2
+ library: str = ""
+ score: bool = False
+
+ @classmethod
+ def parse(cls, node: Node) -> RegularizeConfig | None:
+ if node.absent:
+ return None
+ obj = node.object()
+ return cls(
+ enable=obj.boolean("enable", default=False),
+ locate=obj.boolean("locate", default=True),
+ residential=obj.boolean("residential", default=True),
+ digits=obj.integer("digits", default=2, minimum=1) or 2,
+ library=obj.string("library", default="") or "",
+ score=obj.boolean("score", default=False),
+ )
+
+ def to_dict(self) -> dict[str, object]:
+ return {item.name: getattr(self, item.name) for item in fields(self)}
+
+
+@dataclass
+class GroupConfig:
+ name: str
+ emoji: bool = True
+ list_only: bool = True
+ targets: dict[str, str] = field(default_factory=dict)
+ regularize: RegularizeConfig | None = None
+
+ @classmethod
+ def parse(cls, name: str, node: Node) -> GroupConfig:
+ obj = node.object()
+ targets = {}
+ if obj.has("targets"):
+ mapping = obj.field("targets").object()
+ for key in mapping.keys():
+ category = utils.trim(str(key)).lower()
+ if not category:
+ mapping.field(key).fail("contains an empty type")
+ if category not in CONVERT_TARGETS:
+ mapping.field(key).fail(f"contains unsupported type: {category}")
+ value = mapping.field(key).value
+ if not isinstance(value, str) or not utils.trim(value):
+ mapping.field(key).fail("must be a storage item name")
+ targets[category] = utils.trim(value)
+ if not targets:
+ node.fail("should contain at least one conversion target")
+ return cls(
+ name=name,
+ emoji=obj.boolean("emoji", default=True),
+ list_only=obj.boolean("list_only", default=True),
+ targets=targets,
+ regularize=RegularizeConfig.parse(obj.field("regularize")),
+ )
+
+ def to_dict(self) -> dict[str, object]:
+ payload = {
+ "emoji": self.emoji,
+ "list_only": self.list_only,
+ "targets": dict(self.targets),
+ }
+ if self.regularize is not None:
+ payload["regularize"] = self.regularize.to_dict()
+ return payload
+
+
+@dataclass
+class StorageItem:
+ username: str = ""
+ folder_id: str = ""
+ file_id: str = ""
+ gist_id: str = ""
+ filename: str = ""
+ revision: str = ""
+ password: str = ""
+ expire: int = 0
+ local: str = ""
+
+ @classmethod
+ def parse(cls, node: Node) -> StorageItem:
+ obj = node.object()
+ return cls(
+ username=obj.string("username", default="") or "",
+ folder_id=obj.string("folder_id", default="") or "",
+ file_id=obj.string("file_id", default="") or "",
+ gist_id=obj.string("gist_id", default="") or "",
+ filename=obj.string("filename", default="") or "",
+ revision=obj.string("revision", default="") or "",
+ password=obj.string("password", default="") or "",
+ expire=obj.integer("expire", default=0) or 0,
+ local=obj.string("local", default="") or "",
+ )
+
+ def to_dict(self) -> dict[str, object]:
+ return {item.name: getattr(self, item.name) for item in fields(self)}
+
+
+@dataclass
+class StorageConfig:
+ engine: str = ""
+ token: str = ""
+ base: str = ""
+ domain: str = ""
+ items: dict[str, StorageItem] = field(default_factory=dict)
+
+ @classmethod
+ def parse(cls, node: Node) -> StorageConfig:
+ if node.absent:
+ return cls()
+ obj = node.object()
+ items = {}
+ if obj.has("items"):
+ mapping = obj.field("items").object()
+ for name in mapping.keys():
+ key = utils.trim(str(name))
+ if not key:
+ continue
+ items[key] = StorageItem.parse(mapping.field(name))
+ return cls(
+ engine=obj.string("engine", default="") or "",
+ token=obj.string("token", default="") or "",
+ base=obj.string("base", default="") or "",
+ domain=obj.string("domain", default="") or "",
+ items=items,
+ )
+
+ def to_dict(self) -> dict[str, object]:
+ return {
+ "engine": self.engine,
+ "token": self.token,
+ "base": self.base,
+ "domain": self.domain,
+ "items": {name: item.to_dict() for name, item in self.items.items()},
+ }
+
+
+@dataclass
+class UpdateConfig:
+ enable: bool = False
+ item: StorageItem | None = None
+
+ @classmethod
+ def parse(cls, node: Node) -> UpdateConfig:
+ if node.absent:
+ return cls()
+ obj = node.object()
+ item = StorageItem.parse(node)
+ filled = any(
+ getattr(item, item_field.name) for item_field in fields(StorageItem) if item_field.name != "expire"
+ ) or bool(item.expire)
+ return cls(enable=obj.boolean("enable", default=False), item=item if filled else None)
+
+ def to_dict(self) -> dict[str, object]:
+ payload = {"enable": self.enable}
+ if self.item is not None:
+ payload.update(self.item.to_dict())
+ return payload
+
+
+@dataclass
+class CrawlPersist:
+ subscribe: str = ""
+ nodes: str = ""
+
+ @classmethod
+ def parse(cls, node: Node, storage: StorageConfig) -> CrawlPersist:
+ if node.absent:
+ return cls()
+ obj = node.object()
+
+ def resolve(key: str) -> str:
+ name = obj.string(key, default="") or ""
+ if name and name not in storage.items:
+ obj.field(key).fail(f"storage item not found: {name}")
+ return name
+
+ return cls(subscribe=resolve("subscribe"), nodes=resolve("nodes"))
+
+ def to_dict(self) -> dict[str, object]:
+ return {"subscribe": self.subscribe, "nodes": self.nodes}
+
+
+@dataclass
+class CrawlConfig:
+ enable: bool = True
+ exclude: str = ""
+ max_fails: int = 5
+ include_nodes: bool = True
+ persist: CrawlPersist = field(default_factory=CrawlPersist)
+ task: TaskParams = field(default_factory=TaskParams)
+ google: GoogleConfig | None = None
+ yandex: YandexConfig | None = None
+ telegram: TelegramConfig | None = None
+ twitter: TwitterConfig | None = None
+ github: GithubConfig | None = None
+ repositories: list[RepoConfig] | None = None
+ pages: list[PageJob] | None = None
+ scripts: list[ScriptJob] | None = None
+
+ @classmethod
+ def parse(cls, node: Node, storage: StorageConfig) -> CrawlConfig | None:
+ if node.absent:
+ return None
+ obj = node.object()
+ repositories = None
+ if obj.has("repositories"):
+ repositories = [RepoConfig.parse(item) for item in obj.field("repositories").array()]
+ pages = None
+ if obj.has("pages"):
+ pages = [PageJob.parse(item) for item in obj.field("pages").array()]
+ scripts = None
+ if obj.has("scripts"):
+ scripts = [ScriptJob.parse(item) for item in obj.field("scripts").array()]
+ return cls(
+ enable=obj.boolean("enable", default=True),
+ exclude=obj.regex("exclude", default=""),
+ max_fails=obj.integer("max_fails", default=5, minimum=1) or 5,
+ include_nodes=obj.boolean("include_nodes", default=True),
+ persist=CrawlPersist.parse(obj.field("persist"), storage),
+ task=TaskParams.parse(obj.field("task")),
+ google=GoogleConfig.parse(obj.field("google")),
+ yandex=YandexConfig.parse(obj.field("yandex")),
+ telegram=TelegramConfig.parse(obj.field("telegram")),
+ twitter=TwitterConfig.parse(obj.field("twitter")),
+ github=GithubConfig.parse(obj.field("github")),
+ repositories=repositories,
+ pages=pages,
+ scripts=scripts,
+ )
+
+ def to_dict(self) -> dict[str, object]:
+ payload = {
+ "enable": self.enable,
+ "exclude": self.exclude,
+ "max_fails": self.max_fails,
+ "include_nodes": self.include_nodes,
+ "persist": self.persist.to_dict(),
+ "task": self.task.to_dict(),
+ }
+ if self.google is not None:
+ payload["google"] = self.google.to_dict()
+ if self.yandex is not None:
+ payload["yandex"] = self.yandex.to_dict()
+ if self.telegram is not None:
+ payload["telegram"] = self.telegram.to_dict()
+ if self.twitter is not None:
+ payload["twitter"] = self.twitter.to_dict()
+ if self.github is not None:
+ payload["github"] = self.github.to_dict()
+ if self.repositories is not None:
+ payload["repositories"] = [item.to_dict() for item in self.repositories]
+ if self.pages is not None:
+ payload["pages"] = [item.to_dict() for item in self.pages]
+ if self.scripts is not None:
+ payload["scripts"] = [item.to_dict() for item in self.scripts]
+ return payload
+
+
+@dataclass
+class ProcessConfig:
+ sites: list[SiteConfig] = field(default_factory=list)
+ groups: dict[str, GroupConfig] = field(default_factory=dict)
+ storage: StorageConfig = field(default_factory=StorageConfig)
+ update: UpdateConfig = field(default_factory=UpdateConfig)
+ delay: int = 5000
+ crawl: CrawlConfig | None = None
+
+ @classmethod
+ def parse(cls, data: dict[str, object]) -> ProcessConfig:
+ node = Node(data)
+ obj = node.object()
+ storage = StorageConfig.parse(obj.field("storage"))
+ groups = {}
+ if obj.has("groups"):
+ mapping = obj.field("groups").object()
+ for name in mapping.keys():
+ key = utils.trim(str(name))
+ if not key:
+ mapping.field(name).fail("contains an empty name")
+ groups[key] = GroupConfig.parse(key, mapping.field(name))
+ sites = []
+ if obj.has("sites"):
+ sites = [SiteConfig.parse(item) for item in obj.field("sites").array()]
+ return cls(
+ sites=sites,
+ groups=groups,
+ storage=storage,
+ update=UpdateConfig.parse(obj.field("update")),
+ delay=obj.integer("delay", default=5000, minimum=50) or 5000,
+ crawl=CrawlConfig.parse(obj.field("crawl"), storage),
+ )
+
+ def verify(self, pushtool: PushTo | None) -> None:
+ from push import PushTo as PushTool
+
+ if pushtool is not None and not isinstance(pushtool, PushTool):
+ raise ValueError("invalid pushtool")
+ items = pushtool.filter_push(self.storage.items) if pushtool else {}
+ for name, group in self.groups.items():
+ for category, storage_name in group.targets.items():
+ if storage_name not in items:
+ raise ValueError(f"missing storage configuration for group {name} to convert type to {category}")
+
+ def to_dict(self) -> dict[str, object]:
+ payload = {
+ "sites": [site.to_dict() for site in self.sites],
+ "groups": {name: group.to_dict() for name, group in self.groups.items()},
+ "storage": self.storage.to_dict(),
+ "update": self.update.to_dict(),
+ "delay": self.delay,
+ }
+ if self.crawl is not None:
+ payload["crawl"] = self.crawl.to_dict()
+ return payload
diff --git a/subscribe/config/parse.py b/subscribe/config/parse.py
new file mode 100644
index 0000000000..9a4931ecaf
--- /dev/null
+++ b/subscribe/config/parse.py
@@ -0,0 +1,189 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+import re
+from typing import Iterator
+
+import utils
+
+
+class ParseError(ValueError):
+ def __init__(self, message: str, location: str = "") -> None:
+ self.location = location
+ text = f"{location}: {message}" if location else message
+ super().__init__(text)
+
+
+def join_location(parent: str, key: str | int) -> str:
+ if isinstance(key, int):
+ return f"{parent}[{key}]" if parent else f"[{key}]"
+ return f"{parent}.{key}" if parent else key
+
+
+class Node:
+ def __init__(self, value: object, location: str = "") -> None:
+ self.value = value
+ self.location = location
+
+ @property
+ def absent(self) -> bool:
+ return self.value is None
+
+ def fail(self, message: str) -> None:
+ raise ParseError(message, self.location)
+
+ def object(self) -> ObjectNode:
+ if not isinstance(self.value, dict):
+ self.fail("must be an object")
+ return ObjectNode(self.value, self.location)
+
+ def array(self) -> ArrayNode:
+ if not isinstance(self.value, list):
+ self.fail("must be an array")
+ return ArrayNode(self.value, self.location)
+
+
+class ObjectNode:
+ def __init__(self, data: dict[str, object], location: str = "") -> None:
+ self._data = data
+ self.location = location
+
+ def has(self, key: str) -> bool:
+ return key in self._data
+
+ def keys(self) -> list[str]:
+ return list(self._data.keys())
+
+ def field(self, key: str) -> Node:
+ location = join_location(self.location, key)
+ if key not in self._data:
+ return Node(None, location)
+ return Node(self._data[key], location)
+
+ def rest(self, known: set[str]) -> dict[str, object]:
+ return {key: value for key, value in self._data.items() if key not in known}
+
+ def boolean(self, key: str, default: bool | None = None, required: bool = False) -> bool | None:
+ node = self.field(key)
+ if node.absent:
+ if required:
+ node.fail("is required")
+ return default
+ if not isinstance(node.value, bool):
+ node.fail("must be a boolean")
+ return node.value
+
+ def integer(
+ self,
+ key: str,
+ default: int | None = None,
+ required: bool = False,
+ minimum: int | None = None,
+ maximum: int | None = None,
+ ) -> int | None:
+ node = self.field(key)
+ if node.absent:
+ if required:
+ node.fail("is required")
+ return default
+ if isinstance(node.value, bool) or not isinstance(node.value, int):
+ node.fail("must be an integer")
+ if minimum is not None and node.value < minimum:
+ node.fail(f"must be >= {minimum}")
+ if maximum is not None and node.value > maximum:
+ node.fail(f"must be <= {maximum}")
+ return node.value
+
+ def number(
+ self,
+ key: str,
+ default: float | None = None,
+ required: bool = False,
+ minimum: float | None = None,
+ ) -> float | None:
+ node = self.field(key)
+ if node.absent:
+ if required:
+ node.fail("is required")
+ return default
+ if isinstance(node.value, bool) or not isinstance(node.value, (int, float)):
+ node.fail("must be a number")
+ value = float(node.value)
+ if minimum is not None and value < minimum:
+ node.fail(f"must be >= {minimum}")
+ return value
+
+ def string(
+ self,
+ key: str,
+ default: str | None = None,
+ required: bool = False,
+ allow_empty: bool = True,
+ ) -> str | None:
+ node = self.field(key)
+ if node.absent:
+ if required:
+ node.fail("is required")
+ return default
+ if not isinstance(node.value, str):
+ node.fail("must be a string")
+ value = utils.trim(node.value)
+ if not allow_empty and not value:
+ node.fail("must not be empty")
+ return value
+
+ def string_list(self, key: str, default: list[str] | None = None, unique: bool = True) -> list[str]:
+ node = self.field(key)
+ if node.absent:
+ return [] if default is None else list(default)
+ if not isinstance(node.value, list):
+ node.fail("must be an array")
+ items = []
+ for index, item in enumerate(node.value):
+ if not isinstance(item, str):
+ Node(item, join_location(node.location, index)).fail("must be a string")
+ text = utils.trim(item)
+ if text:
+ items.append(text)
+ return list(dict.fromkeys(items)) if unique else items
+
+ def string_or_list(self, key: str, default: str | list[str] = "") -> str | list[str]:
+ node = self.field(key)
+ if node.absent:
+ return default
+ if isinstance(node.value, str):
+ return utils.trim(node.value)
+ if isinstance(node.value, list):
+ items = []
+ for index, item in enumerate(node.value):
+ if not isinstance(item, str):
+ Node(item, join_location(node.location, index)).fail("must be a string")
+ text = utils.trim(item)
+ if text:
+ items.append(text)
+ return items
+ node.fail("must be a string or array")
+
+ def regex(self, key: str, default: str = "") -> str:
+ text = self.string(key, default=default) or ""
+ if not text:
+ return ""
+ try:
+ re.compile(text)
+ except re.error as exc:
+ self.field(key).fail(f"is not a valid regex: {exc}")
+ return text
+
+
+class ArrayNode:
+ def __init__(self, items: list[object], location: str = "") -> None:
+ self._items = items
+ self.location = location
+
+ def __iter__(self) -> Iterator[Node]:
+ for index, item in enumerate(self._items):
+ yield Node(item, join_location(self.location, index))
+
+ def __len__(self) -> int:
+ return len(self._items)
diff --git a/subscribe/crawl.py b/subscribe/crawl.py
deleted file mode 100644
index 0c8cdc2bed..0000000000
--- a/subscribe/crawl.py
+++ /dev/null
@@ -1,1937 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# @Author : wzdnzd
-# @Time : 2022-07-15
-
-import base64
-import gzip
-import importlib
-import itertools
-import json
-import multiprocessing
-import os
-import random
-import re
-import socket
-import ssl
-import string
-import sys
-import time
-import traceback
-import typing
-import urllib
-import urllib.error
-import urllib.parse
-import urllib.request
-from copy import deepcopy
-from dataclasses import dataclass
-from functools import cache
-from multiprocessing.managers import ListProxy
-from multiprocessing.synchronize import Semaphore
-
-import airport
-import push
-import utils
-import workflow
-import yaml
-from logger import logger
-from origin import Origin
-from urlvalidator import isurl
-from yaml.constructor import ConstructorError
-
-SEPARATOR = "-"
-
-
-@dataclass
-class ValidateResult(object):
- # single node collections
- proxies: set = None
-
- # valid subscription
- available: dict = None
-
- # valid or to be observed
- potential: dict = None
-
- # unknown
- unknown: str = None
-
-
-SINGLE_LINK_FLAG = "singlelink://"
-
-# environment key
-SINGLE_PROXIES_ENV_NAME = "ALLOW_SINGLE_LINK"
-
-
-@cache
-def allow_single_link() -> bool:
- """whether to allow crawling a single proxy link"""
- return os.environ.get(SINGLE_PROXIES_ENV_NAME, "").lower() == "true"
-
-
-def multi_thread_crawl(func: typing.Callable, params: list) -> dict:
- if not func or not params or type(params) != list:
- return {}
-
- # concurrent run
- results = utils.multi_thread_run(func=func, tasks=params)
-
- funcname = getattr(func, "__name__", repr(func)).replace("_", "-")
- tasks, uri = {}, f"{SINGLE_LINK_FLAG}{funcname}"
-
- for r in results:
- for k, v in r.items():
- k = uri if k == SINGLE_LINK_FLAG else k
- item = tasks.get(k, {})
- item["origin"] = v.pop("origin", item.get("origin", ""))
- pts = item.get("push_to", [])
- pts.extend(v.pop("push_to", []))
- item["push_to"] = list(set(pts))
-
- # merge proxies link
- if k.startswith(SINGLE_LINK_FLAG):
- newproxies = v.pop("proxies", [])
- if newproxies and type(newproxies) == list:
- oldproxies = item.get("proxies", [])
- oldproxies.extend(newproxies)
- item["proxies"] = list(set(oldproxies))
-
- item.update(v)
- tasks[k] = item
-
- return tasks
-
-
-def batch_crawl(conf: dict, num_threads: int = 50, display: bool = True) -> list[dict]:
- mode, connectable = crawlable()
-
- if not conf or not conf.get("enable", True):
- # only crawl if mode == 1
- if mode == 1:
- logger.warning(f"exit process because mode=1 and crawling task is disabled")
- sys.exit(0)
-
- else:
- return []
-
- datasets, peristedtasks = [], {}
- try:
- storage = conf.get("storage", {})
- persists = storage.get("items", {})
-
- subspushconf = persists.get("subs", {})
- linkspushconf = persists.get("proxies", {})
-
- pushtool = push.get_instance(config=push.PushConfig.from_dict(storage))
- should_persist = pushtool.validate(config=subspushconf)
- # skip tasks if mode == 1 and not set persistence
- if mode == 1 and not should_persist:
- logger.warning(
- f"[CrawlWarn] skip crawling tasks because the mode is set to crawl only but no valid persistence configuration is set"
- )
- sys.exit(0)
-
- # allow if persistence configuration is valid
- enable = conf.get("singlelink", False)
- allow = enable and pushtool.validate(config=linkspushconf)
-
- # save it to environment
- os.environ[SINGLE_PROXIES_ENV_NAME] = str(allow).lower()
-
- records, threshold = {}, conf.get("threshold", 1)
-
- if connectable:
- # Google
- google_spider = conf.get("google", {})
- if google_spider:
- push_to = google_spider.get("push_to", [])
- exclude = google_spider.get("exclude", "")
- notinurl = google_spider.get("notinurl", [])
- qdr = int(google_spider.get("qdr", 7))
- limits = int(google_spider.get("limits", 100))
- records.update(
- crawl_google(
- qdr=qdr,
- push_to=push_to,
- exclude=exclude,
- limits=limits,
- notinurl=notinurl,
- )
- )
-
- # yanex
- yandex_spider = conf.get("yandex", {})
- if yandex_spider:
- push_to = yandex_spider.get("push_to", [])
- exclude = yandex_spider.get("exclude", "")
- notinurl = yandex_spider.get("notinurl", [])
- within = int(yandex_spider.get("within", 2))
- pages = int(yandex_spider.get("pages", 5))
- records.update(
- crawl_yandex(
- within=within,
- push_to=push_to,
- exclude=exclude,
- pages=pages,
- notinurl=notinurl,
- )
- )
-
- # Telegram
- telegram_spider = conf.get("telegram", {})
- if telegram_spider and telegram_spider.get("users", {}):
- users = telegram_spider.get("users")
- pages = max(telegram_spider.get("pages", 1), 1)
- records.update(crawl_telegram(users=users, pages=pages))
-
- # Twitter
- twitter_spider = conf.get("twitter", {})
- if twitter_spider:
- records.update(crawl_twitter(tasks=twitter_spider))
-
- # skip crawl if mode == 2
- if mode != 2:
- # Github
- github_spider = conf.get("github", {})
- if github_spider and github_spider.get("push_to", []):
- push_to = github_spider.get("push_to")
- pages = github_spider.get("pages", 1)
- exclude = github_spider.get("exclude", "")
- spams = github_spider.get("spams", [])
- records.update(crawl_github(limits=pages, push_to=push_to, exclude=exclude, spams=spams))
-
- # Github Repository
- repositories = conf.get("repositories", {})
- if repositories:
- records.update(crawl_github_repo(repos=repositories))
-
- # Page
- pages = conf.get("pages", {})
- if pages:
- records.update(crawl_pages(pages=pages, origin=Origin.PAGE.name))
-
- # Scripts
- scripts = conf.get("scripts", {})
- if scripts:
- items = batch_call(scripts)
- if items:
- for item in items:
- if not item or type(item) != dict:
- continue
-
- if item.get("saved", False):
- datasets.append(item)
- continue
-
- task = deepcopy(item)
- subs = task.pop("sub", None)
- checked = task.pop("checked", True)
- if type(subs) not in [str, list]:
- continue
- if type(subs) == str:
- subs = [subs]
- for sub in subs:
- if utils.isblank(sub):
- continue
-
- if checked:
- remark(source=task, defeat=0, discovered=True)
- peristedtasks[sub] = task
- else:
- records.update({sub: task})
-
- # remain
- if should_persist:
- url = pushtool.raw_url(config=subspushconf)
- try:
- url, content = url or "", ""
- if not url.startswith(utils.FILEPATH_PROTOCAL):
- content = utils.http_get(url=url)
- else:
- file = url.sub[len(utils.FILEPATH_PROTOCAL) - 1 :]
- if os.path.exists(file) and os.path.isfile(file):
- with open(file, "r", encoding="UTF8") as f:
- content = f.read()
-
- if not utils.isblank(content):
- oldsubs = json.loads(content)
-
- # tasks.update(oldsubs)
- for k, v in oldsubs.items():
- merged = dict(list(v.items()) + list(records.get(k, {}).items()))
- records[k] = merged
- except:
- logger.error("[CrawlError] load old subscriptions from remote error")
- pass
-
- if not records:
- if peristedtasks and should_persist and mode != 2:
- content = json.dumps(peristedtasks)
- pushtool.push_to(content=content, config=subspushconf, group="crawl")
-
- logger.debug("[CrawlInfo] cannot found any subscribe from Google/Telegram/Github and Page with crawler")
- return datasets
-
- exclude = conf.get("exclude", "")
- taskconf = conf.get("config", {})
-
- # dedup by token
- tokens = {utils.parse_token(k): k for k in records.keys()}
- tasks = {k: records[k] for k in tokens.values()}
-
- time.sleep(random.randint(1, 3))
-
- proxiesconf, jobs = {}, []
- for key, value in tasks.items():
- for k, v in taskconf.items():
- if k not in value:
- value[k] = v
-
- jobs.append([key, value, mode, connectable, exclude, threshold])
-
- # generate single link's configuration
- if key.startswith(SINGLE_LINK_FLAG):
- proxiesconf.update(value)
-
- availables, unknowns, potentials, proxylinks = [], [], {}, set()
- results = utils.multi_thread_run(func=validate, tasks=jobs, num_threads=num_threads, show_progress=display)
- for result in results:
- if result.available:
- availables.append(result.available)
- if result.potential:
- potentials.update(result.potential)
- if result.proxies:
- proxylinks.update(result.proxies)
- if result.unknown:
- unknowns.append(result.unknown)
-
- datasets.extend(availables)
- peristedtasks.update(potentials)
-
- if allow_single_link():
- proxies = list(proxylinks)
- logger.info(f"[CrawlInfo] crawl finished, found {len(proxies)} proxy links")
-
- if len(proxies) > 0:
- try:
- content = base64.b64encode("\n".join(proxies).encode()).decode()
- success = pushtool.push_to(content=content, config=linkspushconf, group="proxies")
- if success:
- singlelink = pushtool.raw_url(config=linkspushconf)
- item = {
- "name": "singlelink",
- "sub": singlelink,
- "debut": True,
- }
- item.update(proxiesconf)
- datasets.append(item)
- except:
- logger.error("[CrawlError] base64 encode error for proxies links")
-
- if len(unknowns) > 0:
- unknowns = [utils.mask(url=x) for x in unknowns]
- logger.warning(
- f"[CrawlWarn] some links were found, but could not be confirmed to work, subscriptions: {unknowns}"
- )
-
- logger.info(f"[CrawlInfo] crawl finished, found {len(datasets)} subscriptions")
-
- if mode != 2 and should_persist and peristedtasks:
- survivors = {k: v for k, v in peristedtasks.items() if not v.pop("nocache", False)}
-
- total, rest = len(peristedtasks), len(survivors)
- logger.info(
- f"[CrawlInfo] to be saved subscriptions filtering completed, total: {total}, discard: {total - rest}, rest: {rest}"
- )
-
- if rest:
- content = json.dumps(survivors)
- pushtool.push_to(content=content, config=subspushconf, group="crawl")
- except:
- logger.error("[CrawlError] crawl from web error")
- traceback.print_exc()
-
- # only crawl if mode == 1
- if mode == 1:
- logger.warning(f"skip aggregate and exit process because mode=1 represents only crawling subscriptions")
- sys.exit(0)
-
- return datasets
-
-
-def crawlable() -> tuple[int, bool]:
- # 0: crawl and aggregate | 1: crawl only | 2: aggregate only
- mode = os.environ.get("WORKFLOW_MODE", "0")
- mode = 0 if not mode.isdigit() else min(max(int(mode), 0), 2)
-
- reachable = os.environ.get("REACHABLE", "true") in ["true", "1"]
- runnable = False if mode == 2 else reachable
-
- return mode, runnable
-
-
-def generate_telegram_task(channel: str, config: dict, pages: int, limits: int) -> list[list]:
- include = config.get("include", "")
- exclude = config.get("exclude", "").strip()
- pts = config.get("push_to", [])
- params = config.get("config", {})
- if pages <= 1:
- return [[f"https://t.me/s/{channel}", pts, include, exclude, limits, params]]
-
- count = get_telegram_pages(channel=channel)
- if count == 0:
- return []
-
- arrays = range(count, -1, -100)
- pages = min(pages, len(arrays))
- return [[f"https://t.me/s/{channel}?before={x}", pts, include, exclude, limits, params] for x in arrays[:pages]]
-
-
-def crawl_telegram_page(
- url: str,
- pts: list,
- include: str = "",
- exclude: str = "",
- limits: int = 25,
- config: dict = {},
-) -> dict:
- if not url or not pts:
- return {}
-
- limits = max(1, limits)
- content = utils.http_get(url=url)
- if not content:
- return {}
-
- return extract_subscribes(
- content=content,
- push_to=pts,
- include=include,
- limits=limits,
- source=Origin.TELEGRAM.name,
- exclude=exclude,
- config=config,
- reversed=True,
- )
-
-
-def crawl_telegram(users: dict, pages: int = 1, limits: int = 3) -> dict:
- if not users:
- return {}
-
- pages, limits = max(pages, 1), max(1, limits)
- starttime = time.time()
-
- params = [[k, v, pages, limits] for k, v in users.items()]
- results = utils.multi_thread_run(func=generate_telegram_task, tasks=params)
-
- tasks = list(itertools.chain.from_iterable(results))
- subscribes = multi_thread_crawl(func=crawl_telegram_page, params=tasks)
- cost = "{:.2f}s".format(time.time() - starttime)
- logger.info(f"[TelegramCrawl] finished crawl from Telegram, found {len(subscribes)} subscriptions, cost: {cost}")
- return subscribes
-
-
-def crawl_single_repo(username: str, repo: str, push_to: list = [], limits: int = 5, exclude: str = "") -> dict:
- if not username or not repo:
- logger.error(f"cannot crawl from github, username: {username}\trepo: {repo}")
- return {}
-
- # 列出repo所有文件名
- # url = f"https://api.github.com/repos/{username}/{repo}/contents/"
-
- # 列出repo commit记录
- limits = max(1, limits)
- url = f"https://api.github.com/repos/{username.strip()}/{repo.strip()}/commits?per_page={limits}"
-
- content = utils.http_get(url=url)
- if content == "":
- return {}
-
- try:
- commits = json.loads(content)
- collections = {}
- for item in commits:
- content = utils.http_get(url=item.get("url", ""))
- if not content:
- continue
- commit = json.loads(content)
- files = commit.get("files", [])
- for file in files:
- patch = file.get("patch", "")
- collections.update(
- extract_subscribes(
- content=patch,
- push_to=push_to,
- source=Origin.REPO.name,
- exclude=exclude,
- )
- )
- return collections
- except:
- logger.error(f"[GithubCrawl] crawl from github error, username: {username}\trepo: {repo}")
- return {}
-
-
-def crawl_github_repo(repos: dict) -> list[dict]:
- if not repos:
- return {}
- params = []
-
- starttime = time.time()
- for _, v in repos.items():
- username = v.get("username", "").strip()
- repo_name = v.get("repo_name", "").strip()
- push_to = v.get("push_to", [])
- limits = max(v.get("commits", 2), 1)
- exclude = v.get("exclude", "").strip()
-
- if not username or not repo_name or not push_to:
- continue
- params.append([username, repo_name, push_to, limits, exclude])
-
- subscribes = multi_thread_crawl(func=crawl_single_repo, params=params)
- cost = "{:.2f}s".format(time.time() - starttime)
- logger.info(f"[RepoCrawl] finished crawl from Repositorie, found {len(subscribes)} subscriptions, cost: {cost}")
- return subscribes
-
-
-def crawl_google(
- qdr: int = 10,
- push_to: list = [],
- exclude: str = "",
- limits: int = 100,
- interval: int = 0,
- notinurl: list = [],
-) -> dict:
- items, query = set(), urllib.parse.quote('"/api/v1/client/subscribe?token="')
- if notinurl and type(notinurl) == list:
- for text in notinurl:
- text = utils.trim(text).lower()
- if text and "+" not in text:
- items.add(urllib.parse.quote(f"-site:{text}"))
-
- reject = "+".join(list(items))
- if reject:
- # not search from some site, see: https://zhuanlan.zhihu.com/p/136076792
- query = f"{query}+{reject}"
-
- num, limits = 100, min(max(1, limits), 1000)
- url = f"https://www.google.com/search?q={query}&tbs=qdr:d{max(qdr, 1)}"
-
- params = {
- "hl": "zh-CN",
- "num": num,
- }
-
- starttime = time.time()
- collections = {}
- for start in range(0, limits, num):
- params["start"] = start
- content = re.sub(r"\\\\n", "", utils.http_get(url=url, params=params))
- content = re.sub(r"\?token\\\\u003d", "?token=", content, flags=re.I)
- regex = r'https?://(?:[a-zA-Z0-9_\u4e00-\u9fa5\-]+\.)+[a-zA-Z0-9_\u4e00-\u9fa5\-]+(?::\d+)?/?(?:/?)?api/v1/client/subscribe\?token(?:)?=[a-zA-Z0-9]{16,32}'
- subscribes = re.findall(regex, content)
- for s in subscribes:
- s = re.sub(r'||\s+', "", s).replace("http://", "https://", 1)
- try:
- if exclude and re.search(exclude, s):
- continue
- collections[s] = {"push_to": push_to, "origin": Origin.GOOGLE.name}
- except:
- continue
-
- # no more results
- if re.search(
- r'\s*找不到和您查询的“\s*\s*.*?/api/v1/client/subscribe\?token=.*?\s*\s*”相符的内容或信息。\s*
',
- content,
- flags=re.I,
- ):
- break
-
- time.sleep(interval)
-
- cost = "{:.2f}s".format(time.time() - starttime)
- logger.info(f"[GoogleCrawl] finished crawl from Google, found {len(collections)} subscriptions, cost: {cost}")
- return collections
-
-
-def crawl_yandex(
- within: int = 2,
- push_to: list = [],
- exclude: str = "",
- pages: int = 5,
- interval: int = 0,
- notinurl: list = [],
-) -> dict:
- reject, query = "", urllib.parse.quote("/api/v1/client/subscribe?token=")
- if notinurl and type(notinurl) == list:
- items = list(set([re.escape(utils.trim(x).lower()) for x in notinurl]))
- reject = "|".join(items)
-
- url = f'https://yandex.com/search/?text="{query}"&lr=10599&cee=1'
- if within > 0:
- url = f"{url}&within={within}"
-
- starttime = time.time()
-
- headers = {
- "User-Agent": utils.USER_AGENT,
- "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
- "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
- "Accept-Encoding": "gzip",
- }
-
- # get total pages
- content = utils.http_get(url=url, headers=headers)
- if content:
- regex = r'(\d+)'
- groups = re.findall(regex, content, flags=re.I)
- if groups:
- pages = min(pages, max([int(x) for x in groups]))
-
- collections, pages = {}, max(1, pages)
-
- for page in range(0, pages):
- content = utils.http_get(url=f"{url}&p={page}", headers=headers)
- if not content:
- logger.error(f"[YandexCrawl] cannot get content from page: {page}")
- continue
-
- groups = re.findall(r"([\s\S]*?) ", content)
- if not groups:
- logger.error(f"[YandexCrawl] cannot get any search result from page: {page}")
- continue
-
- for group in groups:
- try:
- if reject:
- regex = r''
- link = re.findall(regex, group, flags=re.I)[0]
- if re.search(reject, link):
- continue
- except:
- logger.error(f"[YandexCrawl] invalid regex pattern: {reject}")
- continue
-
- regex = r"https?://(?:[a-zA-Z0-9_\u4e00-\u9fa5\-]+\.)+[a-zA-Z0-9_\u4e00-\u9fa5\-]+(?::\d+)?/api/v1/client/subscribe\?token=[a-zA-Z0-9]{16,32}"
- links = re.findall(regex, group, flags=re.I)
- for link in links:
- try:
- link = re.sub(r"|", "", link).replace("http://", "https://")
- if exclude and re.search(exclude, link):
- continue
- collections[link] = {
- "push_to": push_to,
- "origin": Origin.YANDEX.name,
- }
- except:
- continue
-
- time.sleep(interval)
-
- cost = "{:.2f}s".format(time.time() - starttime)
- logger.info(f"[YandexCrawl] finished crawl from Yandex, found {len(collections)} subscriptions, cost: {cost}")
- return collections
-
-
-def crawl_github_page(page: int, cookie: str, push_to: list = [], exclude: str = "") -> dict:
- content = search_github_code(page=page, cookie=cookie)
- return extract_subscribes(content=content, push_to=push_to, exclude=exclude, source=Origin.GITHUB.name)
-
-
-def search_github(page: int, cookie: str, searchtype: str, sortedby: str) -> str:
- if page <= 0 or utils.isblank(cookie):
- return ""
-
- searchtype = "Code" if utils.isblank(searchtype) else searchtype
- sortedby = "indexed" if utils.isblank(sortedby) else sortedby
-
- # search with regex for code
- query = "%22%2Fapi%2Fv1%2Fclient%2Fsubscribe%3Ftoken%3D%22"
- if searchtype.lower() == "code":
- query = "%2F%5C%2Fapi%5C%2Fv1%5C%2Fclient%5C%2Fsubscribe%5C%3Ftoken%3D%5Ba-zA-Z0-9%5D%7B8%2C32%7D%2F"
-
- url = f"/search?o=desc&p={page}&q={query}&s={sortedby}&type={searchtype}"
- headers = {
- "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
- "Referer": "https://github.com",
- "User-Agent": utils.USER_AGENT,
- "Cookie": f"user_session={cookie}",
- }
- content = utils.http_get(url=url, headers=headers)
- if re.search(r"Sign in to GitHub
", content, flags=re.I):
- logger.error("[GithubCrawl] session has expired, please provide a valid session and try again")
- return ""
-
- return content
-
-
-def paging(start: int, end: int, peer_page: int) -> list[int]:
- if start > end or peer_page <= 0:
- return []
-
- pages = []
- for i in range(start, end + 1, peer_page):
- pages.append(i // peer_page + 1)
-
- return pages
-
-
-def search_github_issues(page: int, cookie: str) -> list[str]:
- content = search_github(page=page, cookie=cookie, searchtype="Issues", sortedby="created")
- if utils.isblank(content):
- return []
-
- try:
- regex = r'href="(/.*/.*/issues/\d+)">'
- groups = re.findall(regex, content, flags=re.I)
- links = list(set(groups))
- links = [f"https://github.com{x}" for x in links]
- return links
- except:
- return []
-
-
-def search_github_issues_byapi(peer_page: int = 50, page: int = 1) -> list[str]:
- peer_page, page = min(max(peer_page, 1), 100), max(1, page)
- url = f"https://api.github.com/search/issues?q=%22%2Fapi%2Fv1%2Fclient%2Fsubscribe%3Ftoken%3D%22&sort=created&order=desc&per_page={peer_page}&page={page}"
- content = utils.http_get(url=url)
- if utils.isblank(content):
- return []
- try:
- items, links = json.loads(content).get("items", []), set()
- for item in items:
- link = item.get("html_url", "")
- if utils.isblank(link):
- continue
- links.add(link)
-
- return list(links)
- except:
- logger.error("[GithubIssuesCrawl] occur error when search issues from github")
- traceback.print_exc()
- return []
-
-
-def search_github_code_byapi(token: str, peer_page: int = 50, page: int = 1, excludes: list = []) -> list[str]:
- """
- curl -Ls -o response.json -H "Authorization: Bearer " https://api.github.com/search/code?q=%22%2Fapi%2Fv1%2Fclient%2Fsubscribe%3Ftoken%3D%22&sort=indexed&order=desc&per_page=30&page=1
- """
- if utils.isblank(token):
- return []
-
- peer_page, page = min(max(peer_page, 1), 100), max(1, page)
- url = f"https://api.github.com/search/code?q=%22%2Fapi%2Fv1%2Fclient%2Fsubscribe%3Ftoken%3D%22&sort=indexed&order=desc&per_page={peer_page}&page={page}"
- headers = {
- "Accept": "application/vnd.github+json",
- "Authorization": f"Bearer {token}",
- # "X-GitHub-Api-Version": "2022-11-28"
- }
- content, links = utils.http_get(url=url, headers=headers), set()
- if utils.isblank(content):
- return []
- try:
- items = json.loads(content).get("items", [])
- excludes = list(set(excludes))
- for item in items:
- if not item or type(item) != dict:
- continue
-
- link = item.get("html_url", "")
- if utils.isblank(link):
- continue
-
- reponame = item.get("repository", {}).get("full_name", "") + "/"
- if not intercept(text=reponame, excludes=excludes):
- links.add(link)
-
- return list(links)
- except:
- return []
-
-
-def search_github_code(page: int, cookie: str, excludes: list = []) -> list[str]:
- content = search_github(page=page, cookie=cookie, searchtype="Code", sortedby="indexed")
- if utils.isblank(content):
- return []
-
- try:
- regex = r'href="(/[^\s"]+/blob/(?:[^"]+)?)#L\d+"'
- groups = re.findall(regex, content, flags=re.I)
- uris, links = list(set(groups)) if groups else [], set()
- excludes = list(set(excludes))
-
- for uri in uris:
- if not intercept(text=uri, excludes=excludes):
- links.add(f"https://github.com{uri}")
-
- return list(links)
- except:
- return []
-
-
-def intercept(text: str, excludes: list = []) -> bool:
- if not excludes:
- return False
-
- for regex in excludes:
- try:
- if re.search(regex, text, flags=re.I):
- return True
- except:
- logger.error(f"[GithubRepoIntercept] invalid regex pattern: {regex}")
- return False
-
-
-def crawl_github(limits: int = 3, push_to: list = [], spams: list = [], exclude: str = "") -> dict:
- # user_session=${any}
- cookie = os.environ.get("GH_COOKIE", "").strip()
- token = os.environ.get("GH_TOKEN", "").strip()
- if utils.isblank(cookie) and utils.isblank(token):
- logger.error("[GithubCrawl] cannot start crawl from github because cookie and token is missing")
- return {}
-
- links, starttime = [], time.time()
- method = "search on the page" if utils.isblank(token) else "rest api"
-
- if utils.isblank(token):
- # 鉴于github搜索code不稳定，爬取两次
- pages = [x for x in range(1, limits + 1)] * 2
- params = [[x, cookie, spams] for x in pages]
-
- results = utils.multi_thread_run(func=search_github_code, tasks=params)
- items = list(set(itertools.chain.from_iterable(results)))
-
- links.extend(items)
- links.extend(search_github_issues(page=1, cookie=cookie))
- else:
- peer_page, count = 50, 10
- pages = paging(start=1, end=limits * count, peer_page=peer_page)
- params = [[token, peer_page, x, spams] for x in pages] * 2
-
- results = utils.multi_thread_run(func=search_github_code_byapi, tasks=params)
- items = list(set(itertools.chain.from_iterable(results)))
-
- links.extend(items)
- links.extend(search_github_issues_byapi(peer_page=5, page=1))
-
- if links:
- page_tasks = {}
- exclude = "" if not exclude else exclude.strip()
-
- for link in links:
- page_tasks[link] = {"push_to": push_to, "exclude": exclude}
- subscribes = crawl_pages(pages=page_tasks, silent=True, origin=Origin.GITHUB.name)
- else:
- subscribes = {}
-
- cost = "{:.2f}s".format(time.time() - starttime)
- logger.info(
- f"[GithubCrawl] finished crawl from Github through {method}, found {len(subscribes)} subscriptions need check, cost: {cost}"
- )
-
- return subscribes
-
-
-def crawl_single_page(
- url: str,
- push_to: list = [],
- include: str = "",
- exclude: str = "",
- config: dict = {},
- headers: dict = None,
- origin: str = Origin.PAGE.name,
- nocache: bool = False,
-) -> dict:
- if not url or not push_to:
- logger.error(f"[PageCrawl] cannot crawl from page: {url}")
- return {}
-
- content = utils.http_get(url=url, headers=headers)
- if content == "":
- return {}
-
- return extract_subscribes(
- content=content,
- push_to=push_to,
- include=include,
- exclude=exclude,
- config=config,
- source=origin,
- nocache=nocache,
- )
-
-
-def crawl_pages(
- pages: dict,
- silent: bool = False,
- headers: dict = None,
- origin: str = Origin.PAGE.name,
-) -> dict:
- if not pages:
- return {}
-
- params, starttime = [], time.time()
- for k, v in pages.items():
- if not isurl(url=k):
- continue
-
- push_to = v.get("push_to", [])
- include = v.get("include", "").strip()
- exclude = v.get("exclude", "").strip()
- config = v.get("config", {})
- nocache = v.get("nocache", False)
-
- final_headers = deepcopy(headers) if headers and isinstance(headers, dict) else utils.DEFAULT_HTTP_HEADERS
- specific_headers = v.get("headers", {})
- if specific_headers and isinstance(specific_headers, dict):
- final_headers.update(specific_headers)
-
- params.append([k, push_to, include, exclude, config, final_headers, origin, nocache])
-
- subscribes = multi_thread_crawl(func=crawl_single_page, params=params)
- if not silent:
- cost = "{:.2f}s".format(time.time() - starttime)
- logger.info(f"[PageCrawl] finished crawl from Page, found {len(subscribes)} subscriptions, cost: {cost}")
-
- return subscribes
-
-
-def extract_twitter_cookies(retry: int = 2) -> str:
- if retry <= 0:
- return ""
-
- headers = None
- try:
- request = urllib.request.Request(url="https://twitter.com/", headers=utils.DEFAULT_HTTP_HEADERS)
- response = urllib.request.urlopen(request, timeout=10, context=utils.CTX)
- headers = response.headers
- except urllib.error.HTTPError as e:
- if e.code != 302:
- return extract_twitter_cookies(retry=retry - 1)
-
- headers = e.headers
- except (urllib.error.URLError, TimeoutError):
- return extract_twitter_cookies(retry=retry - 1)
-
- if not headers or "set-cookie" not in headers:
- return ""
-
- regex = "(guest_id|guest_id_ads|guest_id_marketing|personalization_id)=(.+?);"
- content = ";".join(headers.get_all("set-cookie", ""))
- groups = re.findall(regex, content, flags=re.I)
- cookies = ";".join(["=".join(x) for x in groups]).strip()
-
- return cookies
-
-
-def get_guest_token() -> str:
- cookies = extract_twitter_cookies(retry=3)
- if not cookies:
- logger.error(f"[TwitterCrawl] cannot extract Twitter cookies")
- return ""
-
- headers = {
- "User-Agent": utils.USER_AGENT,
- "Cookie": cookies,
- "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
- }
- content = utils.http_get(url="https://twitter.com/", headers=headers)
- if not content:
- return ""
-
- matcher = re.findall("gt=([0-9]{19})", content, flags=re.I)
- return matcher[0] if matcher else ""
-
-
-def username_to_id(username: str, headers: dict) -> str:
- if utils.isblank(username):
- return ""
-
- if not headers or "X-Guest-Token" not in headers:
- guest_token = get_guest_token()
- if not guest_token:
- return ""
-
- headers = {
- "User-Agent": utils.USER_AGENT,
- "Authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA",
- "X-Guest-Token": guest_token,
- "Content-Type": "application/json",
- }
-
- variables = {
- "screen_name": username.lower().strip(),
- "withSafetyModeUserFields": True,
- }
- features = {
- "blue_business_profile_image_shape_enabled": True,
- "responsive_web_graphql_exclude_directive_enabled": True,
- "verified_phone_label_enabled": False,
- "responsive_web_graphql_skip_user_profile_image_extensions_enabled": False,
- "responsive_web_graphql_timeline_navigation_enabled": True,
- }
-
- payload = urllib.parse.urlencode({"variables": json.dumps(variables), "features": json.dumps(features)})
- url = f"https://twitter.com/i/api/graphql/sLVLhk0bGj3MVFEKTdax1w/UserByScreenName?{payload}"
- try:
- content = utils.http_get(url=url, headers=headers)
- if not content:
- return ""
-
- data = json.loads(content).get("data", {}).get("user", {}).get("result", "")
- return data.get("rest_id", "")
- except:
- logger.error(f"[TwitterCrawl] cannot query uid by username=[{username}]")
- return ""
-
-
-def crawl_twitter(tasks: dict) -> dict:
- if not tasks:
- return {}
-
- # extract X-Guest-Token
- guest_token, starttime = get_guest_token(), time.time()
- if not guest_token:
- logger.error(f"[TwitterCrawl] cannot extract X-Guest-Token from twitter")
- return {}
-
- headers = {
- "User-Agent": utils.USER_AGENT,
- "Authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA",
- "X-Guest-Token": guest_token,
- "Content-Type": "application/json",
- }
-
- features = {
- "blue_business_profile_image_shape_enabled": True,
- "responsive_web_graphql_exclude_directive_enabled": True,
- "verified_phone_label_enabled": False,
- "responsive_web_graphql_timeline_navigation_enabled": True,
- "responsive_web_graphql_skip_user_profile_image_extensions_enabled": False,
- "tweetypie_unmention_optimization_enabled": True,
- "vibe_api_enabled": True,
- "responsive_web_edit_tweet_api_enabled": True,
- "graphql_is_translatable_rweb_tweet_is_translatable_enabled": True,
- "view_counts_everywhere_api_enabled": True,
- "longform_notetweets_consumption_enabled": True,
- "tweet_awards_web_tipping_enabled": False,
- "freedom_of_speech_not_reach_fetch_enabled": True,
- "standardized_nudges_misinfo": True,
- "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": False,
- "interactive_text_enabled": True,
- "responsive_web_text_conversations_enabled": False,
- "longform_notetweets_rich_text_read_enabled": True,
- "responsive_web_enhance_cards_enabled": False,
- }
-
- candidates, pages = {}, {}
- for k, v in tasks.items():
- if utils.isblank(k) or not v or type(v) != dict:
- continue
- candidates[k] = v
-
- if not candidates:
- return {}
-
- # username to uid
- params = [[k, headers] for k in candidates.keys()]
- uids = utils.multi_thread_run(func=username_to_id, tasks=params)
-
- for i in range(len(uids)):
- uid = uids[i]
- if not uid:
- continue
-
- config = candidates.get(params[i][0])
- count = config.pop("num", 10)
- variables = {
- "userId": uid,
- "count": min(max(count, 1), 100),
- "includePromotedContent": False,
- "withClientEventToken": False,
- "withBirdwatchNotes": False,
- "withVoice": True,
- "withV2Timeline": True,
- }
-
- payload = urllib.parse.urlencode({"variables": json.dumps(variables), "features": json.dumps(features)})
- url = f"https://twitter.com/i/api/graphql/P7qs2Sf7vu1LDKbzDW9FSA/UserMedia?{payload}"
- pages[url] = config
-
- subscribes = crawl_pages(pages=pages, silent=True, headers=headers, origin=Origin.TWITTER.name)
- cost = "{:.2f}s".format(time.time() - starttime)
- logger.info(f"[TwitterCrawl] finished crawl from Twitter, found {len(subscribes)} subscriptions, cost: {cost}")
-
- return subscribes
-
-
-def extract_subscribes(
- content: str,
- push_to: list = [],
- include: str = "",
- exclude: str = "",
- limits: int = sys.maxsize,
- source: str = Origin.OWNED.name,
- config: dict = {},
- reversed: bool = False,
- nocache: bool = False,
-) -> dict:
- if not content:
- return {}
- try:
- limits, collections, proxies = max(1, limits), {}, []
- sub_regex = r"https?://(?:[a-zA-Z0-9\u4e00-\u9fa5\-]+\.)+[a-zA-Z0-9\u4e00-\u9fa5\-]+(?::\d+)?(?:(?:(?:/index.php)?/api/v1/client/subscribe\?token=[a-zA-Z0-9]{16,32})|(?:/link/[a-zA-Z0-9]+\?(?:sub|mu|clash)=\d)|(?:/(?:s|sub)/[a-zA-Z0-9]{32}))|https://jmssub\.net/members/getsub\.php\?service=\d+&id=[a-zA-Z0-9\-]{36}(?:\S+)?"
- extra_regex = r"https?://(?:[a-zA-Z0-9\u4e00-\u9fa5\-]+\.)+[a-zA-Z0-9\u4e00-\u9fa5\-]+/sub\?(?:\S+)?target=\S+"
- protocal_regex = (
- r"(?:vmess|trojan|ss|ssr|snell|hysteria2|vless|hysteria|tuic|anytls)://[a-zA-Z0-9:.?+=@%&#_\-/]{10,}"
- )
-
- regex = f"{sub_regex}|{extra_regex}"
-
- if include:
- try:
- if not include.startswith("|"):
- pattern = f"{regex}|{include}"
- else:
- pattern = f"{regex}{include}"
-
- subscribes = re.findall(pattern, content, flags=re.I)
- except:
- logger.error(f"[ExtractError] maybe pattern 'include' exists some problems, include: {include}")
- subscribes = re.findall(regex, content)
- else:
- subscribes = re.findall(regex, content, flags=re.I)
-
- try:
- parts = re.findall(
- r"(?m)^#(?:\s+)?(?:!MANAGED-CONFIG|订阅链接)[^\n]*?(https?://[^\s\"'<>]+)", content, flags=re.I
- )
- if parts:
- subscribes.extend([utils.trim(p) for p in parts])
- except:
- pass
-
- # 去重会打乱原本按日期排序的特性一致无法优先选择离当前时间较近的元素
- # subscribes = list(set(subscribes))
-
- if reversed:
- subscribes.reverse()
-
- for sub in subscribes:
- items = [sub]
- # subconverter url
- if "url=" in sub:
- qs, items = urllib.parse.urlparse(sub.replace("&", "&")).query, []
- urls = urllib.parse.parse_qs(qs).get("url", [])
- if not urls:
- continue
-
- for url in urls:
- if not utils.isurl(url):
- if allow_single_link():
- proxies.extend([x for x in url.split("|") if re.match(protocal_regex, x, flags=re.I)])
- continue
-
- items.extend([x for x in url.split("|") if not re.match(extra_regex, x, flags=re.I)])
-
- for s in items:
- s = re.sub(r"\\/|\/", "/", s, flags=re.I)
- try:
- if include and not re.match(
- r"https?://(?:[a-zA-Z0-9\u4e00-\u9fa5\-]+\.)+[a-zA-Z0-9\u4e00-\u9fa5\-]+.*",
- s,
- flags=re.I,
- ):
- continue
-
- if exclude and re.search(exclude, s):
- continue
- except:
- logger.error(
- f"[ExtractError] maybe pattern 'include' or 'exclude' exists some problems, include: {include}\texclude: {exclude}"
- )
-
- # 强制使用https协议
- # s = s.replace("http://", "https://", 1).strip()
- params = {"push_to": push_to, "origin": source, "nocache": nocache}
- if config:
- params.update(config)
- collections[s] = params
-
- if len(collections) >= limits:
- break
-
- if allow_single_link():
- try:
- groups = re.findall(protocal_regex, content, flags=re.I)
- if groups:
- proxies.extend([x.lower().strip() for x in groups if x])
- params = {
- "push_to": push_to,
- "origin": source,
- "proxies": list(set(proxies)),
- }
- if config:
- params.update(config)
- collections[SINGLE_LINK_FLAG] = params
- except:
- logger.error(f"[ExtractError] failed to extract single proxy")
-
- return collections
- except:
- logger.error("[ExtractError] extract subscribe error")
- return {}
-
-
-def validate(
- url: str,
- params: dict,
- mode: int,
- connectable: bool,
- exclude: str = "",
- threshold: int = 1,
-) -> ValidateResult:
- if (
- not params
- or not params.get("push_to", None)
- or not params.get("origin", "")
- or (exclude and re.search(exclude, url))
- ):
- return ValidateResult()
-
- result = ValidateResult()
- if url.startswith(SINGLE_LINK_FLAG):
- proxies = params.get("proxies", [])
- if proxies and type(proxies) == list:
- result.proxies = set(proxies)
-
- return result
-
- threshold = max(threshold, 1)
- defeat = params.get("defeat", 0) + 1
- discovered = params.get("discovered", False)
-
- reachable, expired = check_status(url=url, retry=2, remain=5, spare_time=12, tolerance=72, connectable=connectable)
- if reachable:
- item = {"name": naming_task(url), "sub": url, "debut": True}
- item.update(params)
- result.available = item
-
- # reset defeat to 0
- defeat = 0
- discovered = True
-
- if not params.pop("saved", False):
- if reachable or (discovered and defeat <= threshold and not expired):
- # don't storage temporary link shared by someone
- pardon = params.pop("pardon", False)
- if not pardon and not workflow.standard_sub(url=url) and mode != 1:
- return result
-
- remark(source=params, defeat=defeat, discovered=True)
- result.potential = {url: params}
- elif not connectable and not expired:
- result.unknown = url
-
- return result
-
-
-def remark(source: dict, defeat: int = 0, discovered: bool = True) -> None:
- if not source or type(source) != dict or type(defeat) != int or defeat < 0 or type(discovered) != bool:
- return
-
- source["defeat"] = defeat
- source["discovered"] = discovered
- if utils.isblank(source.get("origin", "")):
- source["origin"] = Origin.TEMPORARY.name
-
-
-def check_status(
- url: str,
- retry: int = 2,
- remain: float = 0,
- spare_time: float = 0,
- tolerance: float = 0,
- connectable: bool = True,
-) -> tuple[bool, bool]:
- """
- url: subscription link
- retry: number of retries
- remain: minimum remaining traffic flow
- spare_time: minimum remaining time
- tolerance: waiting time after expiration
-
- Returns:
- tuple[bool, bool]: (available, expired)
- - First bool: whether the subscription is available
- - Second bool: whether the subscription has expired
- """
- if not url or retry <= 0:
- return False, connectable
-
- try:
- headers = {"User-Agent": "clash.meta"}
- request = urllib.request.Request(url=url, headers=headers)
- response = urllib.request.urlopen(request, timeout=10, context=utils.CTX)
- if response.getcode() != 200:
- return False, connectable
-
- # in order to avoid the request to the speed test site causing constant data downloads, limit the maximum read to 15MB
- content = str(response.read(15 * 1024 * 1024), encoding="utf8")
-
- # response text is too short, ignore
- if len(content) < 32:
- return False, False
-
- # 订阅流量信息
- subscription = response.getheader("subscription-userinfo")
-
- # 情况1: base64 编码格式（v2ray）
- if utils.isb64encode(content):
- return is_expired(header=subscription, remain=remain, spare_time=spare_time, tolerance=tolerance)
-
- # 情况2: YAML 格式（clash）
- proxies = _parse_yaml_proxies(content)
-
- if proxies is None:
- # 情况3: 纯协议链接格式
- if all(airport.AirPort.check_protocol(x) for x in content.split("\n") if x):
- return True, False
- # TODO: 如果配置文件为 singbox、quanx、loon、surge等，需要解析出代理节点信息，并判断是否过期
- return False, True
-
- # 情况4: 节点列表为空
- if len(proxies) == 0:
- return False, True
-
- # 根据订阅信息判断是否有效
- return is_expired(header=subscription, remain=remain, spare_time=spare_time, tolerance=tolerance)
- except urllib.error.HTTPError as e:
- try:
- message = str(e.read(), encoding="utf8")
- except:
- message = ""
-
- expired = e.code == 404 or "token is error" in message
- if not expired and e.code in [403, 503]:
- return check_status(
- url=url,
- retry=retry - 1,
- remain=remain,
- spare_time=spare_time,
- tolerance=tolerance,
- connectable=connectable,
- )
-
- return False, expired
- except Exception as e:
- return check_status(
- url=url,
- retry=retry - 1,
- remain=remain,
- spare_time=spare_time,
- tolerance=tolerance,
- connectable=connectable,
- )
-
-
-def _parse_yaml_proxies(content: str) -> list | None:
-
- try:
- return yaml.load(content, Loader=yaml.SafeLoader).get("proxies", [])
- except ConstructorError:
- yaml.add_multi_constructor("str", lambda loader, suffix, node: str(node.value), Loader=yaml.SafeLoader)
- return yaml.load(content, Loader=yaml.FullLoader).get("proxies", [])
- except Exception:
- # 解析失败，返回 None 表示不是 YAML 格式
- return None
-
-
-def is_expired(header: str, remain: float = 0, spare_time: float = 0, tolerance: float = 0) -> tuple[bool, bool]:
- if utils.isblank(header):
- return True, False
-
- remain, spare_time, tolerance = (
- max(0, remain),
- max(spare_time, 0),
- max(tolerance, 0),
- )
- try:
- infos = header.split(";")
- upload, download, total, expire = 0, 0, 0, None
- for info in infos:
- words = info.split("=", maxsplit=1)
- if len(words) <= 1:
- continue
-
- if "upload" == words[0].strip():
- upload = eval(words[1])
- elif "download" == words[0].strip():
- download = eval(words[1])
- elif "total" == words[0].strip():
- total = eval(words[1])
- elif "expire" == words[0].strip():
- expire = None if utils.isblank(words[1]) else eval(words[1])
-
- # 剩余流量大于 ${remain} GB 并且未过期则返回 True，否则返回 False
- flag = total - (upload + download) > remain * pow(1024, 3) and (
- expire is None or expire - time.time() > spare_time * 3600
- )
- expired = False if flag else (expire is not None and (expire + tolerance * 3600) <= time.time())
- return flag, expired
- except:
- return True, False
-
-
-def is_available(url: str, retry: int = 2, remain: float = 0, spare_time: float = 0) -> bool:
- available, _ = check_status(url=url, retry=retry, remain=remain, spare_time=spare_time)
- return available
-
-
-def naming_task(url) -> str:
- prefix = utils.extract_domain(url=url).replace(".", "") + SEPARATOR
- return prefix + "".join(random.sample(string.digits + string.ascii_lowercase, random.randint(3, 5)))
-
-
-def get_telegram_pages(channel: str) -> int:
- if not channel or channel.strip() == "":
- return 0
-
- url = f"https://t.me/s/{channel}"
- content = utils.http_get(url=url)
- before = 0
- try:
- regex = rf''
- groups = re.findall(regex, content)
- before = int(groups[0]) if groups else before
- except:
- logger.error(f"[CrawlError] cannot count page num, chanel: {channel}")
-
- return before
-
-
-def extract_airport_site(url: str) -> list[str]:
- if not url:
- return []
-
- logger.info(f"[AirPortCrawl] start collect airport, url: {url}")
-
- content = utils.http_get(url=url)
- if not content:
- logger.error(f"[CrawlError] cannot any content from url: {url}")
- return []
- try:
- regex = r'href="(https?://(?:[a-zA-Z0-9\u4e00-\u9fa5\-]+\.)+[a-zA-Z0-9\u4e00-\u9fa5\-]+/?)"\s+target="_blank"\s+rel="noopener">'
- groups = re.findall(regex, content)
- return list(set(groups)) if groups else []
- except:
- return []
-
-
-def collect_airport(
- channel: str,
- page_num: int,
- num_thread: int = 64,
- rigid: bool = True,
- display: bool = True,
- filepath: str = "",
- delimiter: str = "",
- chuck: bool = False,
-) -> dict:
- def crawl_channel(channel: str, page_num: int, fun: typing.Callable) -> list[str]:
- """crawl from telegram channel"""
- if not channel or not fun or not isinstance(fun, typing.Callable):
- return []
-
- page_num = max(page_num, 1)
- url = f"https://t.me/s/{channel}"
- if page_num == 1:
- return list(fun(url))
- else:
- count = get_telegram_pages(channel=channel)
- if count == 0:
- return []
-
- pages = range(count, -1, -20)
- page_num = min(page_num, len(pages))
- logger.info(f"[TelegramCrawl] starting crawl from telegram, channel: {channel}, pages: {page_num}")
-
- urls = [f"{url}?before={x}" for x in pages[:page_num]]
- results = utils.multi_thread_run(func=fun, tasks=urls)
-
- return list(itertools.chain.from_iterable(results))
-
- def crawl_ccbh() -> dict:
- url = "https://ccbaohe.com/jcjd.html"
- content = utils.http_get(url=url)
- try:
- groups = re.split(r"【[^【]*】", content, flags=re.M)
- if not groups:
- logger.warning(f"[AirPortCollector] cannot found any domains from [{url}]")
- return {}
-
- candidates, result = {}, {}
- for group in groups:
- if not group:
- continue
-
- texts = group.split("
")
- for text in texts:
- address_regex = r'注册地址：)?([^\s\r\n<）]+)"
- words = re.findall(coupon_regex, text, flags=re.M)
- coupon = words[0] if words else ""
-
- candidates[address] = coupon
-
- urls = list(candidates.keys())
- latest = utils.multi_thread_run(func=get_redirect_url, tasks=urls, num_threads=num_thread)
-
- for i, x in enumerate(urls):
- domain = utils.extract_domain(url=latest[i], include_protocal=True)
- if not domain:
- continue
-
- coupon = candidates.get(x, "")
- result[domain] = coupon
- except:
- logger.error(f"[AirPortCollector] occur error when crawl from [{url}], message: \n{traceback.format_exc()}")
-
- logger.info(f"[AirPortCollector] finished crawl from [{url}], found {len(result)} domains")
- return result
-
- def crawl_maomeng() -> dict:
- return run_crawl(
- url="https://maomeng.xyz/2021/06/11/ji-chang-tui-jian-chang-qi-geng-xin",
- separator=r'官网：.* ",
- coupon_regex=r"
(?:优惠|白嫖)码：([^<]+)
",
- )
-
- def crawl_askahh() -> dict:
- return run_crawl(
- url="https://www.askahh.com/archives/101",
- separator=r"<h2>[^\r\n]+</h2>",
- address_regex=r"<a class="no-external-link" href="(https?://[^\s]+)" target="_blank">",
- coupon_regex=r"使用优惠码(?:<strong>)?([A-Za-z0-9\u4e00-\u9fa5_\-%*:.@&#]+)(?:</strong>(?:[\r\n\s]+)?)?免费购买",
- )
-
- def crawl_ygpy() -> dict:
- def get_links(url: str, prefix: str, regex: str = r'href="(/vpn/\d{4}/\d{2}.html)"') -> list[str]:
- content = utils.http_get(url=url)
-
- groups = re.findall(regex, content, flags=re.I)
- if not groups:
- logger.warning(f"[AirPortCollector] cannot fetch article from url: {url}")
- return []
-
- prefix = utils.trim(prefix)
- return list(set([urllib.parse.urljoin(prefix, x) for x in groups if x]))
-
- base = "https://ygpy.net"
- links = get_links(url=f"{base}/vpn/free", prefix=base, regex=r"/assets/chunks/free.data\.[^\r\n\s]+\.js")
- if not links:
- logger.warning(f"[AirPortCollector] cannot get article from url: {base}")
- return {}
-
- articles = get_links(
- url=links[0], prefix="", regex=r'"(https://ygpy.net/vpn/free/\d{4}/\d{2}/[^\r\n\s">]+\.html)"'
- )
- if not articles:
- logger.warning(f"[AirPortCollector] cannot get articles from url: {links[0]}")
- return {}
-
- separator = r''
- address_regex = r'前往注册'
- coupon_regex = r"使用优惠[码券](?:\s+)?(?:)?([^\r\n\s]+)(?:(?:[\r\n\s]+)?)?0(?:\s+)?元购买"
-
- tasks = [[x, separator, address_regex, coupon_regex] for x in sorted(articles)]
- items = utils.multi_thread_run(func=run_crawl, tasks=tasks)
-
- result = dict()
- for item in items:
- if item and isinstance(item, dict):
- result.update(item)
-
- # Extract javascript link from peer page and then parse airport urls and coupons
- javascripts = utils.multi_thread_run(
- func=get_links,
- tasks=[
- [x, base, r'href="(/assets/vpn_free_\d+_\d+_[^\r\n\s\"]+.md.[A-Za-z0-9_\-]+.lean.js)"']
- for x in articles
- ],
- )
-
- airports = utils.multi_thread_run(
- func=run_crawl,
- tasks=[
- [
- x[0],
- r'"详细信息"|测试报告"|"官方(群组|频道)|"更新于|"联系方式',
- r'"hyperlink":"(https?://[^\s]+/(?:#/register|auth)\?(?:code|invite)=[^\s]+)"',
- r'{code:"([^\r\n\s]+)"}',
- ]
- for x in javascripts
- if x and isinstance(x, list)
- ],
- )
-
- for item in airports:
- if not item or not isinstance(item, dict):
- continue
-
- result.update(item)
-
- return result
-
- def crawl_jctj(convert: bool = False) -> dict:
- url = "https://raw.githubusercontent.com/hwanz/SSR-V2ray-Trojan-vpn/main/README.md"
- content = utils.http_get(url=url)
- groups = re.findall(r"\[.*\]\((https?:\/\/[^\s\r\n]+)\)[^\r\n]+\d+G.*", content, flags=re.I)
- if not groups:
- return {}
-
- try:
- tasks = [utils.trim(x).lower() for x in groups if x]
- if convert:
- links = utils.multi_thread_run(func=get_redirect_url, tasks=tasks, num_threads=num_thread)
- else:
- links = tasks
-
- result = {utils.extract_domain(url=x, include_protocal=True): "" for x in links if x}
- logger.info(f"[AirPortCollector] finished crawl from [{url}], found {len(result)} domains")
-
- return result
- except:
- logger.error(f"[AirPortCollector] occur error when crawl from [{url}], message: \n{traceback.format_exc()}")
- return {}
-
- def get_redirect_url(url: str, retry: int = 3) -> str:
- if not url or retry <= 0:
- return ""
-
- headers = {
- "User-Agent": utils.USER_AGENT,
- "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
- "Accept-Encoding": "gzip, deflate, br, zstd",
- "Accept-Language": "zh-CN,zh;q=0.9",
- }
-
- try:
- request = urllib.request.Request(url=url, headers=headers, method="GET")
- response = urllib.request.urlopen(request, timeout=10, context=utils.CTX)
-
- return response.geturl()
- except:
- time.sleep(random.randint(1, 3))
- return get_redirect_url(url=url, retry=retry - 1)
-
- def run_crawl(url: str, separator: str, address_regex: str, coupon_regex: str) -> dict:
- url = utils.trim(url)
- content = utils.http_get(url=url)
- if not content:
- return {}
-
- result = dict()
- try:
- groups = re.split(utils.trim(separator), content, flags=re.M)
- if not groups:
- logger.warning(f"[AirPortCollector] cannot found any domains from [{url}]")
- return {}
-
- for group in groups:
- if not group or not isinstance(group, str):
- continue
-
- words = re.findall(utils.trim(address_regex), group, flags=re.M)
- address = words[0] if words else ""
- if not address:
- continue
-
- words = re.findall(utils.trim(coupon_regex), group, flags=re.M)
- coupon = words[0] if words else ""
-
- domain = utils.extract_domain(url=address, include_protocal=True)
- result[domain] = coupon
- except:
- logger.error(f"[AirPortCollector] occur error when crawl from [{url}], message: \n{traceback.format_exc()}")
-
- logger.info(f"[AirPortCollector] finished crawl from [{url}], found {len(result)} domains")
- return result
-
- def extract_backend_url(domain: str, retry: int = 2) -> str:
- # TODO: exploring a more generalized approach to backend addresses
- def request_once(suffix: str) -> tuple[bool, str]:
- count, suffix = 0, utils.trim(suffix)
- url = urllib.parse.urljoin(domain, suffix)
-
- while count < retry:
- count += 1
-
- try:
- request = urllib.request.Request(url=url, headers=utils.DEFAULT_HTTP_HEADERS, method="GET")
- response = urllib.request.urlopen(request, timeout=6, context=utils.CTX)
-
- word = "" if not suffix else (suffix if suffix.startswith("/") else "/" + suffix)
- if word and not utils.trim(response.geturl()).endswith(word):
- return True, ""
-
- content = response.read()
- try:
- content = str(content, encoding="utf8")
- except:
- content = gzip.decompress(content).decode("utf8")
-
- return False, content
- except urllib.error.HTTPError as e:
- if e.code == 404:
- return False, ""
- except urllib.error.URLError as e:
- if isinstance(e.reason, (socket.gaierror, ssl.SSLError, socket.timeout)):
- return True, ""
- except Exception as e:
- pass
-
- return False, ""
-
- def attempt_env() -> str:
- status, content = request_once(suffix="/env.js")
- if status:
- return terminated
-
- if not content:
- return ""
-
- if groups := re.findall(r"\bhost\b:(?:\s+)?[\"\'](https?://[^\s\r\t]+)[\"\']", content, flags=re.I):
- return utils.trim(groups[0])
-
- groups = re.findall(r"window.routerBase(?:\s+)?=(?:\s+)?['\"](https?://.*)['\"]", content, flags=re.I)
- return groups[0].rstrip("/") if groups and groups[0] else ""
-
- def attempt_zero() -> str:
- status, content = request_once(suffix="/config.json")
- if status:
- return terminated
-
- if not content:
- return ""
-
- try:
- data = json.loads(content)
- # for https://github.com/amyouran/v2board-Zero-Theme
- link = utils.trim(data.get("api_base", ""))
- if not link:
- # for https://github.com/DyAxy/V2B-Theme-Nest
- link = utils.trim(data.get("apiUrl", ""))
-
- return utils.extract_domain(url=link, include_protocal=True)
- except:
- return ""
-
- def attempt_buddy() -> str:
- # for https://github.com/vlesstop/v2board-theme-buddy
- status, content = request_once(suffix="/config.js")
- if status:
- return terminated
-
- group = re.findall(r"\bhost\b:(?:\s+)?[\"\'](https?://[^\s\r\t]+)[\"\']", content, flags=re.I)
- return "" if not group else utils.trim(group[0])
-
- def attempt_aurora() -> str:
- # for https://github.com/krsunm/Aurora
- status, content = request_once(suffix="")
- if status:
- return terminated
-
- group = re.findall(r"\bserverUrl\b:(?:\s+)?[\"\'](https?://[^\s\r\t]+)[\"\']", content, flags=re.I)
- return "" if not group else utils.trim(group[0])
-
- terminated, retry = "terminated", max(retry, 1)
- for func in [attempt_env, attempt_zero, attempt_buddy, attempt_aurora]:
- backend = func()
- if terminated == backend:
- return ""
- if backend:
- return backend
-
- return domain
-
- domains = crawl_channel(channel=channel, page_num=page_num, fun=extract_airport_site)
- candidates = {} if not domains else {utils.extract_domain(x, True): "" for x in domains}
-
- materials = dict()
- jctj = crawl_jctj(convert=False)
- if jctj:
- materials.update(jctj)
-
- ccbh = crawl_ccbh()
- if ccbh:
- materials.update(ccbh)
-
- maomeng = crawl_maomeng()
- if maomeng:
- materials.update(maomeng)
-
- askahh = crawl_askahh()
- if askahh:
- materials.update(askahh)
-
- ygpy = crawl_ygpy()
- if ygpy:
- materials.update(ygpy)
-
- # save to file cause they often contain coupons and require common emails to use
- save_candidates(candidates=materials, filepath=filepath, delimiter=delimiter)
-
- # merge
- candidates.update(materials)
- domains = list(candidates.keys())
-
- # extract real routing base url
- logger.info(f"[AirPortCollector] fetched {len(domains)} airport, start extracting real routing addresses")
- sites = utils.multi_thread_run(
- func=extract_backend_url,
- tasks=domains,
- num_threads=num_thread,
- show_progress=display,
- )
-
- tasks = [[site, rigid, chuck] for site in sites if site]
- records = {sites[i]: candidates.get(domains[i], "") for i in range(len(sites)) if sites[i]}
-
- # check website availability
- logger.info(f"[AirPortCollector] extract real base url finished, start to check it now")
- result = utils.multi_thread_run(func=validate_domain, tasks=tasks, num_threads=num_thread, show_progress=display)
-
- availables = dict()
- for i in range(len(tasks)):
- if not result[i][0]:
- continue
-
- site = tasks[i][0]
- coupon = records.get(site, "")
- availables[site] = {"coupon": coupon, "api_prefix": result[i][1]}
-
- logger.info(f"[AirPortCollector] finished collect airport, availables: {len(availables)}")
- return availables
-
-
-def save_candidates(candidates: dict, filepath: str, delimiter: str) -> None:
- if not candidates or not isinstance(candidates, dict):
- return
-
- filepath = utils.trim(filepath)
- if not filepath:
- return
-
- delimiter = utils.trim(delimiter) or "@#@#"
-
- lines = []
- for k, v in candidates.items():
- text = k
- if v and isinstance(v, str):
- text += f"\t{delimiter}\t{v}"
- elif v and isinstance(v, dict):
- coupon = utils.trim(v.get("coupon", ""))
- invite_code = utils.trim(v.get("invite_code", ""))
- api_prefix = utils.trim(v.get("api_prefix", ""))
-
- text = f"{text}\t{delimiter}\t{coupon}\t{delimiter}\t{invite_code}\t{delimiter}\t{api_prefix}"
- lines.append(text)
-
- utils.write_file(filename=filepath, lines=lines)
-
-
-def validate_domain(url: str, rigid: bool = True, chuck: bool = False) -> tuple[bool, str]:
- try:
- if not url:
- return False, ""
-
- rr = airport.AirPort.get_register_require(domain=url)
- flag = rr.invite or (chuck and rr.recaptcha) or (rigid and rr.whitelist and rr.verify)
- return not flag, rr.api_prefix
- except:
- return False, ""
-
-
-def batch_call(tasks: dict) -> list[dict]:
- if not tasks:
- return []
-
- try:
- num_thread = max(min(len(tasks), 50), 1)
- with multiprocessing.Manager() as manager:
- availables = manager.list()
- processes = []
- semaphore = multiprocessing.Semaphore(num_thread)
- time.sleep(random.randint(1, 3))
- for k, v in tasks.items():
- semaphore.acquire()
- p = multiprocessing.Process(target=call, args=(k, v, availables, semaphore))
- p.start()
- processes.append(p)
- for p in processes:
- p.join()
-
- return list(availables)
- except:
- traceback.print_exc()
- return []
-
-
-def call(script: str, params: dict, availables: ListProxy, semaphore: Semaphore) -> None:
- try:
- if not script:
- return
-
- subscribes = execute_script(script=script, params=params)
- if subscribes and type(subscribes) == list:
- availables.extend(subscribes)
- finally:
- if semaphore is not None and isinstance(semaphore, Semaphore):
- semaphore.release()
-
-
-def execute_script(script: str, params: dict = {}) -> list[dict]:
- try:
- # format: a.b.c#function or a-b.c#_function or a#function and so on
- regex = r"^([a-zA-Z0-9_]+|([0-9a-zA-Z_]+([a-zA-Z0-9_\-]+)?\.)+)[a-zA-Z0-9_\-]+#[a-zA-Z_]+[0-9a-zA-Z_]+$"
- if not re.match(regex, script):
- logger.info(f"[ScriptError] script execute error because script: {script} is invalidate")
- return []
-
- path, func_name = script.split("#", maxsplit=1)
- path = f"scripts.{path}"
- module = importlib.import_module(path)
- if not hasattr(module, func_name):
- logger.error(f"script: {path} not exists function {func_name}")
- return []
-
- func = getattr(module, func_name)
-
- starttime = time.time()
- logger.info(f"[ScriptInfo] start execute script: scripts.{script}")
-
- subscribes = func(params)
- if type(subscribes) != list:
- logger.error(f"[ScriptError] return value error, need a list, but got a {type(subscribes)}")
- return []
-
- endtime = time.time()
- logger.info(
- "[ScriptInfo] finished execute script: scripts.{}, cost: {:.2f}s".format(script, endtime - starttime)
- )
-
- subscribes = [s for s in subscribes if type(s) == dict and s.get("push_to", [])]
- return subscribes
- except:
- logger.error(f"[ScriptError] occur error run script: {script}, message: \n{traceback.format_exc()}")
- return []
diff --git a/subscribe/crawl/__init__.py b/subscribe/crawl/__init__.py
new file mode 100644
index 0000000000..4b727f5766
--- /dev/null
+++ b/subscribe/crawl/__init__.py
@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+
+from crawl.engine import run
+from crawl.helpers import check_status, is_available, naming_task
+
+__all__ = [
+ "run",
+ "check_status",
+ "is_available",
+ "naming_task",
+]
diff --git a/subscribe/crawl/base.py b/subscribe/crawl/base.py
new file mode 100644
index 0000000000..f61863d3f4
--- /dev/null
+++ b/subscribe/crawl/base.py
@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any, Generic, TypeVar
+
+from crawl.models import ChannelResult, CrawlContext
+
+TConfig = TypeVar("TConfig")
+
+CHANNELS: dict[str, Channel[Any]] = {}
+
+
+class Channel(ABC, Generic[TConfig]):
+ name: str = ""
+
+ @abstractmethod
+ def crawl(self, config: TConfig, ctx: CrawlContext) -> ChannelResult:
+ raise NotImplementedError
+
+
+def register_channel(channel: Channel[Any]) -> Channel[Any]:
+ CHANNELS[channel.name] = channel
+ return channel
diff --git a/subscribe/crawl/channels/__init__.py b/subscribe/crawl/channels/__init__.py
new file mode 100644
index 0000000000..2d2180dbe9
--- /dev/null
+++ b/subscribe/crawl/channels/__init__.py
@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+
+from importlib import import_module
+
+from crawl.base import CHANNELS
+
+for _name in (
+ "github",
+ "google",
+ "page",
+ "repository",
+ "script",
+ "telegram",
+ "twitter",
+ "yandex",
+):
+ import_module(f"{__name__}.{_name}")
+
+__all__ = ["CHANNELS"]
diff --git a/subscribe/crawl/channels/github.py b/subscribe/crawl/channels/github.py
new file mode 100644
index 0000000000..ac012e2a10
--- /dev/null
+++ b/subscribe/crawl/channels/github.py
@@ -0,0 +1,186 @@
+# -*- coding: utf-8 -*-
+
+import itertools
+import json
+import os
+import re
+import time
+import traceback
+
+import utils
+from config.models import GithubConfig
+from crawl.base import Channel, register_channel
+from crawl.channels.page import PageChannel
+from crawl.models import ChannelResult, CrawlContext
+from logger import logger
+from origin import Origin
+
+
+class GithubChannel(Channel[GithubConfig]):
+ name = "github"
+
+ def crawl(self, config: GithubConfig, ctx: CrawlContext) -> ChannelResult:
+ return crawl_github(config, ctx)
+
+
+def intercept(text: str, excludes: list[str] | None = None) -> bool:
+ if not excludes:
+ return False
+ for regex in excludes:
+ try:
+ if re.search(regex, text, flags=re.I):
+ return True
+ except Exception:
+ logger.error(f"[GithubRepoIntercept] invalid regex pattern: {regex}")
+ return False
+
+
+def paging(start: int, end: int, peer_page: int) -> list[int]:
+ if start > end or peer_page <= 0:
+ return []
+ pages = []
+ for i in range(start, end + 1, peer_page):
+ pages.append(i // peer_page + 1)
+ return pages
+
+
+def search_github(page: int, cookie: str, searchtype: str, sortedby: str) -> str:
+ if page <= 0 or utils.isblank(cookie):
+ return ""
+
+ searchtype = "Code" if utils.isblank(searchtype) else searchtype
+ sortedby = "indexed" if utils.isblank(sortedby) else sortedby
+ query = "%22%2Fapi%2Fv1%2Fclient%2Fsubscribe%3Ftoken%3D%22"
+ if searchtype.lower() == "code":
+ query = "%2F%5C%2Fapi%5C%2Fv1%5C%2Fclient%5C%2Fsubscribe%5C%3Ftoken%3D%5Ba-zA-Z0-9%5D%7B8%2C32%7D%2F"
+
+ url = f"/search?o=desc&p={page}&q={query}&s={sortedby}&type={searchtype}"
+ headers = {
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
+ "Referer": "https://github.com",
+ "User-Agent": utils.USER_AGENT,
+ "Cookie": f"user_session={cookie}",
+ }
+ content = utils.http_get(url=url, headers=headers)
+ if re.search(r"Sign in to GitHub
", content, flags=re.I):
+ logger.error("[GithubCrawl] session has expired, please provide a valid session and try again")
+ return ""
+ return content
+
+
+def search_github_issues(page: int, cookie: str) -> list[str]:
+ content = search_github(page=page, cookie=cookie, searchtype="Issues", sortedby="created")
+ if utils.isblank(content):
+ return []
+ try:
+ groups = re.findall(r'href="(/.*/.*/issues/\d+)">', content, flags=re.I)
+ return [f"https://github.com{item}" for item in list(set(groups))]
+ except Exception:
+ return []
+
+
+def search_github_issues_byapi(peer_page: int = 50, page: int = 1) -> list[str]:
+ peer_page, page = min(max(peer_page, 1), 100), max(1, page)
+ url = f"https://api.github.com/search/issues?q=%22%2Fapi%2Fv1%2Fclient%2Fsubscribe%3Ftoken%3D%22&sort=created&order=desc&per_page={peer_page}&page={page}"
+ content = utils.http_get(url=url)
+ if utils.isblank(content):
+ return []
+ try:
+ items, links = json.loads(content).get("items", []), set()
+ for item in items:
+ link = item.get("html_url", "")
+ if link:
+ links.add(link)
+ return list(links)
+ except Exception:
+ logger.error("[GithubIssuesCrawl] occur error when search issues from github")
+ traceback.print_exc()
+ return []
+
+
+def search_github_code_byapi(
+ token: str, peer_page: int = 50, page: int = 1, excludes: list[str] | None = None
+) -> list[str]:
+ if utils.isblank(token):
+ return []
+
+ peer_page, page = min(max(peer_page, 1), 100), max(1, page)
+ url = f"https://api.github.com/search/code?q=%22%2Fapi%2Fv1%2Fclient%2Fsubscribe%3Ftoken%3D%22&sort=indexed&order=desc&per_page={peer_page}&page={page}"
+ headers = {"Accept": "application/vnd.github+json", "Authorization": f"Bearer {token}"}
+ content = utils.http_get(url=url, headers=headers)
+ if utils.isblank(content):
+ return []
+ try:
+ items = json.loads(content).get("items", [])
+ links = set()
+ excludes = list(set(excludes or []))
+ for item in items:
+ if not item or not isinstance(item, dict):
+ continue
+ link = item.get("html_url", "")
+ if not link:
+ continue
+ reponame = item.get("repository", {}).get("full_name", "") + "/"
+ if not intercept(text=reponame, excludes=excludes):
+ links.add(link)
+ return list(links)
+ except Exception:
+ return []
+
+
+def search_github_code(page: int, cookie: str, excludes: list[str] | None = None) -> list[str]:
+ content = search_github(page=page, cookie=cookie, searchtype="Code", sortedby="indexed")
+ if utils.isblank(content):
+ return []
+ try:
+ groups = re.findall(r'href="(/[^\s"]+/blob/(?:[^"]+)?)#L\d+"', content, flags=re.I)
+ uris = list(set(groups)) if groups else []
+ links = set()
+ excludes = list(set(excludes or []))
+ for uri in uris:
+ if not intercept(text=uri, excludes=excludes):
+ links.add(f"https://github.com{uri}")
+ return list(links)
+ except Exception:
+ return []
+
+
+def crawl_github(config: GithubConfig, ctx: CrawlContext) -> ChannelResult:
+ cookie = os.environ.get("GH_COOKIE", "").strip()
+ token = os.environ.get("GH_TOKEN", "").strip()
+ if utils.isblank(cookie) and utils.isblank(token):
+ logger.error("[GithubCrawl] cannot start crawl from github because cookie and token is missing")
+ return ChannelResult()
+
+ links, starttime = [], time.time()
+ method = "search on the page" if utils.isblank(token) else "rest api"
+ excludes = config.exclude_repos or []
+
+ if utils.isblank(token):
+ pages = [item for item in range(1, config.pages + 1)] * 2
+ params = [[item, cookie, excludes] for item in pages]
+ results = utils.multi_thread_run(func=search_github_code, tasks=params, num_threads=ctx.num_threads)
+ links.extend(list(set(itertools.chain.from_iterable(results))))
+ links.extend(search_github_issues(page=1, cookie=cookie))
+ else:
+ peer_page, count = 50, 10
+ pages = paging(start=1, end=config.pages * count, peer_page=peer_page)
+ params = [[token, peer_page, item, excludes] for item in pages] * 2
+ results = utils.multi_thread_run(func=search_github_code_byapi, tasks=params, num_threads=ctx.num_threads)
+ links.extend(list(set(itertools.chain.from_iterable(results))))
+ links.extend(search_github_issues_byapi(peer_page=5, page=1))
+
+ page = PageChannel(
+ include="",
+ exclude=config.exclude,
+ push_to=config.push_to,
+ origin=Origin.GITHUB.name,
+ )
+ result = page.fetch_many(ctx, urls=list(dict.fromkeys(links)))
+ logger.info(
+ f"[GithubCrawl] finished crawl from Github through {method}, found {len(result.items)} subscriptions need check, cost: {time.time() - starttime:.2f}s"
+ )
+ return result
+
+
+register_channel(GithubChannel())
diff --git a/subscribe/crawl/channels/google.py b/subscribe/crawl/channels/google.py
new file mode 100644
index 0000000000..c0ad74a3f6
--- /dev/null
+++ b/subscribe/crawl/channels/google.py
@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+
+import re
+import time
+import urllib.parse
+
+import utils
+from config.models import GoogleConfig, TaskParams
+from crawl.base import Channel, register_channel
+from crawl.helpers import is_reachable
+from crawl.models import ChannelResult, CrawlContext
+from logger import logger
+from origin import Origin
+
+
+class GoogleChannel(Channel[GoogleConfig]):
+ name = "google"
+
+ def crawl(self, config: GoogleConfig, ctx: CrawlContext) -> ChannelResult:
+ if not is_reachable("https://www.google.com"):
+ logger.warning("[GoogleCrawl] skip because google is unreachable")
+ return ChannelResult()
+ return crawl_google(config)
+
+
+def crawl_google(config: GoogleConfig) -> ChannelResult:
+ items, query = set(), urllib.parse.quote('"/api/v1/client/subscribe?token="')
+ for text in config.exclude_sites or []:
+ text = utils.trim(text).lower()
+ if text and "+" not in text:
+ items.add(urllib.parse.quote(f"-site:{text}"))
+
+ reject = "+".join(list(items))
+ if reject:
+ query = f"{query}+{reject}"
+
+ num, limits = 100, min(max(1, config.limit), 1000)
+ url = f"https://www.google.com/search?q={query}&tbs=qdr:d{max(config.days, 1)}"
+ params = {"hl": "zh-CN", "num": num}
+ task = TaskParams(push_to=list(config.push_to))
+ result = ChannelResult()
+ starttime = time.time()
+
+ for start in range(0, limits, num):
+ params["start"] = start
+ content = re.sub(r"\\\\n", "", utils.http_get(url=url, params=params))
+ content = re.sub(r"\?token\\\\u003d", "?token=", content, flags=re.I)
+ regex = r'https?://(?:[a-zA-Z0-9_\u4e00-\u9fa5\-]+\.)+[a-zA-Z0-9_\u4e00-\u9fa5\-]+(?::\d+)?/?(?:/?)?api/v1/client/subscribe\?token(?:)?=[a-zA-Z0-9]{16,32}'
+ subscribes = re.findall(regex, content)
+ for link in subscribes:
+ link = re.sub(r'||\s+', "", link).replace("http://", "https://", 1)
+ try:
+ if config.exclude and re.search(config.exclude, link):
+ continue
+ result.add_subscribe(link, Origin.GOOGLE.name, task)
+ except Exception:
+ continue
+
+ if re.search(
+ r'\s*找不到和您查询的“\s*\s*.*?/api/v1/client/subscribe\?token=.*?\s*\s*”相符的内容或信息。\s*
',
+ content,
+ flags=re.I,
+ ):
+ break
+
+ logger.info(
+ f"[GoogleCrawl] finished crawl from Google, found {len(result.items)} subscriptions, cost: {time.time() - starttime:.2f}s"
+ )
+ return result
+
+
+register_channel(GoogleChannel())
diff --git a/subscribe/crawl/channels/page.py b/subscribe/crawl/channels/page.py
new file mode 100644
index 0000000000..a368cb0fc1
--- /dev/null
+++ b/subscribe/crawl/channels/page.py
@@ -0,0 +1,100 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+from copy import deepcopy
+
+import utils
+from config.models import PageJob, TaskParams
+from crawl.base import Channel, register_channel
+from crawl.extract import extract_subscribes
+from crawl.models import ChannelResult, CrawlContext
+from logger import logger
+from origin import Origin
+from urlvalidator import isurl
+
+
+class PageChannel(Channel[list[PageJob]]):
+ name = "pages"
+
+ def __init__(
+ self,
+ url: str = "",
+ include: str = "",
+ exclude: str = "",
+ push_to: list[str] | None = None,
+ task: TaskParams | None = None,
+ headers: dict[str, str] | None = None,
+ origin: str = Origin.PAGE.name,
+ skip_cache: bool = False,
+ ) -> None:
+ self.url = utils.trim(url)
+ self.include = include
+ self.exclude = exclude
+ self.push_to = list(push_to or [])
+ self.task = task or TaskParams(push_to=self.push_to)
+ self.headers = headers
+ self.origin = origin or Origin.PAGE.name
+ self.skip_cache = skip_cache
+
+ def crawl(self, config: list[PageJob], ctx: CrawlContext) -> ChannelResult:
+ from crawl.helpers import fetch_jobs
+
+ jobs = []
+ for job in config:
+ if not job.enable or not job.push_to:
+ continue
+ for url in job.expand_urls():
+ jobs.append(
+ PageChannel(
+ url=url,
+ include=job.include,
+ exclude=job.exclude,
+ push_to=job.push_to,
+ task=job.task,
+ headers=job.headers,
+ origin=job.origin,
+ skip_cache=job.skip_cache,
+ )
+ )
+ return fetch_jobs(jobs, ctx)
+
+ def fetch(self, ctx: CrawlContext, url: str | None = None) -> ChannelResult:
+ target = utils.trim(url or self.url)
+ if not target or not isurl(target):
+ logger.error(f"[PageCrawl] cannot crawl from page: {target}")
+ return ChannelResult()
+ headers = deepcopy(self.headers) if self.headers else None
+ content = utils.http_get(url=target, headers=headers)
+ if not content:
+ return ChannelResult()
+ return extract_subscribes(
+ content=content,
+ push_to=self.push_to,
+ include=self.include,
+ exclude=self.exclude,
+ source=self.origin,
+ task=self.task,
+ skip_cache=self.skip_cache,
+ include_nodes=ctx.include_nodes,
+ )
+
+ def fetch_many(self, ctx: CrawlContext, urls: list[str] | None = None) -> ChannelResult:
+ from crawl.helpers import merge_results
+
+ targets = urls if urls is not None else ([self.url] if self.url else [])
+ targets = [utils.trim(item) for item in targets if utils.trim(item)]
+ if not targets:
+ return ChannelResult()
+ if len(targets) == 1:
+ return self.fetch(ctx, url=targets[0])
+ results = utils.multi_thread_run(
+ func=self.fetch,
+ tasks=[[ctx, link] for link in targets],
+ num_threads=ctx.num_threads,
+ show_progress=ctx.display,
+ )
+ return merge_results(results)
+
+
+register_channel(PageChannel())
diff --git a/subscribe/crawl/channels/plugins/__init__.py b/subscribe/crawl/channels/plugins/__init__.py
new file mode 100644
index 0000000000..b04b34d6fa
--- /dev/null
+++ b/subscribe/crawl/channels/plugins/__init__.py
@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+
+from importlib import import_module
+
+from .base import PLUGINS, PluginContext
+
+for _name in (
+ "dynamic",
+ "fofa",
+ "gitforks",
+ "scaner",
+ "tempairport",
+ "v2rayfree",
+ "v2rayse",
+):
+ import_module(f"{__name__}.{_name}")
+
+__all__ = ["PLUGINS", "PluginContext"]
diff --git a/subscribe/crawl/channels/plugins/base.py b/subscribe/crawl/channels/plugins/base.py
new file mode 100644
index 0000000000..153daa645a
--- /dev/null
+++ b/subscribe/crawl/channels/plugins/base.py
@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any, Generic, TypeVar
+
+from config.models import StorageItem, TaskParams
+from crawl.models import ChannelResult, CrawlContext
+from push import PushTo
+
+TConfig = TypeVar("TConfig")
+PLUGINS: dict[str, ScriptPlugin[Any]] = {}
+
+
+@dataclass
+class PluginContext:
+ crawl: CrawlContext
+ params: dict[str, object]
+ task: TaskParams
+ persist: StorageItem | dict[str, StorageItem] | None
+ pushtool: PushTo | None
+ storage_items: dict[str, StorageItem]
+
+
+class ScriptPlugin(ABC, Generic[TConfig]):
+ name: str = ""
+ isolate: bool = False
+
+ @abstractmethod
+ def parse(self, ctx: PluginContext) -> TConfig:
+ raise NotImplementedError
+
+ @abstractmethod
+ def run(self, config: TConfig, ctx: PluginContext) -> ChannelResult:
+ raise NotImplementedError
+
+
+def register_plugin(plugin: ScriptPlugin[Any]) -> ScriptPlugin[Any]:
+ PLUGINS[plugin.name] = plugin
+ return plugin
diff --git a/subscribe/crawl/channels/plugins/commons.py b/subscribe/crawl/channels/plugins/commons.py
new file mode 100644
index 0000000000..6622df5360
--- /dev/null
+++ b/subscribe/crawl/channels/plugins/commons.py
@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+
+# @Author : wzdnzd
+# @Time : 2022-11-12
+
+import json
+
+from config.models import NodeInput, StorageItem, TaskParams
+from crawl.models import ChannelResult, SubItem
+from logger import logger
+from origin import Origin
+from push import PushTo
+
+from .base import PluginContext
+
+
+def persist(pushtool: PushTo | None, data: dict[str, object] | None, item: StorageItem | None, meta: str = "") -> None:
+ try:
+ if (
+ data is None
+ or not isinstance(data, dict)
+ or not isinstance(pushtool, PushTo)
+ or not isinstance(item, StorageItem)
+ or not pushtool.validate(item=item)
+ ):
+ logger.debug(f"[{meta}] skip persist subscribes because storage item or data is empty")
+ return
+ pushtool.push_to(content=json.dumps(data), item=item, group="subscribes")
+ except Exception:
+ logger.error(f"[{meta}] occur error when persist subscribes")
+
+
+def as_channel_result(items: list[dict[str, object]] | None) -> ChannelResult:
+ result = ChannelResult()
+ if not items:
+ return result
+ urls, uris, proxies = [], [], []
+ for item in items:
+ if not isinstance(item, dict):
+ continue
+ subscribe = item.get("subscribe", item.get("sub", ""))
+ saved = bool(item.get("saved", False))
+ checked = bool(item.get("checked", True))
+ origin = item.get("origin", Origin.TEMPORARY.name)
+ task = TaskParams(
+ push_to=list(item.get("push_to", [])),
+ rename=item.get("rename"),
+ name=item.get("name"),
+ include=item.get("include"),
+ exclude=item.get("exclude"),
+ )
+ links = subscribe if isinstance(subscribe, list) else [subscribe]
+ for url in links:
+ if not url:
+ continue
+ urls.append(url)
+ result.items.append(
+ SubItem(
+ url=url,
+ origin=origin,
+ task=task,
+ name=item.get("name", ""),
+ ready=saved,
+ persist_only=checked and not saved,
+ skip_cache=bool(item.get("skip_cache", item.get("nocache", False))),
+ allow_nonstandard=bool(item.get("allow_nonstandard", item.get("pardon", False))),
+ )
+ )
+ extra = item.get("proxies")
+ if extra and isinstance(extra, list):
+ if extra and isinstance(extra[0], str):
+ uris.extend(extra)
+ elif extra and isinstance(extra[0], dict):
+ proxies.extend(extra)
+ result.nodes = NodeInput(subscribe=list(dict.fromkeys(urls)), uris=list(dict.fromkeys(uris)), proxies=proxies)
+ return result
+
+
+def plugin_params(ctx: PluginContext) -> dict[str, object]:
+ if not isinstance(ctx, PluginContext):
+ return {}
+ params = dict(ctx.params)
+ params["task"] = ctx.task
+ params["config"] = ctx.task.to_dict()
+ if ctx.task.push_to is not None:
+ params["config"]["push_to"] = list(ctx.task.push_to)
+ return params
diff --git a/subscribe/scripts/dynamic.py b/subscribe/crawl/channels/plugins/dynamic.py
similarity index 75%
rename from subscribe/scripts/dynamic.py
rename to subscribe/crawl/channels/plugins/dynamic.py
index e3c302c40c..e1f7e7f9f7 100644
--- a/subscribe/scripts/dynamic.py
+++ b/subscribe/crawl/channels/plugins/dynamic.py
@@ -7,12 +7,18 @@
from copy import deepcopy
from datetime import datetime, timedelta
-import crawl
import utils
+from config.models import TaskParams
+from crawl.channels.page import PageChannel
+from crawl.helpers import fetch_jobs, is_available
+from crawl.models import ChannelResult, CrawlContext
from logger import logger
from origin import Origin
from urlvalidator import isurl
+from .base import PluginContext, ScriptPlugin, register_plugin
+from .commons import as_channel_result, plugin_params
+
# github content api prefix
GITHUB_CONTENT_API = "https://raw.githubusercontent.com"
@@ -20,7 +26,7 @@
DEFAULT_BRANCH = "main"
-def format(text: str, date: datetime = None) -> str:
+def format(text: str, date: datetime | None = None) -> str:
"""
Replace all time placeholders in text with current time values
@@ -48,7 +54,7 @@ def format(text: str, date: datetime = None) -> str:
if not date or not isinstance(date, datetime):
date = datetime.now()
- def replace(match):
+ def replace(match: re.Match[str]) -> str:
# YYYY, mm, dd, HH, MM, SS
placeholder = match.group(1)
@@ -83,7 +89,7 @@ def replace(match):
return result
-def collect_subs(params: dict) -> list[dict]:
+def collect_subs(params: dict[str, object], ctx: PluginContext | None = None) -> list[dict[str, object]]:
if not params or type(params) != dict:
return []
@@ -141,23 +147,28 @@ def collect_subs(params: dict) -> list[dict]:
target.update({"sub": url, "saved": True})
materials[url] = target
else:
- sources.append([url, push_to, include, exclude, config, None, Origin.PAGE])
+ sources.append(
+ PageChannel(url=url, include=include, exclude=exclude, push_to=push_to, origin=Origin.PAGE.name)
+ )
if sources:
- urls = [x[0] for x in sources]
+ urls = [job.url for job in sources]
logger.info(f"[CollectSub] start to collect subscriptions from {len(urls)} urls: {urls}")
-
- results = utils.multi_thread_run(func=crawl.crawl_single_page, tasks=sources)
- for result in results:
- if not result or not isinstance(result, dict):
- continue
-
- for k, v in result.items():
- if not k or not v or not isinstance(v, dict):
- continue
-
- v.update({"sub": k, "saved": True})
- materials[k] = v
+ crawled = fetch_jobs(
+ sources,
+ (
+ ctx.crawl
+ if ctx and ctx.crawl
+ else CrawlContext(
+ mode=0, include_nodes=True, max_fails=5, exclude="", task=TaskParams(), storage=None, pushtool=None
+ )
+ ),
+ )
+
+ for item in crawled.items:
+ payload = deepcopy(config)
+ payload.update({"sub": item.url, "saved": True})
+ materials[item.url] = payload
# filter conditions
try:
@@ -170,10 +181,23 @@ def collect_subs(params: dict) -> list[dict]:
# check availability
candidates = list(materials.keys())
tasks = [[x, 2, remain, life] for x in candidates]
- masks = utils.multi_thread_run(func=crawl.is_available, tasks=tasks)
+ masks = utils.multi_thread_run(func=is_available, tasks=tasks)
# filter available subscriptions
subs = sorted([candidates[i] for i in range(len(masks)) if masks[i]])
logger.info(f"[CollectSub] collect task finished, found {len(subs)} subscriptions")
return [materials.get(k) for k in subs]
+
+
+class DynamicPlugin(ScriptPlugin[dict[str, object]]):
+ name = "dynamic"
+
+ def parse(self, ctx: PluginContext) -> dict[str, object]:
+ return plugin_params(ctx)
+
+ def run(self, config: dict[str, object], ctx: PluginContext) -> ChannelResult:
+ return as_channel_result(collect_subs(config, ctx))
+
+
+register_plugin(DynamicPlugin())
diff --git a/subscribe/scripts/fofa.py b/subscribe/crawl/channels/plugins/fofa.py
similarity index 87%
rename from subscribe/scripts/fofa.py
rename to subscribe/crawl/channels/plugins/fofa.py
index 30e8ef3e94..45f8e58c6d 100644
--- a/subscribe/scripts/fofa.py
+++ b/subscribe/crawl/channels/plugins/fofa.py
@@ -17,11 +17,15 @@
import utils
import yaml
-from crawl import naming_task
+from crawl.helpers import naming_task
+from crawl.models import ChannelResult
from logger import logger
from origin import Origin
from urlvalidator import isurl
+from .base import PluginContext, ScriptPlugin, register_plugin
+from .commons import as_channel_result, plugin_params
+
def search(exclude: str = "", maxsize: int = sys.maxsize, timesleep: float = 3, timeout: float = 180) -> list[str]:
try:
@@ -67,7 +71,7 @@ def extract_one(url: str) -> list[str]:
regex = r"(?:https?://)?(?:[a-zA-Z0-9\u4e00-\u9fa5\-]+\.)+[a-zA-Z0-9\u4e00-\u9fa5\-]+(?:(?:(?:/index.php)?/api/v1/client/subscribe\?token=[a-zA-Z0-9]{16,32})|(?:/link/[a-zA-Z0-9]+\?(?:sub|mu|clash)=\d)|(?:/(?:s|sub)/[a-zA-Z0-9]{32}))"
- headers = {"User-Agent": "Clash.Meta; Mihomo"}
+ headers = {"User-Agent": f"{utils.USER_AGENT}; Clash.Meta; Mihomo; Shadowrocket;"}
subscriptions, content = [], ""
count, retry = 0, 2
@@ -119,15 +123,15 @@ def extract_one(url: str) -> list[str]:
return subscriptions
-def recall(params: dict) -> list:
- def inwrap(sub: str, nocache: bool = True, pardon: bool = False) -> dict:
+def recall(params: dict[str, object]) -> list[dict[str, object]]:
+ def inwrap(sub: str, nocache: bool = True, pardon: bool = False) -> dict[str, object]:
config = deepcopy(params.get("config", {}))
config["sub"] = sub
config["saved"] = False
config["checked"] = False
config["nocache"] = nocache
config["pardon"] = pardon
- config["name"] = naming_task(link)
+ config["name"] = naming_task(sub)
config["origin"] = Origin.FOFA.name
config["push_to"] = list(set(config.get("push_to", [])))
@@ -167,3 +171,16 @@ def inwrap(sub: str, nocache: bool = True, pardon: bool = False) -> dict:
logger.info(f"[FOFA] search finished, found {len(tasks)} candidates to be check, cost: {cost}")
return tasks
+
+
+class FofaPlugin(ScriptPlugin[dict[str, object]]):
+ name = "fofa"
+
+ def parse(self, ctx: PluginContext) -> dict[str, object]:
+ return plugin_params(ctx)
+
+ def run(self, config: dict[str, object], ctx: PluginContext) -> ChannelResult:
+ return as_channel_result(recall(config))
+
+
+register_plugin(FofaPlugin())
diff --git a/subscribe/scripts/gitforks.py b/subscribe/crawl/channels/plugins/gitforks.py
similarity index 77%
rename from subscribe/scripts/gitforks.py
rename to subscribe/crawl/channels/plugins/gitforks.py
index da4fd37726..0e00d00106 100644
--- a/subscribe/scripts/gitforks.py
+++ b/subscribe/crawl/channels/plugins/gitforks.py
@@ -9,13 +9,19 @@
import time
from copy import deepcopy
-import crawl
-import push
import utils
+from config.models import StorageItem, TaskParams
+from crawl.channels.page import PageChannel
+from crawl.helpers import fetch_jobs, is_available, naming_task
+from crawl.models import ChannelResult, CrawlContext
from logger import logger
from origin import Origin
+from push import PushTo
from urlvalidator import isurl
+from .base import PluginContext, ScriptPlugin, register_plugin
+from .commons import as_channel_result, plugin_params
+
# github rest api prefix
GITHUB_API = "https://api.github.com"
@@ -60,7 +66,7 @@ def query_forks_count(username: str, repository: str, retry: int = 3) -> int:
return -1
-def query_forks(username: str, repository: str, page: int, peer: int = 100, sort: str = "newest") -> dict:
+def query_forks(username: str, repository: str, page: int, peer: int = 100, sort: str = "newest") -> dict[str, object]:
username = utils.trim(username)
repository = utils.trim(repository)
@@ -106,9 +112,9 @@ def query_forks(username: str, repository: str, page: int, peer: int = 100, sort
return subscriptions
-def collect_subs(params: dict) -> list[dict]:
- def update_conf(config: dict, sub: str, name: str = "") -> dict:
- name = crawl.naming_task(url=sub) if not name else name
+def collect_subs(params: dict[str, object], ctx: PluginContext | None = None) -> list[dict[str, object]]:
+ def update_conf(config: dict[str, object], sub: str, name: str = "") -> dict[str, object]:
+ name = naming_task(url=sub) if not name else name
item = deepcopy(config)
item.update({"name": name, "sub": sub, "saved": True})
@@ -131,16 +137,12 @@ def github_warp(ghproxy: str, url: str) -> str:
logger.error(f"[GithubFork] cannot list forks from github due to username or repository is empty")
return []
- # used to store subscriptions
- storage = params.get("storage", {})
- if not storage or type(storage) != dict:
- logger.error(f"[GithubFork] cannot fetch subscriptions due to invalid storage config")
+ if ctx is not None and not isinstance(ctx, PluginContext):
return []
-
- persist = storage.get("items", {})
- pushtool = push.get_instance(config=push.PushConfig.from_dict(storage))
- if not pushtool.validate(config=persist):
- logger.error(f"[GithubFork] cannot fetch subscriptions due to invalid persist config")
+ pushtool = ctx.pushtool if ctx else None
+ persist = ctx.persist if ctx else None
+ if not isinstance(pushtool, PushTo) or not isinstance(persist, StorageItem) or not pushtool.validate(item=persist):
+ logger.error("[GithubFork] cannot fetch subscriptions due to invalid persist config")
return []
# only keep subscriptions, usually used when there are too many nodes to save to the remote service
@@ -159,7 +161,7 @@ def github_warp(ghproxy: str, url: str) -> str:
materials, tasks = {}, []
# load old subscriptions
- content = utils.http_get(url=pushtool.raw_url(config=persist), timeout=30)
+ content = utils.http_get(url=pushtool.raw_url(item=persist), timeout=30)
urls = re.findall(r"^https?:\/\/[^\s]+", content, flags=re.M)
for url in urls:
url = github_warp(ghproxy=ghproxy, url=url)
@@ -226,25 +228,30 @@ def github_warp(ghproxy: str, url: str) -> str:
proxy = github_warp(ghproxy=ghproxy, url=proxy)
materials[proxy] = update_conf(config=config, sub=proxy, name=name)
for sub in subs:
- tasks.append([sub, push_to, include, exclude, config, None, Origin.PAGE])
+ tasks.append(
+ PageChannel(url=sub, include=include, exclude=exclude, push_to=push_to, origin=Origin.PAGE.name)
+ )
# crawl all subscriptions from subscriptions.txt
- results = utils.multi_thread_run(func=crawl.crawl_single_page, tasks=tasks)
- for result in results:
- if not result or not isinstance(result, dict):
- continue
-
- for k, v in result.items():
- if not k or not v or not isinstance(v, dict):
- continue
-
- v.update({"sub": k, "saved": True})
- materials[k] = v
+ crawled = fetch_jobs(
+ tasks,
+ (
+ ctx.crawl
+ if ctx
+ else CrawlContext(
+ mode=0, include_nodes=True, max_fails=5, exclude="", task=TaskParams(), storage=None, pushtool=None
+ )
+ ),
+ )
+ for item in crawled.items:
+ payload = deepcopy(config)
+ payload.update({"sub": item.url, "saved": True})
+ materials[item.url] = payload
# check availability
candidates = list(materials.keys())
tasks = [[x, 2, remain, life] for x in candidates]
- masks = utils.multi_thread_run(func=crawl.is_available, tasks=tasks)
+ masks = utils.multi_thread_run(func=is_available, tasks=tasks)
# filter available subscriptions
effective_subs = sorted([candidates[i] for i in range(len(masks)) if masks[i]])
@@ -253,6 +260,19 @@ def github_warp(ghproxy: str, url: str) -> str:
# save result
if effective_subs:
content = "\n".join(effective_subs)
- pushtool.push_to(content=content, config=persist, group="gitfork")
+ pushtool.push_to(content=content, item=persist, group="gitfork")
return [] if only_sublink else [materials.get(k) for k in effective_subs]
+
+
+class GitForksPlugin(ScriptPlugin[dict[str, object]]):
+ name = "gitforks"
+
+ def parse(self, ctx: PluginContext) -> dict[str, object]:
+ return plugin_params(ctx)
+
+ def run(self, config: dict[str, object], ctx: PluginContext) -> ChannelResult:
+ return as_channel_result(collect_subs(config, ctx))
+
+
+register_plugin(GitForksPlugin())
diff --git a/subscribe/scripts/scaner.py b/subscribe/crawl/channels/plugins/scaner.py
similarity index 85%
rename from subscribe/scripts/scaner.py
rename to subscribe/crawl/channels/plugins/scaner.py
index 99e99bfd76..27cbc7e1c1 100644
--- a/subscribe/scripts/scaner.py
+++ b/subscribe/crawl/channels/plugins/scaner.py
@@ -13,10 +13,15 @@
import warnings
from copy import deepcopy
-import push
import utils
import yaml
+from config.models import StorageItem
+from crawl.models import ChannelResult
from logger import logger
+from push import PushTo
+
+from .base import PluginContext, ScriptPlugin, register_plugin
+from .commons import as_channel_result, plugin_params
warnings.filterwarnings("ignore")
@@ -36,7 +41,7 @@
"""
-def convert(chars: bytes) -> list:
+def convert(chars: bytes) -> list[dict[str, object]]:
if chars is None or b"" == chars:
return []
try:
@@ -77,7 +82,7 @@ def convert(chars: bytes) -> list:
return []
-def parse_vmess(node: dict, uuid: str) -> dict:
+def parse_vmess(node: dict[str, object], uuid: str) -> dict[str, object] | None:
if not uuid:
return None
@@ -128,7 +133,7 @@ def parse_vmess(node: dict, uuid: str) -> dict:
return result
-def login(url, params, headers, retry) -> str:
+def login(url: str, params: dict[str, object], headers: dict[str, str], retry: int) -> str:
try:
data = urllib.parse.urlencode(params).encode(encoding="UTF8")
request = urllib.request.Request(url, data=data, headers=headers, method="POST")
@@ -148,7 +153,7 @@ def login(url, params, headers, retry) -> str:
return login(url, params, headers, retry) if retry > 0 else ""
-def register(url: str, params: dict, retry: int) -> bool:
+def register(url: str, params: dict[str, object], retry: int) -> bool:
try:
data = urllib.parse.urlencode(params).encode(encoding="UTF8")
request = urllib.request.Request(url, data=data, method="POST", headers=HEADER)
@@ -171,7 +176,7 @@ def register(url: str, params: dict, retry: int) -> bool:
return register(url, params, retry) if retry > 0 else False
-def get_cookie(text) -> str:
+def get_cookie(text: str | None) -> str:
regex = "(__cfduid|uid|email|key|ip|expire_in)=(.+?);"
if not text:
return ""
@@ -186,7 +191,7 @@ def fetch_nodes(
domain: str,
email: str,
passwd: str,
- headers: dict = None,
+ headers: dict[str, str] | None = None,
retry: int = 3,
subflag: bool = False,
) -> bytes:
@@ -236,7 +241,7 @@ def check(domain: str) -> bool:
return False
-def get_payload(email: str, passwd: str) -> dict:
+def get_payload(email: str, passwd: str) -> dict[str, object]:
if not email:
email = utils.random_chars(length=8, punctuation=False) + "@gmail.com"
if not passwd:
@@ -253,7 +258,7 @@ def get_payload(email: str, passwd: str) -> dict:
}
-def scanone(domain: str, email: str, passwd: str) -> list:
+def scanone(domain: str, email: str, passwd: str) -> list[dict[str, object]]:
# 获取机场所有节点信息
content = get_userinfo(domain=domain, email=email, passwd=passwd, subflag=False, verify=True)
@@ -306,7 +311,7 @@ def get_userinfo(domain: str, email: str, passwd: str, subflag: bool, verify: bo
return fetch_nodes(domain=domain, email=email, passwd=passwd, subflag=subflag)
-def filter_task(tasks: dict) -> list:
+def filter_task(tasks: dict[str, dict[str, object]]) -> list[list[str]]:
if not tasks or type(tasks) != dict:
return []
@@ -329,7 +334,7 @@ def filter_task(tasks: dict) -> list:
return configs
-def scan(params: dict) -> list:
+def scan(params: dict[str, object], ctx: PluginContext | None = None) -> list[dict[str, object]]:
if not params or type(params) != dict:
return []
@@ -339,31 +344,47 @@ def scan(params: dict) -> list:
return []
config = params.get("config", {})
- storage = params.get("storage", {})
- if not storage or type(storage) != dict:
- logger.error(f"[ScanerError] cannot scan proxies bcause storage config is invalidate")
+ if ctx is not None and not isinstance(ctx, PluginContext):
return []
-
- persist = storage.get("items", {})
- pushtool = push.get_instance(config=push.PushConfig.from_dict(storage))
-
- if not pushtool.validate(config=persist) or not config or type(config) != dict or not config.get("push_to"):
- logger.error(f"[ScanerError] cannot scan proxies bcause missing some parameters")
+ pushtool = ctx.pushtool if ctx else None
+ persist = ctx.persist if ctx else None
+ if (
+ not isinstance(pushtool, PushTo)
+ or not isinstance(persist, StorageItem)
+ or not pushtool.validate(item=persist)
+ or not config
+ or not isinstance(config, dict)
+ or not config.get("push_to")
+ ):
+ logger.error("[ScanerError] cannot scan proxies bcause missing some parameters")
return []
results = utils.multi_process_run(func=scanone, tasks=tasks)
proxies = list(itertools.chain.from_iterable(results))
if proxies:
content = yaml.dump(data={"proxies": proxies}, allow_unicode=True)
- pushtool.push_to(content=content, config=persist, group="scaner")
+ pushtool.push_to(content=content, item=persist, group="scaner")
else:
domains = ",".join(x[0] for x in tasks)
logger.info(f"[ScanerError] cannot found any proxies, domains=[{domains}]")
- config["sub"] = [pushtool.raw_url(config=persist)]
+ config["sub"] = [pushtool.raw_url(item=persist)]
config["name"] = "loophole" if not config.get("name", "") else config.get("name")
config["push_to"] = list(set(config["push_to"]))
config["saved"] = True
logger.info(f"[ScanerInfo] scan finished, found {len(proxies)} proxies")
return [config]
+
+
+class ScanerPlugin(ScriptPlugin[dict[str, object]]):
+ name = "scaner"
+
+ def parse(self, ctx: PluginContext) -> dict[str, object]:
+ return plugin_params(ctx)
+
+ def run(self, config: dict[str, object], ctx: PluginContext) -> ChannelResult:
+ return as_channel_result(scan(config, ctx))
+
+
+register_plugin(ScanerPlugin())
diff --git a/subscribe/scripts/tempairport.py b/subscribe/crawl/channels/plugins/tempairport.py
similarity index 60%
rename from subscribe/scripts/tempairport.py
rename to subscribe/crawl/channels/plugins/tempairport.py
index 76737a03fb..5aac04f769 100644
--- a/subscribe/scripts/tempairport.py
+++ b/subscribe/crawl/channels/plugins/tempairport.py
@@ -6,25 +6,34 @@
import json
from copy import deepcopy
-import push
import utils
from airport import AirPort, issspanel
-from crawl import is_available
+from config.models import NodeInput, StorageItem
+from crawl.helpers import is_available
+from crawl.models import ChannelResult
from logger import logger
+from push import PushTo
from urlvalidator import isurl
from . import commons, scaner
+from .base import PluginContext, ScriptPlugin, register_plugin
+from .commons import as_channel_result, plugin_params
def register(
- domain: str, subtype: int = 1, coupon: str = "", rigid: bool = True, chuck: bool = False, invite_code: str = ""
+ domain: str,
+ subtype: int = 1,
+ coupon: str = "",
+ allow_gmail_alias: bool = False,
+ skip_captcha: bool = False,
+ invite_code: str = "",
) -> AirPort:
url = utils.extract_domain(url=domain, include_protocal=True)
if not isurl(url=url):
logger.error(f"[TempSubError] cannot register because domain=[{domain}] is invalidate")
return None
- airport = AirPort(name=domain.split("//")[1], site=url, sub="", coupon=coupon)
+ airport = AirPort(name=domain.split("//")[1], site=url, nodes=NodeInput(), coupon=coupon)
if issspanel(domain=url):
email = utils.random_chars(length=8, punctuation=False) + "@gmail.com"
passwd = utils.random_chars(length=10, punctuation=True)
@@ -35,32 +44,39 @@ def register(
airport.username = email
airport.password = passwd
- airport.sub = suburl
+ airport.nodes.subscribe = suburl
else:
- airport.get_subscribe(retry=3, rigid=rigid, chuck=chuck, invite_code=invite_code)
+ airport.get_subscribe(
+ retry=3,
+ allow_gmail_alias=allow_gmail_alias,
+ skip_captcha=skip_captcha,
+ invite_code=invite_code,
+ )
return airport
-def fetchsub(params: dict) -> list:
+def fetchsub(params: dict[str, object], ctx: PluginContext | None = None) -> list[dict[str, object]]:
if not params or type(params) != dict:
return []
config = params.get("config", {})
- storage = params.get("storage", {})
- if not storage or type(storage) != dict:
- logger.error(f"[TempSubError] cannot fetch subscribes bcause storage config is invalidate")
+ if ctx is not None and not isinstance(ctx, PluginContext):
return []
-
- persist = storage.get("items", {})
- push_config = push.PushConfig.from_dict(storage)
-
- threshold = max(params.get("threshold", 1), 1)
- if not persist or not config or type(config) != dict or not config.get("push_to"):
- logger.error(f"[TempSubError] cannot fetch subscribes bcause not found arguments 'persist' or 'push_to'")
+ pushtool = ctx.pushtool if ctx else None
+ persist = ctx.persist if ctx else None
+ threshold = max(int(params.get("threshold", 1) or 1), 1)
+ if (
+ not isinstance(pushtool, PushTo)
+ or not isinstance(persist, StorageItem)
+ or not pushtool.validate(item=persist)
+ or not isinstance(config, dict)
+ or not config.get("push_to")
+ ):
+ logger.error("[TempSubError] cannot fetch subscribes bcause not found arguments 'persist' or 'push_to'")
return []
- exists, unregisters, unknowns, data = load(config=push_config, persist=persist, retry=params.get("retry", True))
+ exists, unregisters, unknowns, data = load(pushtool=pushtool, persist=persist, retry=params.get("retry", True))
if not exists and not unregisters and unknowns:
logger.warning(f"[TempSubError] skip fetchsub because cannot get any valid config")
return []
@@ -75,34 +91,36 @@ def fetchsub(params: dict) -> list:
if not task:
task = data.get("unknowns", {}).get(airport.ref, {})
- if not airport.available or not airport.sub:
+ subscribe = airport.nodes.subscribe_list()
+ subscribe_url = subscribe[0] if subscribe else ""
+ if not airport.available or not subscribe_url:
logger.error(
f"[TempSubInfo] cannot get subscribe because domain=[{airport.ref}] forced validation or need pay"
)
- if not utils.isblank(airport.sub):
+ if subscribe_url:
logger.warning(
- f"[TempSubInfo] renew error, domain: {airport.ref} username: {airport.username} password: {airport.password} sub: {airport.sub}"
+ f"[TempSubInfo] renew error, domain: {airport.ref} username: {airport.username} password: {airport.password} sub: {subscribe_url}"
)
- defeat = task.get("defeat", 0) + 1
+ defeat = task.get("errors", task.get("defeat", 0)) + 1
if defeat > threshold:
task["enable"] = False
- task["defeat"] = defeat
+ task["errors"] = defeat
unknowns[airport.ref] = task
else:
task.update(
{
- "sub": airport.sub,
+ "subscribe": subscribe_url,
"username": airport.username,
"password": airport.password,
- "defeat": 0,
+ "errors": 0,
}
)
exists[airport.ref] = task
# persist subscribes
payload = {"usables": exists, "unknowns": unknowns}
- commons.persist(config=push_config, data=payload, persist=persist)
+ commons.persist(pushtool=pushtool, data=payload, item=persist)
if not exists:
logger.info(f"[TempSubInfo] fetchsub finished, cannot found any subscribes")
@@ -114,7 +132,7 @@ def fetchsub(params: dict) -> list:
continue
item = deepcopy(config)
- item["sub"] = subscribe.get("sub")
+ item["sub"] = subscribe.get("subscribe", subscribe.get("sub"))
if "config" in subscribe:
item.update(subscribe.get("config"))
@@ -128,12 +146,13 @@ def fetchsub(params: dict) -> list:
return results
-def load(config: push.PushConfig, persist: dict, retry: bool = False) -> tuple[dict, list, dict, dict]:
- pushtool = push.get_instance(config=config)
- if not pushtool.validate(config=persist):
+def load(
+ pushtool: PushTo | None, persist: StorageItem | None, retry: bool = False
+) -> tuple[dict[str, object], list[object], dict[str, object], dict[str, object]]:
+ if not isinstance(pushtool, PushTo) or not isinstance(persist, StorageItem) or not pushtool.validate(item=persist):
return {}, [], {}, {}
- url = pushtool.raw_url(config=persist)
+ url = pushtool.raw_url(item=persist)
try:
content = utils.http_get(url=url)
data = json.loads(content)
@@ -153,15 +172,15 @@ def load(config: push.PushConfig, persist: dict, retry: bool = False) -> tuple[d
v = unknowns.get(k, {})
if v and v.get("enable", True):
# 包含订阅，再次检测，否则重新注册
- if not utils.isblank(v.get("sub", "")):
+ if not utils.isblank(v.get("subscribe", v.get("sub", ""))):
exists[k] = v
else:
coupon = v.get("coupon", "")
- rigid = v.get("rigid", True)
- chuck = v.get("chuck", False)
+ allow_gmail_alias = bool(v.get("allow_gmail_alias", False))
+ skip_captcha = bool(v.get("skip_captcha", False))
invite_code = v.get("invite_code", "")
- unregisters.append([k, v.get("type", 1), coupon, rigid, chuck, invite_code])
+ unregisters.append([k, v.get("type", 1), coupon, allow_gmail_alias, skip_captcha, invite_code])
unknowns.pop(k, None)
@@ -170,7 +189,7 @@ def load(config: push.PushConfig, persist: dict, retry: bool = False) -> tuple[d
if not v or not v.get("enable", True):
continue
domains.append(k)
- subscribes.append([v.get("sub", ""), 2, 0.5, 1.0])
+ subscribes.append([v.get("subscribe", v.get("sub", "")), 2, 0.5, 1.0])
if not domains:
return exists, unregisters, unknowns, rawdata
@@ -189,3 +208,16 @@ def load(config: push.PushConfig, persist: dict, retry: bool = False) -> tuple[d
return exists, unregisters, unknowns, rawdata
except:
return {}, [], {}, {}
+
+
+class TempAirportPlugin(ScriptPlugin[dict[str, object]]):
+ name = "tempairport"
+
+ def parse(self, ctx: PluginContext) -> dict[str, object]:
+ return plugin_params(ctx)
+
+ def run(self, config: dict[str, object], ctx: PluginContext) -> ChannelResult:
+ return as_channel_result(fetchsub(config, ctx))
+
+
+register_plugin(TempAirportPlugin())
diff --git a/subscribe/scripts/v2rayfree.py b/subscribe/crawl/channels/plugins/v2rayfree.py
similarity index 76%
rename from subscribe/scripts/v2rayfree.py
rename to subscribe/crawl/channels/plugins/v2rayfree.py
index 4f4890b9d3..903476b27a 100644
--- a/subscribe/scripts/v2rayfree.py
+++ b/subscribe/crawl/channels/plugins/v2rayfree.py
@@ -12,11 +12,15 @@
import urllib
import urllib.request
-import push
import utils
+from config.models import StorageItem
+from crawl.models import ChannelResult
from logger import logger
+from push import PushTo
from . import commons
+from .base import PluginContext, ScriptPlugin, register_plugin
+from .commons import as_channel_result, plugin_params
def fetch(email: str, retry: int = 2) -> str:
@@ -67,7 +71,7 @@ def fetch(email: str, retry: int = 2) -> str:
return fetch(email, retry - 1)
-def getrss(params: dict) -> list:
+def getrss(params: dict[str, object], ctx: PluginContext | None = None) -> list[dict[str, object]]:
if not params or type(params) != dict:
return []
@@ -80,23 +84,19 @@ def getrss(params: dict) -> list:
logger.error(f"[V2RayFreeError] cannot fetch subscribes bcause missing some parameters")
return []
- include = params.get("include", "").strip()
- storage = params.get("storage", {})
- if not storage or type(storage) != dict:
- logger.error(f"[V2RayFreeError] cannot fetch subscribes bcause storage config is invalidate")
+ include = str(params.get("include", "") or "").strip()
+ if ctx is not None and not isinstance(ctx, PluginContext):
return []
-
- persist = storage.get("items", {})
- push_config = push.PushConfig.from_dict(storage)
-
- exists = load(config=push_config, persist=persist)
+ pushtool = ctx.pushtool if ctx else None
+ persist = ctx.persist if ctx else None
+ exists = load(pushtool=pushtool, persist=persist)
emails = [x for x in emails if x not in exists.keys()]
results, subscribes = utils.multi_thread_run(func=fetch, tasks=emails), []
exists.update(filter(data=dict(zip(emails, results))))
# persist subscribes
- commons.persist(config=push_config, data=exists, persist=persist)
+ commons.persist(pushtool=pushtool, data=exists, item=persist)
results = list(exists.values())
results.extend(config.get("sub", []))
@@ -123,12 +123,11 @@ def getrss(params: dict) -> list:
return [config]
-def load(config: push.PushConfig, persist: dict) -> dict:
- pushtool = push.get_instance(config=config)
- if not pushtool.validate(config=persist):
+def load(pushtool: PushTo | None, persist: StorageItem | None) -> dict[str, object]:
+ if not isinstance(pushtool, PushTo) or not isinstance(persist, StorageItem) or not pushtool.validate(item=persist):
return {}
- url = pushtool.raw_url(config=persist)
+ url = pushtool.raw_url(item=persist)
try:
content = utils.http_get(url=url)
data = json.loads(content)
@@ -137,7 +136,7 @@ def load(config: push.PushConfig, persist: dict) -> dict:
return {}
-def filter(data: dict) -> dict:
+def filter(data: dict[str, object]) -> dict[str, object]:
if not data or type(data) != dict:
return {}
@@ -163,3 +162,16 @@ def check(subscribe: str) -> bool:
)
is not None
)
+
+
+class V2RayFreePlugin(ScriptPlugin[dict[str, object]]):
+ name = "v2rayfree"
+
+ def parse(self, ctx: PluginContext) -> dict[str, object]:
+ return plugin_params(ctx)
+
+ def run(self, config: dict[str, object], ctx: PluginContext) -> ChannelResult:
+ return as_channel_result(getrss(config, ctx))
+
+
+register_plugin(V2RayFreePlugin())
diff --git a/subscribe/scripts/v2rayse.py b/subscribe/crawl/channels/plugins/v2rayse.py
similarity index 87%
rename from subscribe/scripts/v2rayse.py
rename to subscribe/crawl/channels/plugins/v2rayse.py
index ad284047e0..66a4556b44 100644
--- a/subscribe/scripts/v2rayse.py
+++ b/subscribe/crawl/channels/plugins/v2rayse.py
@@ -15,19 +15,24 @@
from datetime import datetime, timedelta, timezone
from xml.etree import ElementTree
-import push
import utils
import workflow
import yaml
from airport import AirPort
-from crawl import naming_task
+from config.models import StorageItem
+from crawl.helpers import naming_task
+from crawl.models import ChannelResult
from executable import which_bin
from logger import logger
from origin import Origin
+from push import PushTo
import subconverter
from clash import QuotedStr, quoted_scalar
+from .base import PluginContext, ScriptPlugin, register_plugin
+from .commons import as_channel_result, plugin_params
+
# outbind type
SUPPORT_TYPE = [
"ss",
@@ -50,9 +55,6 @@
# last modified key name
LAST_MODIFIED = "lastModified"
-# whether enable special protocols
-SPECIAL_PROTOCOLS = AirPort.enable_special_protocols()
-
def current_time(utc: bool = True) -> datetime:
now = datetime.now(timezone.utc)
@@ -76,7 +78,7 @@ def get_dates(last: datetime) -> list[str]:
return dates
-def detect(proxies: list, nopublic: bool, exclude: str, ignore: str, repeat: int) -> bool:
+def detect(proxies: list[dict[str, object]], nopublic: bool, exclude: str, ignore: str, repeat: int) -> bool:
exclude = utils.trim(text=exclude)
ignore = utils.trim(text=ignore)
repeat = max(1, repeat)
@@ -215,7 +217,7 @@ def fetchone(
text=content,
program=subconverter,
artifact=name,
- special=SPECIAL_PROTOCOLS,
+ special=AirPort.enable_special_protocols(),
throw=True,
)
@@ -243,7 +245,7 @@ def fetchone(
return proxies, list(set(subscriptions)) if subscriptions else []
-def fetch(params: dict) -> list:
+def fetch(params: dict[str, object], ctx: PluginContext | None = None) -> list[dict[str, object]]:
if not params or type(params) != dict:
return []
@@ -252,12 +254,18 @@ def fetch(params: dict) -> list:
logger.error(f"[V2RaySE] skip collect data due to parameter 'url' missing")
return []
- storage = params.get("storage", {})
- pushtool = push.get_instance(config=push.PushConfig.from_dict(storage))
-
- persist = storage.get("items", {})
- if not persist or type(persist) != dict or not pushtool.validate(config=persist.get("proxies", {})):
- logger.error(f"[V2RaySE] invalid persist config, please check it and try again")
+ if ctx is not None and not isinstance(ctx, PluginContext):
+ return []
+ pushtool = ctx.pushtool if ctx else None
+ persist = ctx.persist if ctx and isinstance(ctx.persist, dict) else {}
+ proxies_store = persist.get("proxies")
+ modified_store = persist.get("modified")
+ if (
+ not isinstance(pushtool, PushTo)
+ or not isinstance(proxies_store, StorageItem)
+ or not pushtool.validate(item=proxies_store)
+ ):
+ logger.error("[V2RaySE] invalid persist config, please check it and try again")
return []
nopublic = params.get("nopublic", True)
@@ -271,11 +279,7 @@ def fetch(params: dict) -> list:
mixed = utils.trim(params.get("format", "clash")).lower() != "clash"
display = params.get("display", False)
- # storage config
- proxies_store = persist.get("proxies", {})
- modified_store = persist.get("modified", {})
-
- history_url = pushtool.raw_url(config=modified_store)
+ history_url = pushtool.raw_url(item=modified_store) if modified_store else ""
last = last_history(url=history_url, interval=interval)
dates, manual = params.get("dates", []), True
@@ -367,20 +371,33 @@ def fetch(params: dict) -> list:
filename = os.path.join(os.path.dirname(datapath), "data", "v2rayse.txt")
utils.write_file(filename=filename, lines=content)
- success = pushtool.push_to(content=content or " ", config=proxies_store, group="v2rayse")
+ success = pushtool.push_to(content=content or " ", item=proxies_store, group="v2rayse")
if not success:
return tasks
# save last modified time
- if not manual and pushtool.validate(config=modified_store):
+ if not manual and pushtool.validate(item=modified_store):
content = json.dumps({LAST_MODIFIED: begin})
- pushtool.push_to(content=content, config=modified_store, group="modified")
+ pushtool.push_to(content=content, item=modified_store, group="modified")
config = params.get("config", {})
- config["sub"] = pushtool.raw_url(config=proxies_store)
+ config["sub"] = pushtool.raw_url(item=proxies_store)
config["saved"] = True
config["name"] = "v2rayse" if not config.get("name", "") else config.get("name")
config["push_to"] = list(set(config.get("push_to", [])))
tasks.append(config)
return tasks
+
+
+class V2RaySEPlugin(ScriptPlugin[dict[str, object]]):
+ name = "v2rayse"
+
+ def parse(self, ctx: PluginContext) -> dict[str, object]:
+ return plugin_params(ctx)
+
+ def run(self, config: dict[str, object], ctx: PluginContext) -> ChannelResult:
+ return as_channel_result(fetch(config, ctx))
+
+
+register_plugin(V2RaySEPlugin())
diff --git a/subscribe/crawl/channels/repository.py b/subscribe/crawl/channels/repository.py
new file mode 100644
index 0000000000..19740a90bd
--- /dev/null
+++ b/subscribe/crawl/channels/repository.py
@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*-
+
+import json
+import time
+
+import utils
+from config.models import RepoConfig, TaskParams
+from crawl.base import Channel, register_channel
+from crawl.extract import extract_subscribes
+from crawl.helpers import merge_results
+from crawl.models import ChannelResult, CrawlContext
+from logger import logger
+from origin import Origin
+
+
+class RepositoryChannel(Channel[list[RepoConfig]]):
+ name = "repositories"
+
+ def crawl(self, config: list[RepoConfig], ctx: CrawlContext) -> ChannelResult:
+ return crawl_github_repo(config, ctx)
+
+
+def crawl_single_repo(
+ username: str, repo: str, push_to: list[str], commits: int, exclude: str, include_nodes: bool
+) -> ChannelResult:
+ if not username or not repo:
+ logger.error(f"cannot crawl from github, username: {username}\trepo: {repo}")
+ return ChannelResult()
+
+ commits = max(1, commits)
+ url = f"https://api.github.com/repos/{username.strip()}/{repo.strip()}/commits?per_page={commits}"
+ content = utils.http_get(url=url)
+ if not content:
+ return ChannelResult()
+
+ result = ChannelResult()
+ task = TaskParams(push_to=list(push_to))
+ try:
+ records = json.loads(content)
+ for item in records:
+ payload = utils.http_get(url=item.get("url", ""))
+ if not payload:
+ continue
+ commit = json.loads(payload)
+ for file in commit.get("files", []):
+ result.merge(
+ extract_subscribes(
+ content=file.get("patch", ""),
+ push_to=push_to,
+ source=Origin.REPO.name,
+ exclude=exclude,
+ task=task,
+ include_nodes=include_nodes,
+ )
+ )
+ return result
+ except Exception:
+ logger.error(f"[GithubCrawl] crawl from github error, username: {username}\trepo: {repo}")
+ return ChannelResult()
+
+
+def crawl_github_repo(repos: list[RepoConfig], ctx: CrawlContext) -> ChannelResult:
+ if not repos:
+ return ChannelResult()
+
+ starttime = time.time()
+ params = []
+ for item in repos:
+ if not item.enable:
+ continue
+ username = utils.trim(item.username)
+ repo = utils.trim(item.repo)
+ if not username or not repo or not item.push_to:
+ continue
+ params.append([username, repo, item.push_to, max(item.commits, 1), item.exclude, ctx.include_nodes])
+
+ results = utils.multi_thread_run(func=crawl_single_repo, tasks=params, num_threads=ctx.num_threads)
+ result = merge_results(results)
+ logger.info(
+ f"[RepoCrawl] finished crawl from Repositorie, found {len(result.items)} subscriptions, cost: {time.time() - starttime:.2f}s"
+ )
+ return result
+
+
+register_channel(RepositoryChannel())
diff --git a/subscribe/crawl/channels/script.py b/subscribe/crawl/channels/script.py
new file mode 100644
index 0000000000..a900803968
--- /dev/null
+++ b/subscribe/crawl/channels/script.py
@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+
+from config.models import ScriptJob
+from crawl.base import Channel, register_channel
+from crawl.channels.plugins import PLUGINS, PluginContext
+from crawl.models import ChannelResult, CrawlContext
+from logger import logger
+
+
+class ScriptChannel(Channel[list[ScriptJob]]):
+ name = "scripts"
+
+ def crawl(self, config: list[ScriptJob], ctx: CrawlContext) -> ChannelResult:
+ merged = ChannelResult()
+
+ if not isinstance(ctx, CrawlContext):
+ return merged
+ storage_items = ctx.storage.items if ctx.storage is not None else {}
+ for job in config:
+ if not isinstance(job, ScriptJob) or not job.enable:
+ continue
+ plugin = PLUGINS.get(job.plugin)
+ if plugin is None:
+ continue
+ persist = None
+ if isinstance(job.persist, str):
+ persist = storage_items.get(job.persist)
+ elif isinstance(job.persist, dict):
+ resolved = {}
+ for key, value in job.persist.items():
+ resolved[key] = storage_items.get(value) if isinstance(value, str) else value
+ persist = resolved
+ pctx = PluginContext(
+ crawl=ctx,
+ params=job.options,
+ task=job.task,
+ persist=persist,
+ pushtool=ctx.pushtool,
+ storage_items=storage_items,
+ )
+ try:
+ parsed = plugin.parse(pctx)
+ merged.merge(plugin.run(parsed, pctx))
+ except Exception as exc:
+ logger.error(f"[ScriptError] plugin {job.plugin} failed: {exc}")
+ return merged
+
+
+register_channel(ScriptChannel())
diff --git a/subscribe/crawl/channels/telegram.py b/subscribe/crawl/channels/telegram.py
new file mode 100644
index 0000000000..a6c5868759
--- /dev/null
+++ b/subscribe/crawl/channels/telegram.py
@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+
+import itertools
+import re
+import time
+from dataclasses import replace
+
+import utils
+from config.models import TelegramChannelConfig, TelegramConfig
+from crawl.base import Channel, register_channel
+from crawl.extract import extract_subscribes
+from crawl.helpers import is_reachable, merge_results
+from crawl.models import ChannelResult, CrawlContext
+from logger import logger
+from origin import Origin
+
+
+class TelegramChannel(Channel[TelegramConfig]):
+ name = "telegram"
+
+ def crawl(self, config: TelegramConfig, ctx: CrawlContext) -> ChannelResult:
+ if not is_reachable("https://t.me"):
+ logger.warning("[TelegramCrawl] skip because telegram is unreachable")
+ return ChannelResult()
+ return crawl_telegram(config, ctx)
+
+
+def crawl_telegram(config: TelegramConfig, ctx: CrawlContext) -> ChannelResult:
+ starttime = time.time()
+ params = []
+ for name, item in config.channels.items():
+ if not item.push_to:
+ continue
+ exclude = item.exclude
+ if config.exclude:
+ exclude = f"{exclude}|{config.exclude}".removeprefix("|")
+ if exclude != item.exclude:
+ item = replace(item, exclude=exclude)
+ params.append([name, item, config.pages])
+ page_groups = utils.multi_thread_run(func=_telegram_pages, tasks=params, num_threads=ctx.num_threads)
+ tasks = list(itertools.chain.from_iterable(page_groups))
+ jobs = [[url, item, ctx.include_nodes] for url, item in tasks]
+ results = utils.multi_thread_run(func=_crawl_telegram_page, tasks=jobs, num_threads=ctx.num_threads)
+ result = merge_results(results)
+ logger.info(
+ f"[TelegramCrawl] finished crawl from Telegram, found {len(result.items)} subscriptions, cost: {time.time() - starttime:.2f}s"
+ )
+ return result
+
+
+def _telegram_pages(channel: str, config: TelegramChannelConfig, pages: int) -> list[list]:
+ if pages <= 1:
+ return [[f"https://t.me/s/{channel}", config]]
+
+ count = get_telegram_pages(channel=channel)
+ if count == 0:
+ return []
+
+ arrays = range(count, -1, -100)
+ pages = min(pages, len(arrays))
+ return [[f"https://t.me/s/{channel}?before={item}", config] for item in arrays[:pages]]
+
+
+def get_telegram_pages(channel: str) -> int:
+ if not channel or not channel.strip():
+ return 0
+
+ url = f"https://t.me/s/{channel}"
+ content = utils.http_get(url=url)
+ before = 0
+ try:
+ regex = rf''
+ groups = re.findall(regex, content)
+ before = int(groups[0]) if groups else before
+ except Exception:
+ logger.error(f"[CrawlError] cannot count page num, chanel: {channel}")
+ return before
+
+
+def _crawl_telegram_page(url: str, config: TelegramChannelConfig, include_nodes: bool) -> ChannelResult:
+ if not url or not config.push_to:
+ return ChannelResult()
+ content = utils.http_get(url=url)
+ if not content:
+ return ChannelResult()
+ return extract_subscribes(
+ content=content,
+ push_to=config.push_to,
+ include=config.include,
+ exclude=config.exclude,
+ source=Origin.TELEGRAM.name,
+ task=config.task,
+ reversed=True,
+ include_nodes=include_nodes,
+ )
+
+
+register_channel(TelegramChannel())
diff --git a/subscribe/crawl/channels/twitter.py b/subscribe/crawl/channels/twitter.py
new file mode 100644
index 0000000000..06707da29b
--- /dev/null
+++ b/subscribe/crawl/channels/twitter.py
@@ -0,0 +1,188 @@
+# -*- coding: utf-8 -*-
+
+import json
+import re
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+
+import utils
+from config.models import TwitterConfig
+from crawl.base import Channel, register_channel
+from crawl.channels.page import PageChannel
+from crawl.helpers import fetch_jobs, is_reachable
+from crawl.models import ChannelResult, CrawlContext
+from logger import logger
+from origin import Origin
+
+
+class TwitterChannel(Channel[TwitterConfig]):
+ name = "twitter"
+
+ def crawl(self, config: TwitterConfig, ctx: CrawlContext) -> ChannelResult:
+ if not is_reachable("https://twitter.com"):
+ logger.warning("[TwitterCrawl] skip because twitter is unreachable")
+ return ChannelResult()
+ return crawl_twitter(config, ctx)
+
+
+def extract_twitter_cookies(retry: int = 2) -> str:
+ if retry <= 0:
+ return ""
+
+ headers = None
+ try:
+ request = urllib.request.Request(url="https://twitter.com/", headers=utils.DEFAULT_HTTP_HEADERS)
+ response = urllib.request.urlopen(request, timeout=10, context=utils.CTX)
+ headers = response.headers
+ except urllib.error.HTTPError as exc:
+ if exc.code != 302:
+ return extract_twitter_cookies(retry=retry - 1)
+ headers = exc.headers
+ except (urllib.error.URLError, TimeoutError):
+ return extract_twitter_cookies(retry=retry - 1)
+
+ if not headers or "set-cookie" not in headers:
+ return ""
+
+ regex = "(guest_id|guest_id_ads|guest_id_marketing|personalization_id)=(.+?);"
+ content = ";".join(headers.get_all("set-cookie", ""))
+ groups = re.findall(regex, content, flags=re.I)
+ return ";".join(["=".join(item) for item in groups]).strip()
+
+
+def get_guest_token() -> str:
+ cookies = extract_twitter_cookies(retry=3)
+ if not cookies:
+ logger.error("[TwitterCrawl] cannot extract Twitter cookies")
+ return ""
+
+ headers = {
+ "User-Agent": utils.USER_AGENT,
+ "Cookie": cookies,
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+ }
+ content = utils.http_get(url="https://twitter.com/", headers=headers)
+ if not content:
+ return ""
+ matcher = re.findall("gt=([0-9]{19})", content, flags=re.I)
+ return matcher[0] if matcher else ""
+
+
+def username_to_id(username: str, headers: dict[str, str]) -> str:
+ if utils.isblank(username):
+ return ""
+
+ if not headers or "X-Guest-Token" not in headers:
+ guest_token = get_guest_token()
+ if not guest_token:
+ return ""
+ headers = {
+ "User-Agent": utils.USER_AGENT,
+ "Authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA",
+ "X-Guest-Token": guest_token,
+ "Content-Type": "application/json",
+ }
+
+ variables = {"screen_name": username.lower().strip(), "withSafetyModeUserFields": True}
+ features = {
+ "blue_business_profile_image_shape_enabled": True,
+ "responsive_web_graphql_exclude_directive_enabled": True,
+ "verified_phone_label_enabled": False,
+ "responsive_web_graphql_skip_user_profile_image_extensions_enabled": False,
+ "responsive_web_graphql_timeline_navigation_enabled": True,
+ }
+ payload = urllib.parse.urlencode({"variables": json.dumps(variables), "features": json.dumps(features)})
+ url = f"https://twitter.com/i/api/graphql/sLVLhk0bGj3MVFEKTdax1w/UserByScreenName?{payload}"
+ try:
+ content = utils.http_get(url=url, headers=headers)
+ if not content:
+ return ""
+ data = json.loads(content).get("data", {}).get("user", {}).get("result", "")
+ return data.get("rest_id", "")
+ except Exception:
+ logger.error(f"[TwitterCrawl] cannot query uid by username=[{username}]")
+ return ""
+
+
+def crawl_twitter(config: TwitterConfig, ctx: CrawlContext) -> ChannelResult:
+ if not config or not config.users:
+ return ChannelResult()
+
+ guest_token, starttime = get_guest_token(), time.time()
+ if not guest_token:
+ logger.error("[TwitterCrawl] cannot extract X-Guest-Token from twitter")
+ return ChannelResult()
+
+ headers = {
+ "User-Agent": utils.USER_AGENT,
+ "Authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA",
+ "X-Guest-Token": guest_token,
+ "Content-Type": "application/json",
+ }
+ features = {
+ "blue_business_profile_image_shape_enabled": True,
+ "responsive_web_graphql_exclude_directive_enabled": True,
+ "verified_phone_label_enabled": False,
+ "responsive_web_graphql_timeline_navigation_enabled": True,
+ "responsive_web_graphql_skip_user_profile_image_extensions_enabled": False,
+ "tweetypie_unmention_optimization_enabled": True,
+ "vibe_api_enabled": True,
+ "responsive_web_edit_tweet_api_enabled": True,
+ "graphql_is_translatable_rweb_tweet_is_translatable_enabled": True,
+ "view_counts_everywhere_api_enabled": True,
+ "longform_notetweets_consumption_enabled": True,
+ "tweet_awards_web_tipping_enabled": False,
+ "freedom_of_speech_not_reach_fetch_enabled": True,
+ "standardized_nudges_misinfo": True,
+ "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": False,
+ "interactive_text_enabled": True,
+ "responsive_web_text_conversations_enabled": False,
+ "longform_notetweets_rich_text_read_enabled": True,
+ "responsive_web_enhance_cards_enabled": False,
+ }
+
+ candidates = {key: value for key, value in config.users.items() if key and value.enable and value.push_to}
+ if not candidates:
+ return ChannelResult()
+
+ params = [[key, headers] for key in candidates.keys()]
+ uids = utils.multi_thread_run(func=username_to_id, tasks=params, num_threads=ctx.num_threads)
+ jobs = []
+ for index, uid in enumerate(uids):
+ if not uid:
+ continue
+ config = candidates.get(params[index][0], {})
+ count = config.tweets
+ variables = {
+ "userId": uid,
+ "count": min(max(count, 1), 100),
+ "includePromotedContent": False,
+ "withClientEventToken": False,
+ "withBirdwatchNotes": False,
+ "withVoice": True,
+ "withV2Timeline": True,
+ }
+ payload = urllib.parse.urlencode({"variables": json.dumps(variables), "features": json.dumps(features)})
+ url = f"https://twitter.com/i/api/graphql/P7qs2Sf7vu1LDKbzDW9FSA/UserMedia?{payload}"
+ jobs.append(
+ PageChannel(
+ url=url,
+ include=config.include,
+ exclude=config.exclude,
+ push_to=config.push_to,
+ task=config.task,
+ headers=headers,
+ origin=Origin.TWITTER.name,
+ )
+ )
+
+ result = fetch_jobs(jobs, ctx)
+ logger.info(
+ f"[TwitterCrawl] finished crawl from Twitter, found {len(result.items)} subscriptions, cost: {time.time() - starttime:.2f}s"
+ )
+ return result
+
+
+register_channel(TwitterChannel())
diff --git a/subscribe/crawl/channels/yandex.py b/subscribe/crawl/channels/yandex.py
new file mode 100644
index 0000000000..89fa2826b0
--- /dev/null
+++ b/subscribe/crawl/channels/yandex.py
@@ -0,0 +1,93 @@
+# -*- coding: utf-8 -*-
+
+import re
+import time
+import urllib.parse
+
+import utils
+from config.models import TaskParams, YandexConfig
+from crawl.base import Channel, register_channel
+from crawl.helpers import is_reachable
+from crawl.models import ChannelResult, CrawlContext
+from logger import logger
+from origin import Origin
+
+
+class YandexChannel(Channel[YandexConfig]):
+ name = "yandex"
+
+ def crawl(self, config: YandexConfig, ctx: CrawlContext) -> ChannelResult:
+ if not is_reachable("https://yandex.com"):
+ logger.warning("[YandexCrawl] skip because yandex is unreachable")
+ return ChannelResult()
+ return crawl_yandex(config)
+
+
+def crawl_yandex(config: YandexConfig) -> ChannelResult:
+ reject, query = "", urllib.parse.quote("/api/v1/client/subscribe?token=")
+ if config.exclude_sites:
+ items = list(set([re.escape(utils.trim(item).lower()) for item in config.exclude_sites if utils.trim(item)]))
+ reject = "|".join(items)
+
+ url = f'https://yandex.com/search/?text="{query}"&lr=10599&cee=1'
+ if config.days > 0:
+ url = f"{url}&within={config.days}"
+
+ starttime = time.time()
+ headers = {
+ "User-Agent": utils.USER_AGENT,
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+ "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+ "Accept-Encoding": "gzip",
+ }
+
+ content = utils.http_get(url=url, headers=headers)
+ pages = max(1, config.pages)
+ if content:
+ regex = r'(\d+)'
+ groups = re.findall(regex, content, flags=re.I)
+ if groups:
+ pages = min(pages, max([int(item) for item in groups]))
+
+ task = TaskParams(push_to=list(config.push_to))
+ result = ChannelResult()
+ for page in range(0, pages):
+ content = utils.http_get(url=f"{url}&p={page}", headers=headers)
+ if not content:
+ logger.error(f"[YandexCrawl] cannot get content from page: {page}")
+ continue
+
+ groups = re.findall(r"([\s\S]*?) ", content)
+ if not groups:
+ logger.error(f"[YandexCrawl] cannot get any search result from page: {page}")
+ continue
+
+ for group in groups:
+ try:
+ if reject:
+ regex = r''
+ link = re.findall(regex, group, flags=re.I)[0]
+ if re.search(reject, link):
+ continue
+ except Exception:
+ logger.error(f"[YandexCrawl] invalid regex pattern: {reject}")
+ continue
+
+ regex = r"https?://(?:[a-zA-Z0-9_\u4e00-\u9fa5\-]+\.)+[a-zA-Z0-9_\u4e00-\u9fa5\-]+(?::\d+)?/api/v1/client/subscribe\?token=[a-zA-Z0-9]{16,32}"
+ links = re.findall(regex, group, flags=re.I)
+ for link in links:
+ try:
+ link = re.sub(r"|", "", link).replace("http://", "https://")
+ if config.exclude and re.search(config.exclude, link):
+ continue
+ result.add_subscribe(link, Origin.YANDEX.name, task)
+ except Exception:
+ continue
+
+ logger.info(
+ f"[YandexCrawl] finished crawl from Yandex, found {len(result.items)} subscriptions, cost: {time.time() - starttime:.2f}s"
+ )
+ return result
+
+
+register_channel(YandexChannel())
diff --git a/subscribe/crawl/engine.py b/subscribe/crawl/engine.py
new file mode 100644
index 0000000000..3fdcf76c7c
--- /dev/null
+++ b/subscribe/crawl/engine.py
@@ -0,0 +1,306 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+import base64
+import re
+
+import push
+import utils
+from config.models import (
+ CrawlConfig,
+ NodeInput,
+ SiteConfig,
+ StorageConfig,
+ StorageItem,
+ TaskParams,
+)
+from crawl.channels import CHANNELS
+from crawl.helpers import check_status, load_records, naming_task, save_records
+from crawl.models import ChannelResult, CrawlContext, SubItem
+from logger import logger
+from origin import Origin
+from push import PushTo
+from workflow import standard_sub
+
+
+def _site_from_item(item: SubItem, shared: TaskParams) -> SiteConfig:
+ task = shared.merge(item.task) if item.task else shared
+ site = SiteConfig(
+ name=item.name or naming_task(item.url),
+ nodes=NodeInput(subscribe=item.url),
+ push_to=list(task.push_to or []),
+ rename=task.rename or "",
+ include=task.include or "",
+ exclude=task.exclude or "",
+ ignore_default_exclude=bool(task.ignore_default_exclude) if task.ignore_default_exclude is not None else False,
+ check_alive=True if task.check_alive is None else task.check_alive,
+ max_rate=task.max_rate if task.max_rate is not None else 3.0,
+ require_tls=bool(task.require_tls),
+ origin=item.origin or Origin.TEMPORARY.name,
+ errors=item.errors,
+ debut=True if item.debut or item.ready else False,
+ skip_cache=item.skip_cache,
+ allow_nonstandard=item.allow_nonstandard,
+ persist_only=item.persist_only,
+ discovered=item.discovered or item.ready,
+ )
+ return site
+
+
+def _site_from_nodes(name: str, nodes: NodeInput, origin: str, task: TaskParams) -> SiteConfig:
+ return SiteConfig(
+ name=name or origin.lower(),
+ nodes=NodeInput(subscribe=list(nodes.subscribe_list()), uris=list(nodes.uris), proxies=list(nodes.proxies)),
+ push_to=list(task.push_to or []),
+ rename=task.rename or "",
+ include=task.include or "",
+ exclude=task.exclude or "",
+ ignore_default_exclude=bool(task.ignore_default_exclude) if task.ignore_default_exclude is not None else False,
+ check_alive=True if task.check_alive is None else task.check_alive,
+ max_rate=task.max_rate if task.max_rate is not None else 3.0,
+ require_tls=bool(task.require_tls),
+ origin=origin,
+ debut=True,
+ )
+
+
+def _usable_subscribe(url: str, exclude: str = "") -> bool:
+ url = utils.trim(url)
+ if not url:
+ return False
+ if utils.is_suspicious_url(url):
+ return False
+ exclude = utils.trim(exclude)
+ if exclude:
+ try:
+ if re.search(exclude, url):
+ return False
+ except Exception:
+ pass
+ return True
+
+
+def _collapse_items(items: list[SubItem] | None) -> list[SubItem]:
+ merged: dict[str, SubItem] = {}
+ for item in items or []:
+ url = utils.trim(item.url)
+ if not url:
+ continue
+ item.url = url
+ current = merged.get(url)
+ if current is None:
+ merged[url] = item
+ continue
+ winner, loser = current, item
+ if item.ready and not current.ready:
+ winner, loser = item, current
+ elif (not item.persist_only) and current.persist_only and item.ready == current.ready:
+ winner, loser = item, current
+ winner.errors = max(winner.errors, loser.errors)
+ winner.allow_nonstandard = winner.allow_nonstandard or loser.allow_nonstandard
+ winner.skip_cache = winner.skip_cache or loser.skip_cache
+ if loser.discovered:
+ winner.discovered = True
+ merged[url] = winner
+ return list(merged.values())
+
+
+def _unpack_status(mask: object) -> tuple[bool, bool]:
+ if isinstance(mask, tuple) and len(mask) >= 2:
+ return bool(mask[0]), bool(mask[1])
+ return False, False
+
+
+def run(
+ config: CrawlConfig,
+ storage: StorageConfig | None = None,
+ num_threads: int = 50,
+ display: bool = True,
+ mode: int = 0,
+) -> list[SiteConfig]:
+ mode = 0 if not isinstance(mode, int) else min(max(mode, 0), 2)
+ if not isinstance(config, CrawlConfig) or not config.enable:
+ return []
+
+ pushtool = None
+ try:
+ if isinstance(storage, StorageConfig):
+ pushtool = push.get_instance(storage)
+ except Exception:
+ pushtool = None
+
+ persist_subscribe = storage.items.get(config.persist.subscribe) if storage and config.persist.subscribe else None
+ persist_nodes = storage.items.get(config.persist.nodes) if storage and config.persist.nodes else None
+ if mode == 1 and not (pushtool and persist_subscribe):
+ logger.warning(
+ "[CrawlWarn] skip crawling tasks because the mode is set to crawl only but no valid persistence configuration is set"
+ )
+ return []
+
+ ctx = CrawlContext(
+ mode=mode,
+ include_nodes=config.include_nodes,
+ max_fails=config.max_fails,
+ exclude=config.exclude,
+ task=config.task,
+ storage=storage,
+ pushtool=pushtool,
+ num_threads=num_threads,
+ display=display,
+ )
+
+ result = ChannelResult()
+ if mode != 2:
+ sections = [
+ ("google", config.google),
+ ("yandex", config.yandex),
+ ("telegram", config.telegram),
+ ("twitter", config.twitter),
+ ("github", config.github),
+ ("repositories", config.repositories),
+ ("pages", config.pages),
+ ("scripts", config.scripts),
+ ]
+ for name, section in sections:
+ if section is None:
+ continue
+ if getattr(section, "enable", True) is False:
+ continue
+ channel = CHANNELS.get(name)
+ if channel is None:
+ continue
+ try:
+ result.merge(channel.crawl(section, ctx))
+ except Exception as exc:
+ logger.error(f"[CrawlError] crawl channel {name} failed: {exc}")
+
+ sites: list[SiteConfig] = []
+ cached: dict[str, dict[str, object]] = {}
+ if pushtool and persist_subscribe:
+ cached = load_records(pushtool, persist_subscribe) or {}
+
+ result.items = _collapse_items(result.items)
+ known = {item.url for item in result.items if item.url}
+ for item in result.items:
+ meta = cached.get(item.url)
+ if not isinstance(meta, dict) or item.ready:
+ continue
+ item.errors = max(item.errors, int(meta.get("errors", 0) or 0))
+
+ stale_keys: list[str] = []
+ for raw_url, meta in cached.items():
+ url = utils.trim(raw_url)
+ if not url or not isinstance(meta, dict):
+ stale_keys.append(raw_url)
+ continue
+ if url in known:
+ continue
+ if not _usable_subscribe(url, config.exclude):
+ stale_keys.append(raw_url)
+ continue
+ result.add_subscribe(
+ url,
+ meta.get("origin", Origin.TEMPORARY.name),
+ TaskParams(push_to=list(meta.get("push_to", []))),
+ skip_cache=bool(meta.get("skip_cache", False)),
+ allow_nonstandard=bool(meta.get("allow_nonstandard", False)),
+ errors=int(meta.get("errors", 0) or 0),
+ discovered=True,
+ debut=bool(meta.get("debut", False)),
+ )
+ known.add(url)
+ for key in stale_keys:
+ cached.pop(key, None)
+
+ pending: list[SubItem] = []
+ seen: set[str] = set()
+ for item in result.items:
+ url = utils.trim(item.url)
+ if not url or url in seen:
+ continue
+ if not _usable_subscribe(url, config.exclude):
+ continue
+ seen.add(url)
+ item.url = url
+ site = _site_from_item(item, config.task)
+ if item.persist_only:
+ cached[item.url] = _cache_record(site)
+ continue
+ if item.ready:
+ cached[item.url] = _cache_record(site)
+ sites.append(site)
+ continue
+ pending.append(item)
+
+ if pending:
+ logger.info(f"[CrawlInfo] start to validate {len(pending)} subscriptions")
+ masks = utils.multi_thread_run(
+ func=check_status,
+ tasks=[[item.url, 2, 5, 12, 72] for item in pending],
+ num_threads=ctx.num_threads,
+ show_progress=ctx.display,
+ )
+ for item, mask in zip(pending, masks):
+ available, expired = _unpack_status(mask)
+ site = _site_from_item(item, config.task)
+ if not available:
+ errors = int(cached.get(item.url, {}).get("errors", item.errors) or 0) + 1
+ site.errors = errors
+ if errors > config.max_fails or expired:
+ continue
+ if not item.allow_nonstandard and not standard_sub(item.url) and mode != 1:
+ continue
+ cached[item.url] = _cache_record(site)
+ continue
+ site.errors = 0
+ site.debut = True
+ cached[item.url] = _cache_record(site)
+ sites.append(site)
+
+ extra = NodeInput(uris=list(result.nodes.uris), proxies=list(result.nodes.proxies))
+ if not config.include_nodes:
+ extra.uris = []
+ if not extra.empty():
+ sites.append(_site_from_nodes("crawled-nodes", extra, Origin.TEMPORARY.name, config.task))
+ _snapshot_nodes(pushtool, persist_nodes, extra)
+
+ if pushtool and persist_subscribe and cached:
+ survivors = {key: value for key, value in cached.items() if not value.get("skip_cache")}
+ if survivors:
+ save_records(pushtool, persist_subscribe, survivors)
+
+ if mode == 1:
+ logger.warning("[CrawlWarn] skip aggregate because mode=1 represents only crawling subscriptions")
+
+ logger.info(f"[CrawlInfo] crawl finished, found {len(sites)} sites")
+ return sites
+
+
+def _cache_record(site: SiteConfig) -> dict[str, object]:
+ return {
+ "origin": site.origin,
+ "push_to": site.push_to,
+ "errors": site.errors,
+ "discovered": True,
+ "skip_cache": site.skip_cache,
+ "allow_nonstandard": site.allow_nonstandard,
+ "debut": site.debut,
+ }
+
+
+def _snapshot_nodes(pushtool: PushTo | None, item: StorageItem | None, extra: NodeInput) -> None:
+ if not isinstance(pushtool, PushTo) or not isinstance(item, StorageItem):
+ return
+ try:
+ if extra.uris:
+ content = base64.b64encode("\n".join(extra.uris).encode()).decode()
+ elif extra.proxies:
+ import yaml
+
+ content = yaml.dump({"proxies": extra.proxies}, allow_unicode=True)
+ else:
+ return
+ pushtool.push_to(content=content, item=item, group="proxies")
+ except Exception:
+ logger.error("[CrawlError] persist crawled nodes failed")
diff --git a/subscribe/crawl/extract.py b/subscribe/crawl/extract.py
new file mode 100644
index 0000000000..03ca7f3204
--- /dev/null
+++ b/subscribe/crawl/extract.py
@@ -0,0 +1,135 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+import re
+import sys
+import urllib.parse
+
+import utils
+from config.models import NodeInput, TaskParams
+from crawl.models import ChannelResult, SubItem
+from logger import logger
+from origin import Origin
+
+SUB_REGEX = r"https?://(?:[a-zA-Z0-9\u4e00-\u9fa5\-]+\.)+[a-zA-Z0-9\u4e00-\u9fa5\-]+(?::\d+)?(?:(?:(?:/index.php)?/api/v1/client/subscribe\?token=[a-zA-Z0-9]{16,32})|(?:/link/[a-zA-Z0-9]+\?(?:sub|mu|clash)=\d)|(?:/(?:s|sub)/[a-zA-Z0-9]{32}))|https://jmssub\.net/members/getsub\.php\?service=\d+&id=[a-zA-Z0-9\-]{36}(?:\S+)?"
+EXTRA_REGEX = r"https?://(?:[a-zA-Z0-9\u4e00-\u9fa5\-]+\.)+[a-zA-Z0-9\u4e00-\u9fa5\-]+/sub\?(?:\S+)?target=\S+"
+PROTOCOL_REGEX = r"(?:vmess|trojan|ss|ssr|snell|hysteria2|vless|hysteria|tuic|anytls)://[a-zA-Z0-9:.?+=@%&#_\-/]{10,}"
+
+
+def extract_subscribes(
+ content: str,
+ push_to: list[str] | None = None,
+ include: str = "",
+ exclude: str = "",
+ limits: int = sys.maxsize,
+ source: str = Origin.OWNED.name,
+ task: TaskParams | None = None,
+ reversed: bool = False,
+ skip_cache: bool = False,
+ include_nodes: bool = False,
+) -> ChannelResult:
+ result = ChannelResult()
+ if not content:
+ return result
+
+ push_to = list(push_to or [])
+ task = task or TaskParams(push_to=push_to)
+ if task.push_to is None:
+ task.push_to = push_to
+
+ try:
+ limits = max(1, limits)
+ pattern = f"{SUB_REGEX}|{EXTRA_REGEX}"
+ if include:
+ try:
+ pattern = f"{pattern}|{include}" if include.startswith("|") else f"{pattern}|{include}"
+ if not include.startswith("|"):
+ pattern = f"{SUB_REGEX}|{EXTRA_REGEX}|{include}"
+ else:
+ pattern = f"{SUB_REGEX}|{EXTRA_REGEX}{include}"
+ found = re.findall(pattern, content, flags=re.I)
+ except Exception:
+ logger.error(f"[ExtractError] maybe pattern include exists some problems, include: {include}")
+ found = re.findall(f"{SUB_REGEX}|{EXTRA_REGEX}", content)
+ else:
+ found = re.findall(f"{SUB_REGEX}|{EXTRA_REGEX}", content, flags=re.I)
+
+ try:
+ parts = re.findall(
+ r"(?m)^#(?:\s+)?(?:!MANAGED-CONFIG|订阅链接)[^\n]*?(https?://[^\s\"'<>]+)",
+ content,
+ flags=re.I,
+ )
+ if parts:
+ found.extend([utils.trim(item) for item in parts])
+ except Exception:
+ pass
+
+ if reversed:
+ found.reverse()
+
+ seen = []
+ uris = []
+ for sub in found:
+ items = [sub]
+ if "url=" in sub:
+ query = urllib.parse.urlparse(sub.replace("&", "&")).query
+ items = []
+ urls = urllib.parse.parse_qs(query).get("url", [])
+ if not urls:
+ continue
+ for url in urls:
+ if not utils.isurl(url):
+ if include_nodes:
+ uris.extend([x for x in url.split("|") if re.match(PROTOCOL_REGEX, x, flags=re.I)])
+ continue
+ items.extend([x for x in url.split("|") if not re.match(EXTRA_REGEX, x, flags=re.I)])
+
+ for link in items:
+ link = re.sub(r"\\/|\/", "/", link, flags=re.I)
+ try:
+ if include and not re.match(
+ r"https?://(?:[a-zA-Z0-9\u4e00-\u9fa5\-]+\.)+[a-zA-Z0-9\u4e00-\u9fa5\-]+.*",
+ link,
+ flags=re.I,
+ ):
+ continue
+ if exclude and re.search(exclude, link):
+ continue
+ except Exception:
+ logger.error(
+ f"[ExtractError] maybe pattern include or exclude exists some problems, include: {include}\texclude: {exclude}"
+ )
+
+ if utils.is_suspicious_url(link):
+ continue
+ if link in seen:
+ continue
+ seen.append(link)
+ result.items.append(
+ SubItem(
+ url=link,
+ origin=source,
+ task=task,
+ skip_cache=skip_cache,
+ )
+ )
+ if len(seen) >= limits:
+ break
+ if len(seen) >= limits:
+ break
+
+ if include_nodes:
+ try:
+ groups = re.findall(PROTOCOL_REGEX, content, flags=re.I)
+ if groups:
+ uris.extend([x.lower().strip() for x in groups if x])
+ except Exception:
+ logger.error("[ExtractError] failed to extract single proxy")
+
+ result.nodes = NodeInput(subscribe=seen, uris=list(dict.fromkeys(uris)))
+ return result
+ except Exception:
+ logger.error("[ExtractError] extract subscribe error")
+ return ChannelResult()
diff --git a/subscribe/crawl/helpers.py b/subscribe/crawl/helpers.py
new file mode 100644
index 0000000000..d6d62a6077
--- /dev/null
+++ b/subscribe/crawl/helpers.py
@@ -0,0 +1,492 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+import json
+import os
+import random
+import re
+import socket
+import ssl
+import string
+import time
+import urllib
+import urllib.error
+import urllib.parse
+import urllib.request
+from http.client import HTTPResponse
+
+import airport
+import utils
+from config.models import StorageItem
+from crawl.channels.page import PageChannel
+from crawl.models import ChannelResult, CrawlContext
+from logger import logger
+from origin import Origin
+from push import PushTo
+
+SEPARATOR = "-"
+
+
+_REJECT_CONTENT_TYPE_PREFIXES = (
+ "video/",
+ "audio/",
+ "image/",
+ "font/",
+ "text/css",
+ "text/javascript",
+ "application/javascript",
+ "application/x-javascript",
+ "application/zip",
+ "application/x-zip",
+ "application/gzip",
+ "application/x-gzip",
+ "application/x-tar",
+ "application/x-rar",
+ "application/pdf",
+ "application/x-msdownload",
+ "application/vnd.",
+)
+
+_REJECT_DISPOSITION_EXT = (
+ ".zip",
+ ".iso",
+ ".exe",
+ ".mp4",
+ ".mkv",
+ ".avi",
+ ".tar",
+ ".tgz",
+ ".gz",
+ ".7z",
+ ".rar",
+ ".bin",
+ ".img",
+ ".dmg",
+ ".apk",
+ ".msi",
+ ".pdf",
+ ".torrent",
+ ".css",
+ ".js",
+ ".png",
+ ".jpg",
+ ".jpeg",
+ ".gif",
+ ".svg",
+ ".webp",
+ ".woff",
+ ".woff2",
+ ".ico",
+)
+
+_VALIDATE_UNIDENTIFIED_BYTES = 64 * 1024
+_VALIDATE_CHUNK_SIZE = 8 * 1024
+_VALIDATE_SNIFF_DEADLINE = 8
+_VALIDATE_DEADLINE = 12
+_VALIDATE_CONNECT_TIMEOUT = 10
+
+
+def merge_results(results: list[ChannelResult | None]) -> ChannelResult:
+ merged = ChannelResult()
+ for item in results or []:
+ merged.merge(item)
+ return merged
+
+
+def fetch_jobs(jobs: list[PageChannel] | None, ctx: CrawlContext) -> ChannelResult:
+ if not isinstance(ctx, CrawlContext):
+ return ChannelResult()
+ jobs = [job for job in (jobs or []) if isinstance(job, PageChannel)]
+ if not jobs:
+ return ChannelResult()
+ if len(jobs) == 1:
+ return jobs[0].fetch(ctx)
+ results = utils.multi_thread_run(
+ func=_fetch_job,
+ tasks=[[job, ctx] for job in jobs],
+ num_threads=ctx.num_threads,
+ show_progress=ctx.display,
+ )
+ return merge_results(results)
+
+
+def _fetch_job(job: PageChannel, ctx: CrawlContext) -> ChannelResult:
+ if not isinstance(job, PageChannel) or not isinstance(ctx, CrawlContext):
+ return ChannelResult()
+ return job.fetch(ctx)
+
+
+def is_reachable(url: str, timeout: float = 5, retry: int = 2) -> bool:
+ url = utils.trim(url)
+ if not url:
+ return False
+ timeout = max(1.0, float(timeout))
+ retry = max(1, int(retry))
+ headers = utils.DEFAULT_HTTP_HEADERS
+ for attempt in range(retry):
+ try:
+ request = urllib.request.Request(url=url, headers=headers)
+ response = urllib.request.urlopen(request, timeout=timeout, context=utils.CTX)
+ try:
+ response.close()
+ except Exception:
+ pass
+ return True
+ except urllib.error.HTTPError:
+ return True
+ except (urllib.error.URLError, socket.timeout, TimeoutError, ssl.SSLError, ConnectionError, OSError):
+ if attempt + 1 >= retry:
+ return False
+ except Exception:
+ return False
+ return False
+
+
+def load_records(pushtool: PushTo | None, item: StorageItem | None) -> dict[str, object]:
+ if not isinstance(pushtool, PushTo) or not isinstance(item, StorageItem) or not pushtool.validate(item=item):
+ return {}
+ try:
+ url = pushtool.raw_url(item=item) or ""
+ content = ""
+ if not url.startswith(utils.FILEPATH_PROTOCAL):
+ content = utils.http_get(url=url)
+ else:
+ file = url[len(utils.FILEPATH_PROTOCAL) :]
+ if os.path.exists(file) and os.path.isfile(file):
+ with open(file, "r", encoding="utf8") as reader:
+ content = reader.read()
+ if utils.isblank(content):
+ return {}
+ data = json.loads(content)
+ return data if isinstance(data, dict) else {}
+ except Exception:
+ logger.error("[CrawlError] load old subscriptions from remote error")
+ return {}
+
+
+def save_records(pushtool: PushTo | None, item: StorageItem | None, records: dict[str, object] | None) -> bool:
+ if (
+ not isinstance(pushtool, PushTo)
+ or not isinstance(item, StorageItem)
+ or not records
+ or not pushtool.validate(item=item)
+ ):
+ return False
+ try:
+ return pushtool.push_to(content=json.dumps(records), item=item, group="crawl")
+ except Exception:
+ logger.error("[CrawlError] save subscriptions failed")
+ return False
+
+
+def remark(source: dict[str, object], defeat: int = 0, discovered: bool = True) -> None:
+ if not source or type(source) != dict or type(defeat) != int or defeat < 0 or type(discovered) != bool:
+ return
+
+ source["defeat"] = defeat
+ source["discovered"] = discovered
+ if utils.isblank(source.get("origin", "")):
+ source["origin"] = Origin.TEMPORARY.name
+
+
+def check_status(
+ url: str,
+ retry: int = 2,
+ remain: float = 0,
+ spare_time: float = 0,
+ tolerance: float = 0,
+ connectable: bool = True,
+) -> tuple[bool, bool]:
+ """
+ url: subscription link
+ retry: number of retries
+ remain: minimum remaining traffic flow
+ spare_time: minimum remaining time
+ tolerance: waiting time after expiration
+
+ Returns:
+ tuple[bool, bool]: (available, expired)
+ - First bool: whether the subscription is available
+ - Second bool: whether the subscription has expired
+ """
+ if not url or retry <= 0:
+ return False, connectable
+
+ if utils.is_suspicious_url(url):
+ logger.debug(f"[Validate] skip suspicious url: {utils.mask(url)}")
+ return False, True
+
+ deadline = time.monotonic() + _VALIDATE_DEADLINE
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ return False, connectable
+
+ response = None
+ try:
+ headers = {"User-Agent": f"{utils.USER_AGENT}; Clash.Meta; Mihomo; Shadowrocket;"}
+ request = urllib.request.Request(url=url, headers=headers)
+ timeout = max(1.0, min(_VALIDATE_CONNECT_TIMEOUT, remaining))
+ response = urllib.request.urlopen(request, timeout=timeout, context=utils.CTX)
+ if response.getcode() != 200:
+ return False, connectable
+
+ if _should_reject_response(response):
+ logger.debug(f"[Validate] reject by header: {utils.mask(url)}")
+ return False, True
+
+ subscription = response.getheader("subscription-userinfo")
+ raw = _read_subscription_body(response, deadline=deadline)
+ if raw is None:
+ return False, True
+
+ try:
+ content = raw.decode("utf-8")
+ except UnicodeDecodeError:
+ return False, True
+
+ if len(content) < 32:
+ return False, False
+
+ if utils.isb64encode(content):
+ return is_expired(header=subscription, remain=remain, spare_time=spare_time, tolerance=tolerance)
+
+ if _looks_like_subscription(raw) == "yes":
+ return is_expired(header=subscription, remain=remain, spare_time=spare_time, tolerance=tolerance)
+
+ lines = [line for line in content.splitlines() if line.strip()]
+ if lines and all(airport.AirPort.check_protocol(line) for line in lines):
+ return True, False
+ return False, True
+ except urllib.error.HTTPError as e:
+ try:
+ message = str(e.read(4096), encoding="utf8")
+ except:
+ message = ""
+
+ expired = e.code == 404 or "token is error" in message
+ if not expired and e.code in [403, 503]:
+ return check_status(
+ url=url,
+ retry=retry - 1,
+ remain=remain,
+ spare_time=spare_time,
+ tolerance=tolerance,
+ connectable=connectable,
+ )
+
+ return False, expired
+ except (socket.timeout, TimeoutError, ssl.SSLError, ConnectionError, OSError):
+ return check_status(
+ url=url,
+ retry=retry - 1,
+ remain=remain,
+ spare_time=spare_time,
+ tolerance=tolerance,
+ connectable=connectable,
+ )
+ except Exception as e:
+ logger.debug(f"[Validate] unexpected error for {utils.mask(url)}: {e}")
+ return False, connectable
+ finally:
+ if response is not None:
+ try:
+ response.close()
+ except Exception:
+ pass
+
+
+def is_expired(header: str, remain: float = 0, spare_time: float = 0, tolerance: float = 0) -> tuple[bool, bool]:
+ if utils.isblank(header):
+ return True, False
+
+ remain, spare_time, tolerance = (
+ max(0, remain),
+ max(spare_time, 0),
+ max(tolerance, 0),
+ )
+ try:
+ infos = header.split(";")
+ upload, download, total, expire = 0, 0, 0, None
+ for info in infos:
+ words = info.split("=", maxsplit=1)
+ if len(words) <= 1:
+ continue
+
+ if "upload" == words[0].strip():
+ upload = eval(words[1])
+ elif "download" == words[0].strip():
+ download = eval(words[1])
+ elif "total" == words[0].strip():
+ total = eval(words[1])
+ elif "expire" == words[0].strip():
+ expire = None if utils.isblank(words[1]) else eval(words[1])
+
+ # 剩余流量大于 ${remain} GB 并且未过期则返回 True，否则返回 False
+ flag = total - (upload + download) > remain * pow(1024, 3) and (
+ expire is None or expire - time.time() > spare_time * 3600
+ )
+ expired = False if flag else (expire is not None and (expire + tolerance * 3600) <= time.time())
+ return flag, expired
+ except:
+ return True, False
+
+
+def _should_reject_response(response: HTTPResponse | None) -> bool:
+ """响应头检查 - 明显不是订阅"""
+ if response is None:
+ return False
+ ctype = (response.getheader("Content-Type") or "").split(";")[0].strip().lower()
+ if ctype:
+ for prefix in _REJECT_CONTENT_TYPE_PREFIXES:
+ if ctype.startswith(prefix):
+ return True
+
+ disposition = response.getheader("Content-Disposition") or ""
+ if disposition:
+ m = re.search(r"filename\*?=(?:UTF-8''|\"|')?([^\";]+)", disposition, re.I)
+ if m:
+ name = urllib.parse.unquote(m.group(1).strip().strip("\"'")).lower()
+ if name.endswith(_REJECT_DISPOSITION_EXT):
+ return True
+
+ return False
+
+
+def _read_subscription_body(response: HTTPResponse | None, deadline: float | None = None) -> bytes | None:
+ """Sniff a subscription from the response body and stop as soon as it is confirmed"""
+ if response is None:
+ return None
+ start = time.monotonic()
+ buf = bytearray()
+ limit = _VALIDATE_UNIDENTIFIED_BYTES
+ verdict = "maybe"
+
+ while len(buf) < limit:
+ now = time.monotonic()
+ if deadline is not None and now >= deadline:
+ try:
+ response.close()
+ except Exception:
+ pass
+ return None
+ if verdict != "yes" and (now - start) > _VALIDATE_SNIFF_DEADLINE:
+ try:
+ response.close()
+ except Exception:
+ pass
+ return None
+
+ to_read = min(_VALIDATE_CHUNK_SIZE, limit - len(buf))
+ try:
+ chunk = response.read(to_read)
+ except (socket.timeout, TimeoutError, OSError):
+ try:
+ response.close()
+ except Exception:
+ pass
+ return None
+
+ if not chunk:
+ break
+
+ buf.extend(chunk)
+
+ if verdict != "yes":
+ verdict = _looks_like_subscription(bytes(buf))
+ if verdict == "no":
+ try:
+ response.close()
+ except Exception:
+ pass
+ return None
+ if verdict == "yes":
+ try:
+ response.close()
+ except Exception:
+ pass
+ return bytes(buf)
+
+ if verdict != "yes" and len(buf) >= _VALIDATE_UNIDENTIFIED_BYTES:
+ try:
+ extra = response.read(1)
+ except Exception:
+ extra = b""
+ if extra:
+ try:
+ response.close()
+ except Exception:
+ pass
+ return None
+
+ return bytes(buf)
+
+
+def _looks_like_subscription(data: bytes) -> str:
+ """内容嗅探 - 返回 'yes' / 'no' / 'maybe'"""
+ if not data:
+ return "maybe"
+
+ sample = data[:8192]
+ if b"\x00" in sample:
+ return "no"
+
+ try:
+ text = data.decode("utf-8")
+ except UnicodeDecodeError:
+ return "no"
+
+ sample_text = text.lstrip(" \t\r\n")
+ if not sample_text:
+ return "maybe"
+
+ head = sample_text[:64].lower()
+ if head.startswith(("= 32 and re.match(r"^[A-Za-z0-9+/=]+$", compact):
+ return "yes"
+
+ # 纯注释开头，机场订阅常见前言
+ lines = [ln.strip() for ln in sample_text.splitlines() if ln.strip()]
+ if lines and all(ln.startswith("#") or ln.startswith("//") for ln in lines):
+ return "maybe"
+
+ return "maybe"
+
+
+def is_available(url: str, retry: int = 2, remain: float = 0, spare_time: float = 0) -> bool:
+ available, _ = check_status(url=url, retry=retry, remain=remain, spare_time=spare_time)
+ return available
+
+
+def naming_task(url: str) -> str:
+ prefix = utils.extract_domain(url=url).replace(".", "") + SEPARATOR
+ return prefix + "".join(random.sample(string.digits + string.ascii_lowercase, random.randint(3, 5)))
diff --git a/subscribe/crawl/models.py b/subscribe/crawl/models.py
new file mode 100644
index 0000000000..7d71e92436
--- /dev/null
+++ b/subscribe/crawl/models.py
@@ -0,0 +1,103 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from config.models import NodeInput, StorageConfig, TaskParams
+from push import PushTo
+
+
+@dataclass
+class SubItem:
+ url: str
+ origin: str
+ task: TaskParams = field(default_factory=TaskParams)
+ name: str = ""
+ ready: bool = False
+ persist_only: bool = False
+ skip_cache: bool = False
+ allow_nonstandard: bool = False
+ errors: int = 0
+ discovered: bool = False
+ debut: bool = False
+
+
+@dataclass
+class ChannelResult:
+ nodes: NodeInput = field(default_factory=NodeInput)
+ items: list[SubItem] = field(default_factory=list)
+
+ def merge(self, other: ChannelResult | None) -> ChannelResult:
+ if not other:
+ return self
+ subs = self.nodes.subscribe_list() + other.nodes.subscribe_list()
+ self.nodes.subscribe = list(dict.fromkeys(subs))
+ self.nodes.uris = list(dict.fromkeys(self.nodes.uris + other.nodes.uris))
+ self.nodes.proxies.extend(other.nodes.proxies)
+ self.items.extend(other.items)
+ return self
+
+ def add_subscribe(
+ self,
+ url: str,
+ origin: str,
+ task: TaskParams | None = None,
+ name: str = "",
+ ready: bool = False,
+ persist_only: bool = False,
+ skip_cache: bool = False,
+ allow_nonstandard: bool = False,
+ errors: int = 0,
+ discovered: bool = False,
+ debut: bool = False,
+ ) -> None:
+ url = (url or "").strip()
+ if not url:
+ return
+ task = task or TaskParams()
+ self.items.append(
+ SubItem(
+ url=url,
+ origin=origin,
+ task=task,
+ name=name,
+ ready=ready,
+ persist_only=persist_only,
+ skip_cache=skip_cache,
+ allow_nonstandard=allow_nonstandard,
+ errors=errors,
+ discovered=discovered,
+ debut=debut,
+ )
+ )
+ current = self.nodes.subscribe_list()
+ if url not in current:
+ current.append(url)
+ self.nodes.subscribe = current
+
+ def add_uris(self, uris: list[str] | None) -> None:
+ if not uris:
+ return
+ self.nodes.uris = list(dict.fromkeys(self.nodes.uris + [item for item in uris if item]))
+
+
+@dataclass
+class CrawlContext:
+ mode: int
+ include_nodes: bool
+ max_fails: int
+ exclude: str
+ task: TaskParams
+ storage: StorageConfig | None
+ pushtool: PushTo | None
+ num_threads: int = 50
+ display: bool = True
+
+
+@dataclass
+class ValidateResult:
+ proxies: set[str] | None = None
+ available: object | None = None
+ potential: dict[str, object] | None = None
+ unknown: str | None = None
diff --git a/subscribe/discovery.py b/subscribe/discovery.py
new file mode 100644
index 0000000000..fbff588657
--- /dev/null
+++ b/subscribe/discovery.py
@@ -0,0 +1,693 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+import gzip
+import itertools
+import json
+import random
+import re
+import socket
+import ssl
+import time
+import traceback
+import urllib.error
+import urllib.parse
+import urllib.request
+from abc import ABC, abstractmethod
+from collections.abc import Sequence
+from dataclasses import dataclass
+
+import utils
+from airport import AirPort
+from logger import logger
+
+DEFAULT_DELIMITER = "@#@#"
+
+
+@dataclass(frozen=True)
+class CollectOptions:
+ num_threads: int = 64
+ display: bool = True
+ persist_path: str = ""
+ delimiter: str = ""
+ allow_gmail_alias: bool = False
+ skip_captcha: bool = False
+
+
+@dataclass
+class AirportRecord:
+ domain: str
+ coupon: str = ""
+ invite_code: str = ""
+ api_prefix: str = ""
+ source: str = ""
+
+ def to_line(self, delimiter: str, full: bool = False) -> str:
+ delimiter = utils.trim(delimiter) or DEFAULT_DELIMITER
+ domain = utils.trim(self.domain)
+ coupon = utils.trim(self.coupon)
+ invite_code = utils.trim(self.invite_code)
+ api_prefix = utils.trim(self.api_prefix)
+ if full:
+ return f"{domain}\t{delimiter}\t{coupon}\t{delimiter}\t{invite_code}\t{delimiter}\t{api_prefix}"
+ if coupon:
+ return f"{domain}\t{delimiter}\t{coupon}"
+ return domain
+
+ @classmethod
+ def from_line(cls, line: str, delimiter: str) -> AirportRecord | None:
+ text = utils.trim(line)
+ if not text or text.startswith("#"):
+ return None
+
+ delimiter = utils.trim(delimiter) or DEFAULT_DELIMITER
+ parts = text.rsplit(delimiter, maxsplit=3)
+ domain = utils.trim(parts[0])
+ if not domain:
+ return None
+
+ return cls(
+ domain=domain,
+ coupon=utils.trim(parts[1]) if len(parts) > 1 else "",
+ invite_code=utils.trim(parts[2]) if len(parts) > 2 else "",
+ api_prefix=utils.trim(parts[3]) if len(parts) > 3 else "",
+ )
+
+
+def parse_records(content: str, delimiter: str) -> dict[str, AirportRecord]:
+ if not content or not isinstance(content, str):
+ logger.warning("cannot found any domain due to content is empty or not string")
+ return {}
+
+ records: dict[str, AirportRecord] = {}
+ for line in content.split("\n"):
+ record = AirportRecord.from_line(line, delimiter)
+ if record:
+ records[record.domain] = record
+ return records
+
+
+def save_records(
+ records: dict[str, AirportRecord],
+ filepath: str,
+ delimiter: str,
+ full: bool = False,
+) -> None:
+ if not records or not isinstance(records, dict):
+ return
+
+ filepath = utils.trim(filepath)
+ if not filepath:
+ return
+
+ delimiter = utils.trim(delimiter) or DEFAULT_DELIMITER
+ lines = [record.to_line(delimiter, full=full) for record in records.values() if record and record.domain]
+ utils.write_file(filename=filepath, lines=lines)
+
+
+def count_telegram_pages(channel: str) -> int:
+ if not channel or channel.strip() == "":
+ return 0
+
+ url = f"https://t.me/s/{channel}"
+ content = utils.http_get(url=url)
+ cursor = 0
+ try:
+ pattern = rf''
+ groups = re.findall(pattern, content)
+ cursor = int(groups[0]) if groups else cursor
+ except Exception:
+ logger.error(f"[CrawlError] cannot count page num, channel: {channel}")
+
+ return cursor
+
+
+def extract_telegram_sites(url: str) -> list[str]:
+ if not url:
+ return []
+
+ logger.info(f"[AirPortCrawl] start collect airport, url: {url}")
+
+ content = utils.http_get(url=url)
+ if not content:
+ logger.error(f"[CrawlError] cannot any content from url: {url}")
+ return []
+ try:
+ pattern = r'href="(https?://(?:[a-zA-Z0-9\u4e00-\u9fa5\-]+\.)+[a-zA-Z0-9\u4e00-\u9fa5\-]+/?)"\s+target="_blank"\s+rel="noopener">'
+ groups = re.findall(pattern, content)
+ return list(set(groups)) if groups else []
+ except Exception:
+ return []
+
+
+def follow_redirect(url: str, retry: int = 3) -> str:
+ if not url or retry <= 0:
+ return ""
+
+ headers = {
+ "User-Agent": utils.USER_AGENT,
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
+ "Accept-Encoding": "gzip, deflate, br, zstd",
+ "Accept-Language": "zh-CN,zh;q=0.9",
+ }
+
+ try:
+ request = urllib.request.Request(url=url, headers=headers, method="GET")
+ response = urllib.request.urlopen(request, timeout=10, context=utils.CTX)
+ return response.geturl()
+ except Exception:
+ time.sleep(random.randint(1, 3))
+ return follow_redirect(url=url, retry=retry - 1)
+
+
+def extract_listing(url: str, separator: str, address_regex: str, coupon_regex: str) -> dict[str, str]:
+ url = utils.trim(url)
+ content = utils.http_get(url=url)
+ if not content:
+ return {}
+
+ mapping: dict[str, str] = {}
+ try:
+ groups = re.split(utils.trim(separator), content, flags=re.M)
+ if not groups:
+ logger.warning(f"[AirPortCollector] cannot found any domains from [{url}]")
+ return {}
+
+ for group in groups:
+ if not group or not isinstance(group, str):
+ continue
+
+ words = re.findall(utils.trim(address_regex), group, flags=re.M)
+ address = words[0] if words else ""
+ if not address:
+ continue
+
+ words = re.findall(utils.trim(coupon_regex), group, flags=re.M)
+ coupon = words[0] if words else ""
+ domain = utils.extract_domain(url=address, include_protocal=True)
+ mapping[domain] = coupon
+ except Exception:
+ logger.error(f"[AirPortCollector] occur error when crawl from [{url}], message: \n{traceback.format_exc()}")
+
+ logger.info(f"[AirPortCollector] finished crawl from [{url}], found {len(mapping)} domains")
+ return mapping
+
+
+def extract_links(url: str, prefix: str, pattern: str) -> list[str]:
+ content = utils.http_get(url=url)
+ groups = re.findall(pattern, content, flags=re.I)
+ if not groups:
+ logger.warning(f"[AirPortCollector] cannot fetch article from url: {url}")
+ return []
+
+ prefix = utils.trim(prefix)
+ return list(set([urllib.parse.urljoin(prefix, item) for item in groups if item]))
+
+
+class AirportSource(ABC):
+ name: str = ""
+ persist: bool = True
+
+ @abstractmethod
+ def crawl(self, options: CollectOptions) -> list[AirportRecord]:
+ raise NotImplementedError
+
+ def to_records(self, mapping: dict[str, str] | None) -> list[AirportRecord]:
+ records: list[AirportRecord] = []
+ for domain, coupon in (mapping or {}).items():
+ domain = utils.trim(domain)
+ if not domain:
+ continue
+ records.append(AirportRecord(domain=domain, coupon=utils.trim(coupon), source=self.name))
+ return records
+
+
+class RegexAirportSource(AirportSource):
+ url: str = ""
+ separator: str = ""
+ address_regex: str = ""
+ coupon_regex: str = ""
+
+ def crawl(self, options: CollectOptions) -> list[AirportRecord]:
+ return self.to_records(
+ extract_listing(
+ url=self.url,
+ separator=self.separator,
+ address_regex=self.address_regex,
+ coupon_regex=self.coupon_regex,
+ )
+ )
+
+
+class MaomengSource(RegexAirportSource):
+ name = "maomeng"
+ url = "https://maomeng.xyz/2021/06/11/ji-chang-tui-jian-chang-qi-geng-xin"
+ separator = r'官网：.* "
+ coupon_regex = r"
[^<]*(?:优惠|白嫖)码：([^<]+)
"
+
+
+class AskahhSource(RegexAirportSource):
+ name = "askahh"
+ url = "https://www.askahh.com/archives/17"
+ separator = r"]*>[^<]+
"
+ address_regex = r''
+ coupon_regex = r"使用优惠码\s*(?:)?([^<]+?)(?:)?\s*免费购买"
+
+
+class TelegramSource(AirportSource):
+ name = "telegram"
+ persist = False
+
+ def __init__(self, channel: str, page_num: int) -> None:
+ self.channel = utils.trim(channel)
+ self.page_num = page_num
+
+ def crawl(self, options: CollectOptions) -> list[AirportRecord]:
+ urls = self._page_urls()
+ if not urls:
+ return []
+
+ if len(urls) == 1:
+ sites = extract_telegram_sites(urls[0])
+ else:
+ batches = utils.multi_thread_run(func=extract_telegram_sites, tasks=urls)
+ sites = list(itertools.chain.from_iterable(batch or [] for batch in batches))
+
+ records: list[AirportRecord] = []
+ seen: set[str] = set()
+ for site in sites:
+ domain = utils.extract_domain(site, True)
+ if not domain or domain in seen:
+ continue
+ seen.add(domain)
+ records.append(AirportRecord(domain=domain, source=self.name))
+ return records
+
+ def _page_urls(self) -> list[str]:
+ if not self.channel:
+ return []
+
+ page_num = max(self.page_num, 1)
+ url = f"https://t.me/s/{self.channel}"
+ if page_num == 1:
+ return [url]
+
+ cursor = count_telegram_pages(self.channel)
+ if cursor == 0:
+ return []
+
+ pages = range(cursor, -1, -20)
+ page_num = min(page_num, len(pages))
+ logger.info(f"[TelegramCrawl] starting crawl from telegram, channel: {self.channel}, pages: {page_num}")
+ return [f"{url}?before={item}" for item in pages[:page_num]]
+
+
+class HwanzSource(AirportSource):
+ name = "hwanz"
+ url = "https://raw.githubusercontent.com/hwanz/SSR-V2ray-Trojan-vpn/main/README.md"
+
+ def crawl(self, options: CollectOptions) -> list[AirportRecord]:
+ content = utils.http_get(url=self.url)
+ groups = re.findall(r"\[.*\]\((https?:\/\/[^\s\r\n]+)\)[^\r\n]+\d+G.*", content, flags=re.I)
+ if not groups:
+ return []
+
+ try:
+ links = [utils.trim(item).lower() for item in groups if item]
+ mapping = {utils.extract_domain(url=item, include_protocal=True): "" for item in links if item}
+ logger.info(f"[AirPortCollector] finished crawl from [{self.url}], found {len(mapping)} domains")
+ return self.to_records(mapping)
+ except Exception:
+ logger.error(
+ f"[AirPortCollector] occur error when crawl from [{self.url}], message: \n{traceback.format_exc()}"
+ )
+ return []
+
+
+class CcbaoheSource(AirportSource):
+ name = "ccbaohe"
+ url = "https://ccbaohe.com/jcjd.html"
+
+ def crawl(self, options: CollectOptions) -> list[AirportRecord]:
+ content = utils.http_get(url=self.url)
+ mapping: dict[str, str] = {}
+ try:
+ groups = re.split(r"【[^【]*】", content, flags=re.M)
+ if not groups:
+ logger.warning(f"[AirPortCollector] cannot found any domains from [{self.url}]")
+ return []
+
+ candidates: dict[str, str] = {}
+ for group in groups:
+ if not group:
+ continue
+
+ for text in re.split(r"\s*", group, flags=re.I):
+ words = re.findall(r'注册地址：]+>\s*)*([A-Za-z0-9_\-]+)", text, flags=re.M
+ )
+ candidates[address] = words[0] if words else ""
+
+ urls = list(candidates.keys())
+ redirected = utils.multi_thread_run(
+ func=follow_redirect,
+ tasks=urls,
+ num_threads=options.num_threads,
+ )
+
+ for index, original in enumerate(urls):
+ domain = utils.extract_domain(url=redirected[index], include_protocal=True)
+ if not domain:
+ continue
+ mapping[domain] = candidates.get(original, "")
+ except Exception:
+ logger.error(
+ f"[AirPortCollector] occur error when crawl from [{self.url}], message: \n{traceback.format_exc()}"
+ )
+
+ logger.info(f"[AirPortCollector] finished crawl from [{self.url}], found {len(mapping)} domains")
+ return self.to_records(mapping)
+
+
+class YgpySource(AirportSource):
+ name = "ygpy"
+ base_url = "https://ygpy.net"
+ index_urls = ("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/vpn/free.html", "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/vpn/free")
+
+ def crawl(self, options: CollectOptions) -> list[AirportRecord]:
+ pages = self._monthly_pages()
+ if not pages:
+ logger.warning(f"[AirPortCollector] cannot get article from url: {self.base_url}")
+ return []
+
+ separator = r''
+ address_regex = r'
网站：)?([^\s<]+)(?:)?\s*0\s*元购买"
+
+ tasks = [[page, separator, address_regex, coupon_regex] for page in pages]
+ listings = utils.multi_thread_run(func=extract_listing, tasks=tasks)
+
+ mapping: dict[str, str] = {}
+ for listing in listings:
+ if listing and isinstance(listing, dict):
+ mapping.update(listing)
+
+ return self.to_records(self._drop_non_sites(mapping))
+
+ def _drop_non_sites(self, mapping: dict[str, str]) -> dict[str, str]:
+ skipped = {
+ "t.me",
+ "www.t.me",
+ "telegram.me",
+ "telegram.org",
+ "ygpy.net",
+ "www.ygpy.net",
+ "ygpy.org",
+ "www.ygpy.org",
+ }
+ cleaned: dict[str, str] = {}
+ for domain, coupon in mapping.items():
+ host = utils.extract_domain(domain, include_protocal=False).lower()
+ if not host or host in skipped:
+ continue
+ cleaned[domain] = coupon
+ return cleaned
+
+ def _monthly_pages(self) -> list[str]:
+ paths: list[str] = []
+ for suffix in self.index_urls:
+ content = utils.http_get(url=f"{self.base_url}{suffix}")
+ if not content:
+ continue
+ found = re.findall(r"/vpn/test/\d{4}/\d{2}\.html", content)
+ if found:
+ paths = found
+ break
+
+ pages: list[str] = []
+ seen: set[str] = set()
+ for path in sorted(paths):
+ if path in seen:
+ continue
+ seen.add(path)
+ pages.append(urllib.parse.urljoin(self.base_url, path))
+ return pages
+
+
+@dataclass
+class ProbeResult:
+ blocked: bool = False
+ backend: str = ""
+
+
+class BackendResolver:
+ def resolve(self, domain: str, retry: int = 2) -> str:
+ # TODO: exploring a more generalized approach to backend addresses
+ attempts = max(retry, 1)
+ for probe in (self._probe_env, self._probe_zero_theme, self._probe_buddy, self._probe_aurora):
+ result = probe(domain, attempts)
+ if result.blocked:
+ return ""
+ if result.backend:
+ return result.backend
+ return domain
+
+ def _fetch_path(self, domain: str, suffix: str, retry: int) -> tuple[bool, str]:
+ count, suffix = 0, utils.trim(suffix)
+ url = urllib.parse.urljoin(domain, suffix)
+
+ while count < retry:
+ count += 1
+ try:
+ request = urllib.request.Request(url=url, headers=utils.DEFAULT_HTTP_HEADERS, method="GET")
+ response = urllib.request.urlopen(request, timeout=6, context=utils.CTX)
+
+ word = "" if not suffix else (suffix if suffix.startswith("/") else "/" + suffix)
+ if word and not utils.trim(response.geturl()).endswith(word):
+ return True, ""
+
+ content = response.read()
+ try:
+ content = str(content, encoding="utf8")
+ except Exception:
+ content = gzip.decompress(content).decode("utf8")
+
+ return False, content
+ except urllib.error.HTTPError as exc:
+ if exc.code == 404:
+ return False, ""
+ except urllib.error.URLError as exc:
+ if isinstance(exc.reason, (socket.gaierror, ssl.SSLError, socket.timeout)):
+ return True, ""
+ except Exception:
+ pass
+
+ return False, ""
+
+ def _probe_env(self, domain: str, retry: int) -> ProbeResult:
+ blocked, content = self._fetch_path(domain, suffix="/env.js", retry=retry)
+ if blocked:
+ return ProbeResult(blocked=True)
+ if not content:
+ return ProbeResult()
+
+ groups = re.findall(r"\bhost\b:(?:\s+)?[\"\'](https?://[^\s\r\t]+)[\"\']", content, flags=re.I)
+ if groups:
+ return ProbeResult(backend=utils.trim(groups[0]))
+
+ groups = re.findall(r"window.routerBase(?:\s+)?=(?:\s+)?['\"](https?://.*)['\"]", content, flags=re.I)
+ backend = groups[0].rstrip("/") if groups and groups[0] else ""
+ return ProbeResult(backend=backend)
+
+ def _probe_zero_theme(self, domain: str, retry: int) -> ProbeResult:
+ # for https://github.com/amyouran/v2board-Zero-Theme
+ blocked, content = self._fetch_path(domain, suffix="/config.json", retry=retry)
+ if blocked:
+ return ProbeResult(blocked=True)
+ if not content:
+ return ProbeResult()
+
+ try:
+ data = json.loads(content)
+ link = utils.trim(data.get("api_base", ""))
+ if not link:
+ # for https://github.com/DyAxy/V2B-Theme-Nest
+ link = utils.trim(data.get("apiUrl", ""))
+ return ProbeResult(backend=utils.extract_domain(url=link, include_protocal=True))
+ except Exception:
+ return ProbeResult()
+
+ def _probe_buddy(self, domain: str, retry: int) -> ProbeResult:
+ # for https://github.com/vlesstop/v2board-theme-buddy
+ blocked, content = self._fetch_path(domain, suffix="/config.js", retry=retry)
+ if blocked:
+ return ProbeResult(blocked=True)
+
+ groups = re.findall(r"\bhost\b:(?:\s+)?[\"\'](https?://[^\s\r\t]+)[\"\']", content, flags=re.I)
+ return ProbeResult(backend="" if not groups else utils.trim(groups[0]))
+
+ def _probe_aurora(self, domain: str, retry: int) -> ProbeResult:
+ # for https://github.com/krsunm/Aurora
+ blocked, content = self._fetch_path(domain, suffix="", retry=retry)
+ if blocked:
+ return ProbeResult(blocked=True)
+
+ groups = re.findall(r"\bserverUrl\b:(?:\s+)?[\"\'](https?://[^\s\r\t]+)[\"\']", content, flags=re.I)
+ return ProbeResult(backend="" if not groups else utils.trim(groups[0]))
+
+
+class DomainValidator:
+ def validate(self, url: str, allow_gmail_alias: bool = False, skip_captcha: bool = False) -> tuple[bool, str]:
+ try:
+ if not url:
+ return False, ""
+
+ requirement = AirPort.get_register_require(domain=url)
+ blocked = (
+ requirement.invite
+ or (skip_captcha and requirement.recaptcha)
+ or (
+ requirement.whitelist
+ and requirement.verify
+ and (not allow_gmail_alias or "gmail.com" not in requirement.whitelist)
+ )
+ )
+ return not blocked, requirement.api_prefix
+ except Exception:
+ return False, ""
+
+
+class AirportCollector:
+ def __init__(
+ self,
+ sources: Sequence[AirportSource],
+ options: CollectOptions,
+ resolver: BackendResolver | None = None,
+ validator: DomainValidator | None = None,
+ ) -> None:
+ self.sources = list(sources)
+ self.options = options
+ self.resolver = resolver or BackendResolver()
+ self.validator = validator or DomainValidator()
+
+ @classmethod
+ def default(cls, channel: str, page_num: int, options: CollectOptions) -> AirportCollector:
+ return cls(sources=default_sources(channel, page_num), options=options)
+
+ def collect(self) -> dict[str, AirportRecord]:
+ discovered, persisted = self._discover()
+ self._persist(persisted)
+ resolved = self._resolve(discovered)
+ return self._validate(resolved)
+
+ def _discover(self) -> tuple[dict[str, AirportRecord], dict[str, AirportRecord]]:
+ discovered: dict[str, AirportRecord] = {}
+ persisted: dict[str, AirportRecord] = {}
+ for source in self.sources:
+ items = source.crawl(self.options) or []
+ for record in items:
+ domain = utils.trim(record.domain)
+ if not domain:
+ continue
+ record.domain = domain
+ if not record.source:
+ record.source = source.name
+ discovered[domain] = record
+ if source.persist:
+ persisted[domain] = record
+ return discovered, persisted
+
+ def _persist(self, records: dict[str, AirportRecord]) -> None:
+ save_records(
+ records=records,
+ filepath=self.options.persist_path,
+ delimiter=self.options.delimiter,
+ full=False,
+ )
+
+ def _resolve(self, records: dict[str, AirportRecord]) -> list[AirportRecord]:
+ domains = list(records.keys())
+ logger.info(f"[AirPortCollector] fetched {len(domains)} airport, start extracting real routing addresses")
+ backends = utils.multi_thread_run(
+ func=self.resolver.resolve,
+ tasks=domains,
+ num_threads=self.options.num_threads,
+ show_progress=self.options.display,
+ )
+
+ resolved: list[AirportRecord] = []
+ for index, backend in enumerate(backends):
+ if not backend:
+ continue
+ origin = records.get(domains[index])
+ if origin is None:
+ continue
+ resolved.append(
+ AirportRecord(
+ domain=backend,
+ coupon=origin.coupon,
+ invite_code=origin.invite_code,
+ source=origin.source,
+ )
+ )
+ return resolved
+
+ def _validate(self, records: list[AirportRecord]) -> dict[str, AirportRecord]:
+ logger.info("[AirPortCollector] extract real base url finished, start to check it now")
+ tasks = [[record.domain, self.options.allow_gmail_alias, self.options.skip_captcha] for record in records]
+ results = utils.multi_thread_run(
+ func=self.validator.validate,
+ tasks=tasks,
+ num_threads=self.options.num_threads,
+ show_progress=self.options.display,
+ )
+
+ available: dict[str, AirportRecord] = {}
+ for index, record in enumerate(records):
+ accepted, api_prefix = results[index]
+ if not accepted:
+ continue
+ record.api_prefix = api_prefix or ""
+ available[record.domain] = record
+
+ logger.info(f"[AirPortCollector] finished collect airport, availables: {len(available)}")
+ return available
+
+
+def default_sources(channel: str, page_num: int) -> list[AirportSource]:
+ return [
+ TelegramSource(channel=channel, page_num=page_num),
+ HwanzSource(),
+ CcbaoheSource(),
+ MaomengSource(),
+ AskahhSource(),
+ YgpySource(),
+ ]
+
+
+def collect_airport(
+ channel: str,
+ page_num: int,
+ num_threads: int = 64,
+ display: bool = True,
+ filepath: str = "",
+ delimiter: str = "",
+ allow_gmail_alias: bool = False,
+ skip_captcha: bool = False,
+) -> dict[str, AirportRecord]:
+ options = CollectOptions(
+ num_threads=num_threads,
+ display=display,
+ persist_path=filepath,
+ delimiter=delimiter,
+ allow_gmail_alias=allow_gmail_alias,
+ skip_captcha=skip_captcha,
+ )
+ return AirportCollector.default(channel=channel, page_num=page_num, options=options).collect()
diff --git a/subscribe/config/config.default.json b/subscribe/examples/config.default.json
similarity index 64%
rename from subscribe/config/config.default.json
rename to subscribe/examples/config.default.json
index d64f123541..66bf59433c 100644
--- a/subscribe/config/config.default.json
+++ b/subscribe/examples/config.default.json
@@ -1,28 +1,28 @@
{
- "domains": [
- {
+ "sites": [
+ {
"name": "",
- "sub": [],
+ "subscribe": [],
"domain": "",
"enable": true,
"rename": "",
"include": "",
"exclude": "",
"push_to": [],
- "ignorede": true,
- "liveness": true,
- "rate": 2.5,
+ "ignore_default_exclude": true,
+ "check_alive": true,
+ "max_rate": 2.5,
"count": 2,
"coupon": "",
- "secure": false,
+ "require_tls": false,
"renew": {
- "account": [
+ "accounts": [
{
"email": "",
- "passwd": "",
+ "password": "",
"ticket": {
"enable": true,
- "autoreset": false,
+ "auto_reset": false,
"subject": "",
"message": "",
"level": 1
@@ -32,41 +32,34 @@
"plan_id": 3,
"package": "",
"method": 1,
- "coupon_code": "",
- "chatgpt": {
- "enable": true,
- "regex": "",
- "operate": "IN"
- }
+ "coupon_code": ""
}
}
],
"crawl": {
"enable": true,
"exclude": "",
- "threshold": 5,
- "singlelink": true,
+ "max_fails": 5,
+ "include_nodes": true,
"persist": {
- "subs": "crawledsubs",
- "proxies": "crawledproxies"
+ "subscribe": "crawledsubs",
+ "nodes": "crawledproxies"
},
- "config": {
+ "task": {
"rename": "",
"include": "",
- "exclude": "",
- "xxxxxxx": ""
+ "exclude": ""
},
"telegram": {
"enable": true,
"pages": 5,
"exclude": "",
- "users": {
+ "channels": {
"channel": {
"include": "",
"exclude": "",
- "config": {
- "rename": "",
- "xxxxxx": ""
+ "task": {
+ "rename": ""
},
"push_to": []
}
@@ -75,16 +68,17 @@
"google": {
"enable": false,
"exclude": "",
- "limits": 100,
- "notinurl": [],
+ "limit": 100,
+ "days": 7,
+ "exclude_sites": [],
"push_to": []
},
"yandex": {
"enable": false,
"exclude": "",
- "within": 3,
+ "days": 3,
"pages": 5,
- "notinurl": [],
+ "exclude_sites": [],
"push_to": []
},
"github": {
@@ -92,19 +86,18 @@
"pages": 2,
"push_to": [],
"exclude": "",
- "spams": []
+ "exclude_repos": []
},
"twitter": {
"enable": true,
"users": {
"username": {
"enable": true,
- "num": 30,
+ "tweets": 30,
"include": "",
"exclude": "",
- "config": {
- "rename": "",
- "xxxxxx": ""
+ "task": {
+ "rename": ""
},
"push_to": []
}
@@ -114,7 +107,7 @@
{
"enable": false,
"username": "",
- "repo_name": "",
+ "repo": "",
"commits": 3,
"exclude": "",
"push_to": []
@@ -126,11 +119,11 @@
"url": "",
"include": "",
"exclude": "",
- "multiple": true,
+ "paged": true,
"placeholder": "",
"start": 1,
"end": 10,
- "config": {
+ "task": {
"rename": ""
},
"push_to": []
@@ -139,19 +132,14 @@
"scripts": [
{
"enable": false,
- "script": "file#function",
- "params": {
- "persist": {
- "fileid": ""
- },
- "any": "xxx",
- "config": {
- "enable": true,
- "liveness": true,
- "exclude": "",
- "rename": "",
- "push_to": []
- }
+ "plugin": "dynamic",
+ "persist": "",
+ "task": {
+ "enable": true,
+ "check_alive": true,
+ "exclude": "",
+ "rename": "",
+ "push_to": []
}
}
]
@@ -159,7 +147,7 @@
"groups": {
"xxx": {
"emoji": true,
- "list": true,
+ "list_only": true,
"targets": {
"clash": "xxx-clash",
"singbox": "xxx-singbox",
@@ -169,7 +157,7 @@
"enable": false,
"locate": true,
"residential": true,
- "bits": 2
+ "digits": 2
}
}
},
@@ -180,26 +168,26 @@
"items": {
"xxx-clash": {
"username": "",
- "folderid": "",
- "fileid": ""
+ "folder_id": "",
+ "file_id": ""
},
"xxx-singbox": {
- "fileid": ""
+ "file_id": ""
},
"xxx-v2ray": {
"username": "",
- "fileid": ""
+ "file_id": ""
},
"crawledsubs": {
"username": "",
- "folderid": "",
- "fileid": ""
+ "folder_id": "",
+ "file_id": ""
},
"crawledproxies": {
"username": "",
- "folderid": "",
- "fileid": ""
+ "folder_id": "",
+ "file_id": ""
}
}
}
-}
\ No newline at end of file
+}
diff --git a/subscribe/iplibrary.py b/subscribe/iplibrary.py
new file mode 100644
index 0000000000..667a1d181b
--- /dev/null
+++ b/subscribe/iplibrary.py
@@ -0,0 +1,541 @@
+# -*- coding: utf-8 -*-
+
+# @Author : wzdnzd
+# @Time : 2026-09-07
+
+import html
+import ipaddress
+import json
+import random
+import re
+import urllib.parse
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import Any, Optional
+
+RequestFn = Callable[..., tuple[bool, Any]]
+
+import utils
+from logger import logger
+
+# HTTPS only. Fetched via CONNECT + TLS through the HTTP inbound.
+EGRESS_IPV4_URLS = (
+ "https://ipinfo.io/ip",
+ "https://api-ipv4.ip.sb/ip",
+ "https://ipv4.ip.sb/ip",
+ "https://ipv4.icanhazip.com",
+ "https://icanhazip.com",
+ "https://api4.ipify.org",
+ "https://api.ipify.org",
+ "https://v4.ident.me",
+ "https://ident.me",
+ "https://iplark.com/ipapi/public/ip",
+ "https://ipv4.wtfismyip.com/text",
+ "https://ifconfig.me/ip",
+ "https://ifconfig.co/ip",
+ "https://ifconfig.io/ip",
+ "https://ipecho.net/plain",
+ "https://wgetip.com",
+ "https://myexternalip.com/raw",
+ "https://ip.tyk.nu",
+ "https://eth0.me",
+ "https://ipapi.co/ip",
+ "https://ipv4.nsupdate.info/myip",
+ "https://myip.ipip.net",
+ "https://v4.ip.zxinc.org/getip",
+ "https://www.cloudflare.com/cdn-cgi/trace",
+ "https://1.1.1.1/cdn-cgi/trace",
+ "https://httpbin.org/ip",
+ "https://jsonip.com",
+ "https://ipv4.seeip.org",
+ "https://l2.io/ip",
+)
+_EGRESS_IPV4_TRIES = 8
+_IPV4_RE = re.compile(r"(?:\d{1,3}\.){3}\d{1,3}")
+ECHO_HEADERS = {
+ "Accept": "text/plain, application/json, */*",
+ "Accept-Language": "en-US,en;q=0.8,zh-CN;q=0.6",
+ "Cache-Control": "no-cache",
+ "Origin": None,
+ "Referer": None,
+}
+
+
+def _valid_public_ipv4(text: str) -> str:
+ try:
+ addr = ipaddress.ip_address(utils.trim(text))
+ except ValueError:
+ return ""
+ if addr.version != 4 or not addr.is_global:
+ return ""
+ return str(addr)
+
+
+def _parse_egress_ipv4(content: object) -> str:
+ if content is None:
+ return ""
+ if isinstance(content, dict):
+ chunks = []
+ for key in ("ip", "origin", "query", "address", "IPv4", "ipv4"):
+ value = content.get(key)
+ if value is not None:
+ chunks.append(str(value))
+ content = "\n".join(chunks) if chunks else json.dumps(content, ensure_ascii=False)
+ elif not isinstance(content, str):
+ content = str(content)
+
+ text = utils.trim(content)
+ if not text:
+ return ""
+ if text[0] in "{[":
+ try:
+ parsed = json.loads(text)
+ except Exception:
+ parsed = None
+ if isinstance(parsed, dict):
+ ip = _parse_egress_ipv4(parsed)
+ if ip:
+ return ip
+
+ # Cloudflare trace lists h=1.1.1.1 before ip=; prefer the ip field
+ for match in re.finditer(r"(?im)^ip=([0-9.]+)\s*$", text):
+ ip = _valid_public_ipv4(match.group(1))
+ if ip:
+ return ip
+
+ for match in _IPV4_RE.finditer(text):
+ ip = _valid_public_ipv4(match.group(0))
+ if ip:
+ return ip
+ return ""
+
+
+def resolve_egress_ipv4(port: int, request: RequestFn, max_retries: int = 2, timeout: int = 15) -> str:
+ urls = list(EGRESS_IPV4_URLS)
+ random.shuffle(urls)
+ tries = min(len(urls), max(_EGRESS_IPV4_TRIES, max_retries + 1))
+ for url in urls[:tries]:
+ success, content = request(
+ port=port,
+ url=url,
+ max_retries=1,
+ timeout=timeout,
+ headers=ECHO_HEADERS,
+ deserialize=False,
+ quiet=True,
+ )
+ ip = _parse_egress_ipv4(content) if success else ""
+ if ip:
+ return ip
+
+ return ""
+
+
+@dataclass
+class IPClassifyResult:
+ country_code: str = ""
+ company_type: str = ""
+ asn_type: str = ""
+ score: Optional[int] = None
+ raw: dict[str, object] = field(default_factory=dict)
+
+
+class IPLibrary:
+ name: str = ""
+ needs_egress_ip: bool = False
+
+ def fetch(
+ self, port: int, request: RequestFn, max_retries: int = 2, timeout: int = 12, ip: str = ""
+ ) -> dict[str, object]:
+ raise NotImplementedError
+
+ def classify(self, data: dict[str, object]) -> IPClassifyResult:
+ raise NotImplementedError
+
+ def _resolve_egress_ip(
+ self, port: int, request: RequestFn, max_retries: int = 2, timeout: int = 15, ip: str = ""
+ ) -> str:
+ text = _valid_public_ipv4(ip) if ip else ""
+ if text:
+ return text
+ return resolve_egress_ipv4(port, request, max_retries=max_retries, timeout=timeout)
+
+ def _get(
+ self,
+ request: RequestFn,
+ port: int,
+ url: str,
+ max_retries: int,
+ timeout: int,
+ headers: dict[str, str] | None = None,
+ deserialize: bool = True,
+ ) -> object | None:
+ success, response = request(
+ port=port,
+ url=url,
+ max_retries=max_retries,
+ timeout=timeout,
+ headers=headers,
+ deserialize=deserialize,
+ )
+ if not success:
+ return None
+ return response
+
+ @staticmethod
+ def _parse_trust_score(value: object, *, invert: bool = False) -> Optional[int]:
+ if value is None:
+ return None
+
+ try:
+ n = int(value)
+ except (TypeError, ValueError):
+ try:
+ n = int(float(value))
+ except (TypeError, ValueError):
+ return None
+
+ if invert:
+ n = 100 - n
+
+ return max(0, min(100, n))
+
+ @staticmethod
+ def _nested(data: object, *keys: str) -> dict[str, object]:
+ current = data
+ for key in keys:
+ if not isinstance(current, dict):
+ return {}
+ current = current.get(key)
+ return current if isinstance(current, dict) else {}
+
+
+class IPNetCoffeeLibrary(IPLibrary):
+ name = "ipnetcoffee"
+ needs_egress_ip = True
+
+ def fetch(
+ self, port: int, request: RequestFn, max_retries: int = 2, timeout: int = 12, ip: str = ""
+ ) -> dict[str, object]:
+ ip = self._resolve_egress_ip(port, request, max_retries=max_retries, timeout=timeout, ip=ip)
+ if not ip:
+ return {}
+
+ url = f"https://ip.net.coffee/api/ip/lookup/{urllib.parse.quote(ip, safe='')}"
+ response = self._get(request, port, url, max_retries, timeout)
+ return response if isinstance(response, dict) else {}
+
+ def classify(self, data: dict[str, object]) -> IPClassifyResult:
+ data = data if isinstance(data, dict) else {}
+ if data.get("isResidential") is True and utils.trim(data.get("company_type", "")) != "business":
+ company_type, asn_type = "isp", "isp"
+ else:
+ company_type, asn_type = "hosting", "hosting"
+
+ return IPClassifyResult(
+ country_code=utils.trim(data.get("countryCode", "")).upper(),
+ company_type=company_type,
+ asn_type=asn_type,
+ score=self._parse_trust_score(data.get("trust_score")),
+ raw=data,
+ )
+
+
+class MeowVPSLibrary(IPLibrary):
+ name = "meowvps"
+ needs_egress_ip = True
+
+ def fetch(
+ self, port: int, request: RequestFn, max_retries: int = 2, timeout: int = 12, ip: str = ""
+ ) -> dict[str, object]:
+ ip = self._resolve_egress_ip(port, request, max_retries=max_retries, timeout=timeout, ip=ip)
+ if not ip:
+ return {}
+
+ url = f"https://meowvps.com/api/ip-aggregator/{urllib.parse.quote(ip, safe='')}"
+ headers = {
+ "Accept": "*/*",
+ "Accept-Language": "zh-CN,zh;q=0.8",
+ "Origin": "https://meowvps.com",
+ "Referer": "https://meowvps.com/tools/ip-check/",
+ "User-Agent": (
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
+ "Chrome/152.0.0.0 Safari/537.36"
+ ),
+ }
+ response = self._get(request, port, url, max_retries, timeout, headers=headers)
+ if not isinstance(response, dict):
+ return {}
+ if response.get("success") is False:
+ logger.warning(f"MeowVPS lookup failed, ip: {ip}")
+ return {}
+
+ return response
+
+ def classify(self, data: dict[str, object]) -> IPClassifyResult:
+ data = data if isinstance(data, dict) else {}
+ core = self._nested(data, "core_data")
+ digital = self._nested(data, "api4", "digital")
+ traits = self._nested(data, "minfraud", "traits")
+ scores = self._nested(data, "risk_assessment", "ipdata", "scores")
+
+ digital_type = "" if digital.get("type") is None else str(digital.get("type")).strip().lower()
+ user_type = str(traits.get("user_type") or "").strip().lower()
+
+ if digital_type == "edu" or user_type in {"college", "education", "edu"}:
+ company_type, asn_type = "edu", "edu"
+ elif self._is_residential(digital_type, user_type, data):
+ company_type, asn_type = "isp", "isp"
+ else:
+ company_type, asn_type = "hosting", "hosting"
+
+ country_code = utils.trim(core.get("country_code") or digital.get("country_code") or "").upper()
+ return IPClassifyResult(
+ country_code=country_code,
+ company_type=company_type,
+ asn_type=asn_type,
+ score=self._parse_trust_score(scores.get("trust_score")),
+ raw=data,
+ )
+
+ @classmethod
+ def _is_residential(cls, digital_type: str, user_type: str, data: dict[str, object]) -> bool:
+ if digital_type in {"hosting", "edu"}:
+ return False
+ if user_type in {"hosting", "content_delivery_network", "college"}:
+ return False
+ if user_type in {"residential", "traveler", "cellular"}:
+ return True
+ if digital_type:
+ return False
+
+ ipapi = cls._nested(data, "risk_assessment", "ipapi")
+ threat = cls._nested(data, "risk_assessment", "ipdata", "threat")
+ if ipapi.get("hosting") is True or threat.get("is_datacenter") is True:
+ return False
+
+ return True
+
+
+class IPPureLibrary(IPLibrary):
+ name = "ippure"
+
+ def fetch(
+ self, port: int, request: RequestFn, max_retries: int = 2, timeout: int = 12, ip: str = ""
+ ) -> dict[str, object]:
+ response = self._get(request, port, "https://my.ippure.com/v1/info", max_retries, timeout)
+ return response if isinstance(response, dict) else {}
+
+ def classify(self, data: dict[str, object]) -> IPClassifyResult:
+ data = data if isinstance(data, dict) else {}
+ if data.get("isResidential", False):
+ company_type, asn_type = "isp", "isp"
+ else:
+ company_type, asn_type = "hosting", "hosting"
+
+ return IPClassifyResult(
+ country_code=utils.trim(data.get("countryCode", "")).upper(),
+ company_type=company_type,
+ asn_type=asn_type,
+ score=self._parse_trust_score(data.get("fraudScore"), invert=True),
+ raw=data,
+ )
+
+
+class IP2LocationLibrary(IPLibrary):
+ name = "ip2location"
+
+ def fetch(
+ self, port: int, request: RequestFn, max_retries: int = 2, timeout: int = 12, ip: str = ""
+ ) -> dict[str, object]:
+ headers = {"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"}
+ response = self._get(
+ request,
+ port,
+ "https://www.ip2location.com/demo",
+ max_retries,
+ timeout,
+ headers=headers,
+ deserialize=False,
+ )
+ if not isinstance(response, str):
+ return {}
+
+ data = self._extract_data(response)
+ if not data:
+ logger.warning("Failed to extract JSON payload from ip2location demo HTML")
+ return {}
+ return data
+
+ def classify(self, data: dict[str, object]) -> IPClassifyResult:
+ data = data if isinstance(data, dict) else {}
+ usage_type = utils.trim(data.get("usage_type", "")).lower()
+ as_info = data.get("as_info", {})
+ if not isinstance(as_info, dict):
+ as_info = {}
+ as_usage_type = utils.trim(as_info.get("as_usage_type", "")).lower()
+
+ check = lambda usage: usage.startswith("isp") or usage == "mob"
+ if check(usage_type) and check(as_usage_type):
+ company_type, asn_type = "isp", "isp"
+ else:
+ company_type, asn_type = "hosting", "hosting"
+
+ return IPClassifyResult(
+ country_code=utils.trim(data.get("country_code", "")).upper(),
+ company_type=company_type,
+ asn_type=asn_type,
+ score=self._parse_trust_score(data.get("fraud_score"), invert=True),
+ raw=data,
+ )
+
+ @staticmethod
+ def _extract_data(content: str) -> dict[str, object]:
+ if not content or not isinstance(content, str):
+ return {}
+
+ pattern = r']*class=["\'][^"\']*\blanguage-json\b[^"\']*["\'][^>]*>(.*?)\s* '
+ groups = re.findall(pattern, content, flags=re.I | re.S)
+ if not groups:
+ return {}
+
+ for group in groups:
+ payload = utils.trim(group)
+ if not payload:
+ continue
+
+ payload = re.sub(r"<[^>]+>", "", payload, flags=re.I | re.S)
+ payload = html.unescape(payload)
+
+ try:
+ data = json.loads(payload)
+ except Exception:
+ continue
+
+ if isinstance(data, dict) and data:
+ return data
+
+ return {}
+
+
+class IPLarkLibrary(IPLibrary):
+ name = "iplark"
+
+ def fetch(
+ self, port: int, request: RequestFn, max_retries: int = 2, timeout: int = 12, ip: str = ""
+ ) -> dict[str, object]:
+ response = self._get(request, port, "https://iplark.com/ipapi/public/ipinfo", max_retries, timeout)
+ return response if isinstance(response, dict) else {}
+
+ def classify(self, data: dict[str, object]) -> IPClassifyResult:
+ data = data if isinstance(data, dict) else {}
+ node_type = utils.trim(data.get("type", "")).lower()
+ if node_type == "isp":
+ company_type, asn_type = "isp", "isp"
+ elif node_type == "business":
+ company_type, asn_type = "business", "business"
+ else:
+ company_type, asn_type = "hosting", "hosting"
+
+ return IPClassifyResult(
+ country_code=utils.trim(data.get("country_code", "")).upper(),
+ company_type=company_type,
+ asn_type=asn_type,
+ raw=data,
+ )
+
+
+class IPInfoLibrary(IPLibrary):
+ name = "ipinfo"
+ needs_egress_ip = True
+
+ def fetch(
+ self, port: int, request: RequestFn, max_retries: int = 2, timeout: int = 12, ip: str = ""
+ ) -> dict[str, object]:
+ ip = self._resolve_egress_ip(port, request, max_retries=max_retries, timeout=timeout, ip=ip)
+ if not ip:
+ return {}
+
+ url = f"https://ipinfo.io/widget/demo/{ip}"
+ response = self._get(request, port, url, max_retries, timeout)
+ if not isinstance(response, dict):
+ return {}
+
+ data = response.get("data", response)
+ return data if isinstance(data, dict) else {}
+
+ def classify(self, data: dict[str, object]) -> IPClassifyResult:
+ data = data if isinstance(data, dict) else {}
+ company = data.get("company", {}) if isinstance(data.get("company"), dict) else {}
+ asn = data.get("asn", {}) if isinstance(data.get("asn"), dict) else {}
+ return IPClassifyResult(
+ country_code=utils.trim(data.get("country", "")).upper(),
+ company_type=utils.trim(company.get("type", "")).lower(),
+ asn_type=utils.trim(asn.get("type", "")).lower(),
+ raw=data,
+ )
+
+
+class IPApiLibrary(IPLibrary):
+ name = "ipapi"
+
+ def __init__(self, api_key: str = "") -> None:
+ self.api_key = utils.trim(api_key)
+
+ def fetch(
+ self, port: int, request: RequestFn, max_retries: int = 2, timeout: int = 12, ip: str = ""
+ ) -> dict[str, object]:
+ url = "https://api.ipapi.is"
+ if self.api_key:
+ url += f"?key={self.api_key}"
+
+ response = self._get(request, port, url, max_retries, timeout)
+ return response if isinstance(response, dict) else {}
+
+ def classify(self, data: dict[str, object]) -> IPClassifyResult:
+ data = data if isinstance(data, dict) else {}
+ location = data.get("location", {}) if isinstance(data.get("location"), dict) else {}
+ company = data.get("company", {}) if isinstance(data.get("company"), dict) else {}
+ asn = data.get("asn", {}) if isinstance(data.get("asn"), dict) else {}
+ return IPClassifyResult(
+ country_code=utils.trim(location.get("country_code", "")).upper(),
+ company_type=utils.trim(company.get("type", "")).lower(),
+ asn_type=utils.trim(asn.get("type", "")).lower(),
+ raw=data,
+ )
+
+
+PROVIDERS_ORDER = ["ipnetcoffee", "meowvps", "ippure", "ip2location", "ipinfo"]
+
+LIBRARIES = {
+ "ipnetcoffee": IPNetCoffeeLibrary,
+ "meowvps": MeowVPSLibrary,
+ "ippure": IPPureLibrary,
+ "ip2location": IP2LocationLibrary,
+ "iplark": IPLarkLibrary,
+ "ipinfo": IPInfoLibrary,
+ "ipapi": IPApiLibrary,
+}
+
+
+def get_providers(preferred: str) -> list[str]:
+ library = utils.trim(preferred).lower()
+ if library not in PROVIDERS_ORDER:
+ if library != "":
+ logger.warning(f"IP library {library} is not be supported")
+
+ library = "ipnetcoffee"
+
+ return [library] + [item for item in PROVIDERS_ORDER if item != library]
+
+
+def create_library(name: str, api_key: str = "") -> IPLibrary:
+ key = utils.trim(name).lower()
+ if key not in LIBRARIES:
+ key = "ipnetcoffee"
+
+ cls = LIBRARIES[key]
+ if cls is IPApiLibrary:
+ return IPApiLibrary(api_key=api_key)
+
+ return cls()
diff --git a/subscribe/location.py b/subscribe/location.py
index 1a00668397..54cb63b0d5 100644
--- a/subscribe/location.py
+++ b/subscribe/location.py
@@ -3,42 +3,56 @@
# @Author : wzdnzd
# @Time : 2024-07-12
+import gzip
+import http.client
import json
import math
import os
import random
import re
import socket
+import ssl
import subprocess
-import sys
import time
-import urllib
+import urllib.parse
from collections import defaultdict
from dataclasses import dataclass
+from functools import partial
+from typing import Optional
import utils
import yaml
from executable import which_bin
from geoip2 import database
+from iplibrary import create_library, get_providers, resolve_egress_ipv4
from logger import logger
from clash import is_mihomo
@dataclass
-class ProxyInfo:
- """Proxy query result information"""
+class GeoInfo:
+ """Country and CDN attributes of an IP"""
- name: str = ""
country: str = ""
+ is_cdn: bool = False
+
+
+@dataclass
+class ProxyInfo(GeoInfo):
+ """Proxy query result, including geo attributes"""
+
+ name: str = ""
ip_type: str = ""
+ score: Optional[int] = None
+ provider: str = ""
@dataclass
class ProxyQueryResult:
"""Complete proxy query result"""
- proxy: dict
+ proxy: dict[str, object]
result: ProxyInfo
success: bool
@@ -296,6 +310,34 @@ class ProxyQueryResult:
}
+# Pattern for CDN providers and Loyalsoldier custom ISO codes
+CDN_PATTERN = r"cloudflare|cloudfront|fastly|google"
+_CDN_NAME_RE = re.compile(CDN_PATTERN, flags=re.I)
+
+
+def is_cdn_label(value: str) -> bool:
+ """Return True if a country name or ISO code refers to a CDN instead of a location"""
+ text = utils.trim(value)
+ return bool(text and _CDN_NAME_RE.search(text))
+
+
+def _mark_cdn(proxy: dict[str, object]) -> None:
+ if isinstance(proxy, dict):
+ proxy["cdn"] = True
+
+
+def _is_cdn_proxy(proxy: dict[str, object]) -> bool:
+ return isinstance(proxy, dict) and (bool(proxy.get("cdn")) or is_cdn_label(str(proxy.get("name", ""))))
+
+
+def _remove_temp_flags(proxies: list[dict]) -> list[dict]:
+ for proxy in proxies:
+ if isinstance(proxy, dict):
+ proxy.pop("cdn", None)
+ proxy.pop("renamed", None)
+ return proxies
+
+
def download_mmdb(repo: str, target: str, filepath: str, retry: int = 3) -> bool:
"""
Download GeoLite2-City.mmdb from github release
@@ -387,51 +429,54 @@ def load_mmdb(
return database.Reader(filepath)
-def query_ip_country(ip: str, reader: database.Reader) -> str:
+def lookup_ip_geo(ip: str, reader: database.Reader) -> GeoInfo:
"""
Query country information for an IP address using mmdb database
- Args:
- ip: The IP address to query
- reader: The mmdb database reader
-
- Returns:
- The country name in Chinese
+ CDN ranges such as Cloudflare are reported via is_cdn and never as a country
"""
if not ip or not reader:
- return ""
+ return GeoInfo()
try:
# fake ip
if ip.startswith("198.18.0."):
logger.warning("cannot get geolocation because IP address is faked")
- return ""
+ return GeoInfo()
response = reader.country(ip)
+ names = response.country.names or {}
+ iso_code = utils.trim(response.country.iso_code).upper()
+ country = utils.trim(names.get("zh-CN", ""))
- # Try to get country name in Chinese
- country = response.country.names.get("zh-CN", "")
-
- # If Chinese name is not available, try to convert ISO code to Chinese country name
- if not country and response.country.iso_code:
- iso_code = response.country.iso_code
- # Try to get Chinese country name from ISO code mapping
+ if not country and iso_code:
country = ISO_TO_CHINESE.get(iso_code, iso_code)
- # Special handling for well-known IPs
- if not country:
- if ip == "1.1.1.1" or ip == "1.0.0.1":
- country = "Cloudflare"
- elif ip.startswith("8.8.8.") or ip.startswith("8.8.4."):
- country = "Google"
+ well_known_cdn = ip in ("1.1.1.1", "1.0.0.1") or ip.startswith("8.8.8.") or ip.startswith("8.8.4.")
+ if well_known_cdn or is_cdn_label(iso_code) or is_cdn_label(country):
+ return GeoInfo(is_cdn=True)
- return country
+ return GeoInfo(country=country)
except Exception as e:
logger.error(f"query ip country failed, ip: {ip}, error: {str(e)}")
- return ""
+ return GeoInfo()
-def locate_by_geoip(proxy: dict, reader: database.Reader) -> dict:
+def query_ip_country(ip: str, reader: database.Reader) -> str:
+ """
+ Query country information for an IP address using mmdb database
+
+ Args:
+ ip: The IP address to query
+ reader: The mmdb database reader
+
+ Returns:
+ The country name in Chinese
+ """
+ return lookup_ip_geo(ip, reader).country
+
+
+def locate_by_geoip(proxy: dict[str, object], reader: database.Reader) -> dict[str, object]:
if not proxy or not isinstance(proxy, dict):
return None
@@ -446,258 +491,82 @@ def locate_by_geoip(proxy: dict, reader: database.Reader) -> dict:
return proxy
ip = socket.gethostbyname(address)
- country = query_ip_country(ip, reader)
-
- if country:
- proxy["name"] = country
+ geo = lookup_ip_geo(ip, reader)
+ if geo.is_cdn:
+ _mark_cdn(proxy)
+ logger.debug(f"server IP belongs to CDN, skip as location, address: {address}")
+ elif geo.country:
+ proxy["name"] = geo.country
proxy["renamed"] = True
else:
- logger.warning(f"cannot get geolocation and rename, address: {address}")
+ logger.warning(f"cannot get geolocation and name, address: {address}")
except Exception as e:
logger.error(f"query ip geolocation failed, address: {address}, error: {str(e)}")
return proxy
-# Cache for checked port statuses
-_PORT_STATUS_CACHE = {}
-_AVAILABLE_PORTS = set()
-
-
-def get_listening_ports() -> set:
- """Get the set of listening ports in the system, cross-platform compatible"""
- listening_ports = set()
-
- try:
- # Windows system
- if os.name == "nt":
- try:
- # Use 'cp437' encoding to handle Windows command line output
- output = subprocess.check_output("netstat -an", shell=True).decode("cp437", errors="replace")
- for line in output.split("\n"):
- if "LISTENING" in line:
- parts = line.split()
- if len(parts) >= 2:
- addr_port = parts[1]
- if ":" in addr_port:
- try:
- port = int(addr_port.split(":")[-1])
- listening_ports.add(port)
- except ValueError:
- pass
- except Exception as e:
- logger.warning(f"Windows netstat command failed: {str(e)}")
- return listening_ports
-
- # macOS system
- elif sys.platform == "darwin":
- try:
- output = subprocess.check_output("lsof -i -P -n | grep LISTEN", shell=True).decode(
- "utf-8", errors="replace"
- )
- for line in output.split("\n"):
- if ":" in line:
- try:
- port_part = line.split(":")[-1].split(" ")[0]
- port = int(port_part)
- listening_ports.add(port)
- except (ValueError, IndexError):
- pass
- except Exception as e:
- logger.warning(f"macOS lsof command failed: {str(e)}")
- return listening_ports
-
- # Linux and other systems
- else:
- # Try using ss command (newer Linux systems)
- try:
- output = subprocess.check_output("ss -tuln", shell=True).decode("utf-8", errors="replace")
- for line in output.split("\n"):
- if "LISTEN" in line:
- parts = line.split()
- for part in parts:
- if ":" in part:
- try:
- port = int(part.split(":")[-1])
- listening_ports.add(port)
- except ValueError:
- pass
- except Exception as e:
- logger.warning(f"Linux ss command failed, trying netstat: {str(e)}")
- # Fall back to netstat command (older Linux systems)
- try:
- output = subprocess.check_output("netstat -tuln", shell=True).decode("utf-8", errors="replace")
- for line in output.split("\n"):
- if "LISTEN" in line:
- parts = line.split()
- for part in parts:
- if ":" in part:
- try:
- port = int(part.split(":")[-1])
- listening_ports.add(port)
- except ValueError:
- pass
- except Exception as e:
- logger.warning(f"Linux netstat command also failed: {str(e)}")
- return listening_ports
- except Exception as e:
- logger.warning(f"Failed to get listening ports: {str(e)}")
-
- return listening_ports
-
-
-def scan_ports_batch(start_port: int, count: int = 100) -> dict:
- """Batch scan port statuses, return a dictionary of port statuses"""
- global _PORT_STATUS_CACHE, _AVAILABLE_PORTS
-
- # Create a list of ports to scan (excluding ports with known status)
- ports_to_scan = [p for p in range(start_port, start_port + count) if p not in _PORT_STATUS_CACHE]
-
- if not ports_to_scan:
- # If all ports are already cached, return cached results directly
- return {p: _PORT_STATUS_CACHE.get(p, True) for p in range(start_port, start_port + count)}
-
- # Use a more efficient way to check ports in batch
- results = {}
+class PortReservation:
+ """Reserve local TCP ports by binding them without listen/connect."""
- try:
- # Get the ports that are currently listening in the system
- listening_ports = get_listening_ports()
-
- # Update results
- for port in ports_to_scan:
- in_use = port in listening_ports
- results[port] = in_use
- _PORT_STATUS_CACHE[port] = in_use
- if not in_use:
- _AVAILABLE_PORTS.add(port)
- except Exception as e:
- logger.warning(f"Batch port scanning failed, falling back to individual port checks: {str(e)}")
- # If batch checking fails, fall back to individual port checks
- for port in ports_to_scan:
- in_use = check_single_port(port)
- results[port] = in_use
- _PORT_STATUS_CACHE[port] = in_use
- if not in_use:
- _AVAILABLE_PORTS.add(port)
-
- # Merge cached and newly scanned results
- return {
- **{
- p: _PORT_STATUS_CACHE.get(p, True) for p in range(start_port, start_port + count) if p in _PORT_STATUS_CACHE
- },
- **results,
- }
+ def __init__(self) -> None:
+ self._sockets = []
+ def reserve(self, n: int) -> list[int]:
+ if n <= 0:
+ return []
-def check_single_port(port: int) -> bool:
- """Helper function for checking a single port, checks if the port is listening"""
- try:
- # Use socket to check TCP port
- sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- sock.settimeout(0.2)
- result = sock.connect_ex(("127.0.0.1", port))
- sock.close()
- if result == 0:
- return True
-
- # Also check IPv6
+ ports = []
try:
- sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
- sock.settimeout(0.2)
- result = sock.connect_ex(("::1", port))
- sock.close()
- return result == 0
- except:
- pass
-
- return False
- except:
- # Assume port is not in use when an error occurs
- return False
-
-
-def is_port_in_use(port: int) -> bool:
- """Check if a port is in use (using cache)"""
- global _PORT_STATUS_CACHE, _AVAILABLE_PORTS
-
- # If port is known to be available, return directly
- if port in _AVAILABLE_PORTS:
- return False
-
- # If port status is already cached, return directly
- if port in _PORT_STATUS_CACHE:
- return _PORT_STATUS_CACHE[port]
-
- # Otherwise check the port and cache the result
- in_use = check_single_port(port)
- _PORT_STATUS_CACHE[port] = in_use
- if not in_use:
- _AVAILABLE_PORTS.add(port)
- return in_use
+ for _ in range(n):
+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ sock.bind(("127.0.0.1", 0))
+ self._sockets.append(sock)
+ ports.append(sock.getsockname()[1])
+ return ports
+ except Exception:
+ self.close()
+ raise
+
+ def close(self) -> None:
+ sockets, self._sockets = self._sockets, []
+ for sock in sockets:
+ try:
+ sock.close()
+ except Exception:
+ pass
-def generate_mihomo_config(proxies: list[dict]) -> tuple[dict, dict]:
+def generate_mihomo_config(proxies: list[dict], listener_ports: list[int], mixed_port: int) -> tuple[dict, dict]:
"""Generate mihomo configuration for the given proxies"""
- # Base configuration
config = {
- "mixed-port": 7890,
+ "mixed-port": mixed_port,
"allow-lan": True,
"mode": "global",
"log-level": "error",
+ "ipv6": False,
+ "tcp-concurrent": True,
"proxies": proxies,
"dns": {
"enable": True,
- "enhanced-mode": "fake-ip",
- "fake-ip-range": "198.18.0.1/16",
+ "ipv6": False,
+ "enhanced-mode": "redir-host",
"default-nameserver": ["114.114.114.114", "223.5.5.5", "8.8.8.8"],
- "nameserver": ["https://doh.pub/dns-query"],
+ "nameserver": ["114.114.114.114", "8.8.8.8"],
},
"listeners": [],
}
- # Record the port assigned to each proxy
records = dict()
-
- # If there are no proxies, return directly
if not proxies:
return config, records
- # Pre-scan ports in batch to improve efficiency
- start_port = 32001
-
- # Scan enough ports to ensure there are sufficient available ports
- port_count = len(proxies) * 2
- port_status = scan_ports_batch(start_port, port_count)
-
- # Find all available ports
- available_ports = [p for p, in_use in port_status.items() if not in_use]
-
- # If available ports are insufficient, scan more ports
- if len(available_ports) < len(proxies):
- additional_ports = scan_ports_batch(start_port + port_count, port_count * 2)
- available_ports.extend([p for p, in_use in additional_ports.items() if not in_use])
-
- # Assign an available port to each proxy
for index, proxy in enumerate(proxies):
- if index < len(available_ports):
- port = available_ports[index]
- else:
- # If available ports are insufficient, use traditional method to find available ports
- port = start_port + port_count + index
- max_attempts = 1000
- attempts = 0
-
- while is_port_in_use(port) and attempts < max_attempts:
- port += 1
- attempts += 1
-
- if attempts >= max_attempts:
- logger.warning(
- f"Could not find an available port for proxy {proxy['name']} after {max_attempts} attempts"
- )
- continue
+ if index >= len(listener_ports):
+ logger.warning(f"No reserved port for proxy {proxy['name']}")
+ continue
+ port = listener_ports[index]
listener = {
"name": f"http-{index}",
"type": "http",
@@ -712,13 +581,233 @@ def generate_mihomo_config(proxies: list[dict]) -> tuple[dict, dict]:
return config, records
+def _idna_host(host: str) -> str:
+ host = utils.trim(host)
+ if not host:
+ return host
+ try:
+ return host.encode("idna").decode("ascii")
+ except Exception:
+ return host
+
+
+def _origin_headers(url: str, extra: dict[str, str] | None = None) -> dict[str, object]:
+ parsed = urllib.parse.urlparse(url)
+ base = f"{parsed.scheme}://{parsed.netloc}" if parsed.scheme and parsed.netloc else ""
+ result = {
+ "User-Agent": utils.USER_AGENT,
+ "Accept": "application/json, text/plain, */*",
+ "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+ "Accept-Encoding": "identity",
+ "Cache-Control": "no-cache",
+ "Pragma": "no-cache",
+ "Connection": "close",
+ "Referer": f"{base}/" if base else url,
+ "Origin": base if base else url,
+ }
+ if extra and isinstance(extra, dict):
+ for key, value in extra.items():
+ name = utils.trim(str(key))
+ if not name:
+ continue
+ if value is None:
+ result.pop(name, None)
+ else:
+ result[name] = value
+ return result
+
+
+def _proxy_ssl_context() -> ssl.SSLContext:
+ ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+ ctx.check_hostname = False
+ ctx.verify_mode = ssl.CERT_NONE
+ ctx.options |= ssl.OP_NO_TICKET
+ if hasattr(ssl, "OP_IGNORE_UNEXPECTED_EOF"):
+ ctx.options |= ssl.OP_IGNORE_UNEXPECTED_EOF
+ try:
+ ctx.set_alpn_protocols(["http/1.1"])
+ except Exception:
+ pass
+ return ctx
+
+
+def _recv_until(sock: socket.socket, marker: bytes, limit: int = 65536) -> bytes:
+ buf = bytearray()
+ while marker not in buf:
+ chunk = sock.recv(min(4096, max(1, limit - len(buf))))
+ if not chunk:
+ raise OSError(f"tunnel: incomplete proxy response: {bytes(buf)!r}")
+ buf += chunk
+ if len(buf) > limit:
+ raise OSError("tunnel: proxy response too large")
+ index = buf.find(marker) + len(marker)
+ if index < len(buf):
+ raise OSError("tunnel: unexpected data after CONNECT")
+ return bytes(buf[:index])
+
+
+def _tls_server_hostname(host: str) -> Optional[str]:
+ text = _idna_host(host)
+ if not text:
+ return None
+ try:
+ socket.inet_pton(socket.AF_INET, text)
+ return None
+ except OSError:
+ pass
+ if ":" in text:
+ try:
+ socket.inet_pton(socket.AF_INET6, text.strip("[]"))
+ return None
+ except OSError:
+ pass
+ return text
+
+
+def _open_http_tunnel(sock: socket.socket, host: str, port: int) -> None:
+ host = _idna_host(host)
+ target = f"{host}:{port}"
+ # Do not send Connection/Proxy-Connection: close. Mihomo hijacks after 200;
+ # a close flag makes Windows RST the socket (WinError 10054) at TLS.
+ sock.sendall(f"CONNECT {target} HTTP/1.1\r\nHost: {target}\r\n\r\n".encode("ascii"))
+ head = _recv_until(sock, b"\r\n\r\n")
+ status_line = head.split(b"\r\n", 1)[0].decode("iso-8859-1", "replace")
+ parts = status_line.split(" ", 2)
+ if len(parts) < 2:
+ raise OSError(f"tunnel: invalid CONNECT response: {status_line}")
+ try:
+ code = int(parts[1])
+ except ValueError as e:
+ raise OSError(f"tunnel: invalid CONNECT status: {status_line}") from e
+ if code != 200:
+ raise OSError(f"tunnel: CONNECT failed: {status_line}")
+
+
+def _assert_tunnel_open(sock: socket.socket, wait: float = 0.2) -> None:
+ """Fail fast if the CONNECT already died; otherwise wait out mihomo's peek."""
+ previous = sock.gettimeout()
+ sock.settimeout(max(wait, 0.01))
+ try:
+ peeked = sock.recv(1, socket.MSG_PEEK)
+ if not peeked:
+ raise OSError("tunnel: closed before TLS")
+ except (TimeoutError, socket.timeout):
+ return
+ except (ConnectionResetError, ConnectionAbortedError, BrokenPipeError) as e:
+ raise OSError(f"tunnel: closed before TLS ({e})") from e
+ finally:
+ sock.settimeout(previous)
+
+
+def _http_get_on_sock(sock: socket.socket, host: str, path: str, headers: dict[str, str]) -> tuple[int, str, bytes]:
+ host = _idna_host(host)
+ lines = [f"GET {path} HTTP/1.1", f"Host: {host}"]
+ sent = {"host"}
+ for key, value in (headers or {}).items():
+ name = utils.trim(str(key))
+ if not name or name.lower() in sent or name.lower().startswith("proxy-"):
+ continue
+ lines.append(f"{name}: {value}")
+ sent.add(name.lower())
+ if "connection" not in sent:
+ lines.append("Connection: close")
+
+ sock.sendall(("\r\n".join(lines) + "\r\n\r\n").encode("latin-1"))
+ response = http.client.HTTPResponse(sock, method="GET")
+ try:
+ response.begin()
+ return response.status, utils.trim(response.getheader("Location", "")), response.read()
+ finally:
+ try:
+ response.close()
+ except Exception:
+ pass
+
+
+def _decode_body(body: bytes) -> str:
+ if not body:
+ return ""
+ if body[:2] == b"\x1f\x8b":
+ try:
+ body = gzip.decompress(body)
+ except Exception:
+ pass
+ for encoding in ("utf-8", "gbk"):
+ try:
+ return body.decode(encoding)
+ except Exception:
+ continue
+ return body.decode("utf-8", "replace")
+
+
+def _request_through_proxy(
+ port: int, url: str, headers: dict[str, str], timeout: int, redirects: int = 3
+) -> tuple[int, bytes]:
+ """
+ Fetch URL via mihomo HTTP inbound.
+
+ http:// → absolute-form GET (plain HTTP proxy).
+ https:// → CONNECT then TLS in Python. Do not send `GET https://...`:
+ mihomo handles that with client.Do + TLS on net.Pipe, which races the
+ outbound dial and comes back as HTTP 502.
+ """
+ parsed = urllib.parse.urlparse(url)
+ scheme = utils.trim(parsed.scheme).lower()
+ host = parsed.hostname
+ if not host or scheme not in ("http", "https"):
+ raise OSError(f"http: unsupported url: {url}")
+
+ dst_port = parsed.port or (443 if scheme == "https" else 80)
+ path = parsed.path or "/"
+ if parsed.query:
+ path += "?" + parsed.query
+
+ sock = socket.create_connection(("127.0.0.1", port), timeout=timeout)
+ wrapped = None
+ try:
+ sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+ sock.settimeout(timeout)
+ target = sock
+ request_path = url
+ if scheme == "https":
+ _open_http_tunnel(sock, host, dst_port)
+ _assert_tunnel_open(sock)
+ try:
+ wrapped = _proxy_ssl_context().wrap_socket(
+ sock,
+ server_hostname=_tls_server_hostname(host),
+ suppress_ragged_eofs=True,
+ )
+ except OSError as e:
+ raise OSError(f"tls: {e}") from e
+ target = wrapped
+ request_path = path
+ status, location, body = _http_get_on_sock(target, host, request_path, headers)
+ finally:
+ for item in (wrapped, sock):
+ if item is None:
+ continue
+ try:
+ item.close()
+ except Exception:
+ pass
+
+ if status in (301, 302, 303, 307, 308) and redirects > 0 and location:
+ text = urllib.parse.urljoin(url, location)
+ if text.lower().startswith("http"):
+ return _request_through_proxy(port, text, _origin_headers(text, headers), timeout, redirects - 1)
+
+ return status, body
+
+
def make_proxy_request(
port: int,
url: str,
max_retries: int = 5,
timeout: int = 10,
- headers: dict = None,
+ headers: dict[str, str] = None,
deserialize: bool = True,
+ quiet: bool = False,
) -> tuple[bool, dict]:
"""
Make an HTTP request through a proxy and return the response
@@ -739,41 +828,26 @@ def make_proxy_request(
logger.warning("No port provided for proxy")
return False, {}
- # Configure the proxy for the request
- proxy_url = f"http://127.0.0.1:{port}"
- proxies_config = {"http": proxy_url, "https": proxy_url}
-
- # Configure proxy handler
- proxy_handler = urllib.request.ProxyHandler(proxies_config)
-
- # Build opener with proxy handler
- opener = urllib.request.build_opener(proxy_handler)
- if headers and isinstance(headers, dict):
- opener.addheaders = [(k, v) for k, v in headers.items() if k]
- else:
- opener.addheaders = [
- ("User-Agent", utils.USER_AGENT),
- ("Accept", "application/json"),
- ("Connection", "close"),
- ]
-
- # Try to get response with retry and backoff
+ default_headers = _origin_headers(url, headers)
attempt, success, data = 0, False, None
while not success and attempt < max(max_retries, 1):
try:
- # Random sleep to avoid being blocked by the API (increasing with each retry)
if attempt > 0:
wait_time = min(2**attempt * random.uniform(0.5, 1.5), 6)
time.sleep(wait_time)
- # Make request
- response = opener.open(url, timeout=timeout)
- if response.getcode() == 200:
- content = response.read().decode("utf-8")
- data = json.loads(content) if deserialize else content
- success = True
+ status, body = _request_through_proxy(port, url, default_headers, timeout)
+ if status != 200:
+ raise OSError(f"http: status {status}")
+ content = _decode_body(body)
+ data = json.loads(content) if deserialize else content
+ success = True
except Exception as e:
- logger.warning(f"Attempt {attempt+1} failed to request {url} through proxy port {port}: {str(e)}")
+ message = f"Attempt {attempt+1} failed to request {url} through proxy port {port}: {str(e)}"
+ if quiet or attempt + 1 < max(max_retries, 1):
+ logger.debug(message)
+ else:
+ logger.warning(message)
attempt += 1
@@ -781,44 +855,52 @@ def make_proxy_request(
def get_ipv4(port: int, max_retries: int = 5) -> str:
- """
- Get the IPv4 address by accessing https://api.ipify.org?format=json through a proxy
-
- Args:
- port: The port of the proxy
- max_retries: Maximum number of retry attempts
-
- Returns:
- The IPv4 address or empty string if failed
- """
+ """Get the egress IPv4 address through a proxy listener."""
if not port:
logger.warning("No port provided for proxy")
return ""
- success, data = make_proxy_request(port=port, url="https://api.ipify.org?format=json", max_retries=max_retries)
- return data.get("ip", "") if success else ""
+ return resolve_egress_ipv4(port, make_proxy_request, max_retries=max_retries, timeout=10)
+
+
+def _wait_listener(port: int, process: subprocess.Popen, timeout: int = 20) -> None:
+ deadline = time.time() + max(timeout, 1)
+ while time.time() < deadline:
+ if process.poll() is not None:
+ raise RuntimeError(f"mihomo exited before becoming ready, code={process.returncode}")
+ try:
+ sock = socket.create_connection(("127.0.0.1", port), timeout=1)
+ sock.close()
+ return
+ except OSError:
+ time.sleep(0.2)
+ raise RuntimeError(f"mihomo listener 127.0.0.1:{port} not ready after {timeout}s")
# Online API services for IP location
LOCATION_API_SERVICES = [
{"url": "https://ipinfo.io", "country_key": "country"},
- {"url": "https://ipapi.co/json/", "country_key": "country_code"},
+ {"url": "https://api.ip2location.io", "country_key": "country_code"},
{"url": "https://ipwho.is", "country_key": "country_code"},
- {"url": "https://freeipapi.com/api/json", "country_key": "countryCode"},
- {"url": "https://api.country.is", "country_key": "country"},
+ {"url": "https://free.freeipapi.com/api/json", "country_key": "countryCode"},
{"url": "https://api.ip.sb/geoip", "country_key": "country_code"},
]
-# Pattern for CDN providers
-CDN_PATTERN = r"cloudflare|cloudfront|fastly|google"
-
-def random_delay(min_delay: float = 0.01, max_delay: float = 0.5):
+def random_delay(min_delay: float = 0.01, max_delay: float = 0.5) -> None:
"""Random delay to avoid API rate limiting"""
time.sleep(random.uniform(min_delay, max_delay))
-def check_residential(proxy: dict, port: int, api_key: str = "", use_ipinfo: bool = True) -> ProxyQueryResult:
+def check_residential(
+ proxy: dict[str, object],
+ port: int,
+ api_key: str = "",
+ ip_library: str = "ipnetcoffee",
+ reader: database.Reader = None,
+ max_retries: int = 2,
+ timeout: int = 12,
+) -> ProxyQueryResult:
"""
Check if a proxy is residential by making a request through it
@@ -826,40 +908,14 @@ def check_residential(proxy: dict, port: int, api_key: str = "", use_ipinfo: boo
proxy: The proxy information dict
port: The port of the proxy
api_key: Optional API key for ipapi.is. Uses free tier if not provided
- use_ipinfo: Whether to use ipinfo.io instead of ipapi.is, defaults to True
+ ip_library: IP query provider, supported: ipnetcoffee/meowvps/ippure/ip2location/iplark/ipinfo/ipapi
+ reader: Optional mmdb reader used to detect CDN egress IPs
+ max_retries: Retry count for provider queries
+ timeout: Timeout in seconds for provider queries
Returns:
ProxyQueryResult: Complete proxy query result
"""
-
- def _get_ipapi_url(key: str = "") -> str:
- url, key = "https://api.ipapi.is", utils.trim(key)
- if key:
- url += f"?key={key}"
- return url
-
- def _get_ipinfo_url(port: int, name: str) -> str:
- # First, get the IP address
- success, content = make_proxy_request(
- port=port,
- url="https://ipinfo.io/ip",
- max_retries=2,
- timeout=15,
- deserialize=False,
- )
- if not success or not content:
- logger.warning(f"Failed to get IP from ipinfo.io for proxy {name}")
- return ""
-
- # Extract IP from response
- ip = utils.trim(content)
- if not ip:
- logger.warning(f"Invalid IP address from ipinfo.io for proxy {name}")
- return ""
-
- # Now get detailed information using the IP
- return f"https://ipinfo.io/widget/demo/{ip}"
-
name = proxy.get("name", "")
result = ProxyInfo(name=name)
@@ -867,49 +923,91 @@ def _get_ipinfo_url(port: int, name: str) -> str:
logger.warning(f"No port found for proxy {name}")
return ProxyQueryResult(proxy=proxy, result=result, success=False)
- # Random delay to avoid being blocked by the API
random_delay()
try:
- url = ""
- if use_ipinfo:
- url = _get_ipinfo_url(port=port, name=name)
-
- if not url:
- url = _get_ipapi_url(key=api_key)
- use_ipinfo = False
-
- # Call API for IP information through the proxy
- success, response = make_proxy_request(port=port, url=url, max_retries=2, timeout=12)
-
- # Parse data from response
- if success:
- try:
- data = response.get("data", {}) if use_ipinfo else response
+ providers = get_providers(ip_library)
+ classified, provider = None, ""
+ egress_ip = None
+ request = partial(make_proxy_request, quiet=True)
+
+ def _cached_egress_ip() -> str:
+ nonlocal egress_ip
+ if egress_ip is None:
+ egress_ip = resolve_egress_ipv4(
+ port,
+ request,
+ max_retries=max_retries,
+ timeout=max(timeout, 15),
+ )
+ if not egress_ip:
+ logger.debug(f"Failed to get egress IP for proxy {name}")
+ return egress_ip
- # Extract country code from data
- if use_ipinfo:
- country_code = data.get("country", "")
+ if reader:
+ ip = _cached_egress_ip()
+ if ip and lookup_ip_geo(ip, reader).is_cdn:
+ result.is_cdn = True
+ _mark_cdn(proxy)
+ logger.debug(f"Egress IP for proxy {name} belongs to CDN, continue locating")
+ return ProxyQueryResult(proxy=proxy, result=result, success=False)
+
+ for idx, item in enumerate(providers):
+ library = create_library(item, api_key=api_key)
+ ip = _cached_egress_ip() if library.needs_egress_ip else ""
+ if library.needs_egress_ip and not ip:
+ if idx < len(providers) - 1:
+ fallback = providers[idx + 1]
+ logger.debug(f"Skip {item} for proxy {name}, no egress IP, trying fallback: {fallback}")
else:
- country_code = data.get("location", {}).get("country_code", "")
+ logger.debug(f"Skip {item} for proxy {name}, no egress IP")
+ continue
- result.country = ISO_TO_CHINESE.get(country_code, "") if country_code else ""
+ data = library.fetch(port, request, max_retries=max_retries, timeout=timeout, ip=ip)
+ if data:
+ provider = item
+ classified = library.classify(data)
+ logger.debug(f"IP info for proxy {name} successfully retrieved, provider: {provider}")
+ break
- company_type = data.get("company", {}).get("type", "")
- asn_type = data.get("asn", {}).get("type", "")
+ if idx < len(providers) - 1:
+ fallback = providers[idx + 1]
+ logger.debug(f"Failed to query {item} for proxy {name}, trying fallback: {fallback}")
+ else:
+ logger.debug(f"Failed to query {item} for proxy {name}")
- # Check if it's residential (both company and asn type should be "isp")
+ if classified:
+ try:
+ result.provider = utils.trim(provider)
+ country_code = utils.trim(classified.country_code).upper()
+ if country_code:
+ result.country = ISO_TO_CHINESE.get(country_code, "")
+
+ if not result.country:
+ raw = classified.raw or {}
+ result.country = utils.trim(
+ raw.get("country_zh", "") or raw.get("country", "") or raw.get("country_name", "")
+ )
+
+ company_type = utils.trim(classified.company_type).lower()
+ asn_type = utils.trim(classified.asn_type).lower()
if company_type == "isp" and asn_type == "isp":
result.ip_type = "isp"
elif company_type in ["business", "isp"] and asn_type in ["business", "isp"]:
result.ip_type = "business"
+ result.score = classified.score
except Exception as e:
- logger.error(f"Error parsing {url} response for proxy {name}: {str(e)}")
+ logger.error(f"Error parsing response for proxy {name}: {str(e)}")
else:
- logger.warning(f"Failed to query {url} for proxy {name}")
+ logger.warning(f"Failed to query residential info for proxy {name} with providers: {providers}")
+
+ if is_cdn_label(result.country):
+ result.is_cdn = True
+ _mark_cdn(proxy)
+ logger.debug(f"Residential country for proxy {name} is CDN, continue locating")
+ return ProxyQueryResult(proxy=proxy, result=result, success=False)
- # Determine if query was successful
flag = result.country != "" or result.ip_type != ""
return ProxyQueryResult(proxy=proxy, result=result, success=flag)
@@ -918,90 +1016,67 @@ def _get_ipinfo_url(port: int, name: str) -> str:
return ProxyQueryResult(proxy=proxy, result=result, success=False)
-def locate_by_ipinfo(proxy: dict, port: int, reader: database.Reader = None) -> ProxyQueryResult:
+def locate_by_ipinfo(proxy: dict[str, object], port: int, reader: database.Reader = None) -> ProxyQueryResult:
"""Check the location of a single proxy by making a request through it"""
+ name = proxy.get("name", "")
+
+ is_cdn = _is_cdn_proxy(proxy)
- def _create_failed_result(reason: str = "") -> ProxyQueryResult:
- """Helper to create failed query result"""
- name = proxy.get("name", "")
+ def _failed(reason: str = "") -> ProxyQueryResult:
if reason:
logger.warning(f"Location query failed for proxy {name}: {reason}")
- return ProxyQueryResult(proxy=proxy, result=ProxyInfo(name=name), success=False)
+ if is_cdn:
+ _mark_cdn(proxy)
+ return ProxyQueryResult(proxy=proxy, result=ProxyInfo(name=name, is_cdn=is_cdn), success=False)
- def _create_success_result(country: str) -> ProxyQueryResult:
- """Helper to create successful query result"""
- info = ProxyInfo(name=proxy.get("name", ""))
- info.country = country
+ def _success(country: str) -> ProxyQueryResult:
+ info = ProxyInfo(name=name, country=country)
return ProxyQueryResult(proxy=proxy, result=info, success=True)
- def _try_local_mmdb_lookup() -> str:
- """Attempt to get country from local MMDB database"""
- if not reader:
- return ""
+ if not port:
+ return _failed("No port specified")
- ip = get_ipv4(port=port, max_retries=2)
- if ip:
- return query_ip_country(ip, reader) or ""
- return ""
+ random_delay()
- def _try_online_api_services() -> str:
- """Attempt to get country from online API services with retry logic"""
- retries = 3
+ try:
+ if reader:
+ ip = get_ipv4(port=port, max_retries=2)
+ geo = lookup_ip_geo(ip, reader) if ip else GeoInfo()
+ if geo.is_cdn:
+ is_cdn = True
+ _mark_cdn(proxy)
+ logger.debug(f"Egress IP for proxy {name} belongs to CDN, try online APIs")
+ elif geo.country:
+ logger.debug(f"Location found via MMDB for proxy {name}: {geo.country}")
+ return _success(geo.country)
+ retries = 3
for attempt in range(retries):
- # Select a random service for this attempt
service = random.choice(LOCATION_API_SERVICES)
-
- # Make the API request
- success, data = make_proxy_request(port=port, url=service["url"], max_retries=1, timeout=12)
-
+ success, data = make_proxy_request(port=port, url=service["url"], max_retries=1, timeout=12, quiet=True)
if success and data:
- # Parse country code from response
- key = service["country_key"]
- code = data.get(key, "")
-
+ code = data.get(service["country_key"], "")
if code:
- # Convert to Chinese country name
- return ISO_TO_CHINESE.get(code, code)
+ country = ISO_TO_CHINESE.get(code, code)
+ if is_cdn_label(code) or is_cdn_label(country):
+ is_cdn = True
+ _mark_cdn(proxy)
+ logger.debug(f"API country for proxy {name} is CDN, continue locating")
+ else:
+ logger.debug(f"Location found via API for proxy {name}: {country}")
+ return _success(country)
- # Handle retry delay for failed attempts
if attempt < retries - 1:
delay = min(2**attempt * random.uniform(1, 2), 6)
- logger.warning(
- f"API attempt {attempt+1} failed for proxy {proxy.get('name', '')} "
- f"using {service['url']}, retrying in {delay:.2f}s"
+ logger.debug(
+ f"API attempt {attempt+1} failed for proxy {name} using {service['url']}, retrying in {delay:.2f}s"
)
time.sleep(delay)
- return ""
-
- # Validate input parameters
- if not port:
- return _create_failed_result("No port specified")
-
- # Apply rate limiting
- random_delay()
-
- # Main location detection logic
- try:
- # Strategy 1: Try local MMDB database first (faster and more reliable)
- country = _try_local_mmdb_lookup()
- if country:
- logger.debug(f"Location found via MMDB for proxy {proxy.get('name', '')}: {country}")
- return _create_success_result(country)
-
- # Strategy 2: Fall back to online API services
- country = _try_online_api_services()
- if country:
- logger.debug(f"Location found via API for proxy {proxy.get('name', '')}: {country}")
- return _create_success_result(country)
-
- # No location detected from any source
- return _create_failed_result("Unable to determine location from any source")
-
+ return _failed("Unable to determine location from any source")
except Exception as e:
- logger.error(f"Unexpected error during location query for {proxy.get('name', '')}: {str(e)}")
- return _create_failed_result(f"Exception: {str(e)}")
+ logger.error(f"Unexpected error during location query for {name}: {str(e)}")
+ return _failed(f"Exception: {str(e)}")
def batch_query(
@@ -1011,8 +1086,6 @@ def batch_query(
show_progress: bool = True,
description: str = "Querying",
digits: int = 2,
- reader: database.Reader = None,
- api_key: str = "",
) -> list[ProxyQueryResult]:
"""
Run mihomo to query proxies information using the specified function
@@ -1024,84 +1097,87 @@ def batch_query(
show_progress: Whether to show progress
description: Description for progress display
digits: Number of digits for proxy naming
- reader: Optional mmdb reader for locate_by_ipinfo
- api_key: Optional API key for check_residential function
Returns:
List of ProxyQueryResult with complete information
"""
- if not proxies or not is_mihomo():
+ if not proxies:
return []
- # Rename proxies for consistent naming
- nodes = rename(proxies, digits, False)
+ if not is_mihomo():
+ return [
+ ProxyQueryResult(proxy=proxy, result=ProxyInfo(name=proxy.get("name", "")), success=False)
+ for proxy in proxies
+ ]
- logger.info(f"Generate clash listeners configuration for {len(nodes)} proxies")
- # Generate mihomo configuration
- config, records = generate_mihomo_config(nodes)
+ nodes = rename(proxies, digits, False)
+ failed = [ProxyQueryResult(proxy=proxy, result=ProxyInfo(name=proxy["name"]), success=False) for proxy in nodes]
- # Save the configuration to clash/config.yaml in the project directory
workspace = os.path.join(os.path.abspath(os.path.dirname(os.path.dirname(__file__))), "clash")
- config_path = os.path.join(workspace, "config.yaml")
- with open(config_path, "w", encoding="utf-8") as f:
- yaml.dump(config, f, allow_unicode=True)
-
- logger.info(f"Mihomo configuration saved to {config_path}")
-
- # Check if we can find the mihomo binary
mihomo_bin = os.path.join(workspace, which_bin()[0])
if not os.path.exists(mihomo_bin) or not os.path.isfile(mihomo_bin):
logger.error("Mihomo binary not found, skipping proxy check")
- return [ProxyQueryResult(proxy=proxy, result=ProxyInfo(name=proxy["name"]), success=False) for proxy in nodes]
+ return failed
- # Make the binary executable
utils.chmod(mihomo_bin)
- # Start mihomo with the configuration
- logger.info(f"Starting mihomo with configuration {config_path}")
+ logger.info(f"Generate clash listeners configuration for {len(nodes)} proxies")
+ reservation = PortReservation()
process = None
try:
- # Run mihomo in background
+ ports = reservation.reserve(1 + len(nodes))
+ mixed_port, listener_ports = ports[0], ports[1:]
+ config, records = generate_mihomo_config(nodes, listener_ports, mixed_port)
+
+ config_path = os.path.join(workspace, "config.yaml")
+ with open(config_path, "w", encoding="utf-8") as f:
+ yaml.dump(config, f, allow_unicode=True)
+
+ logger.info(f"Mihomo configuration saved to {config_path}")
+ reservation.close()
+
+ logger.info(f"Starting mihomo with configuration {config_path}")
process = subprocess.Popen(
[mihomo_bin, "-d", workspace, "-f", config_path],
stdout=subprocess.DEVNULL,
- stderr=subprocess.PIPE,
+ stderr=subprocess.DEVNULL,
)
- # Wait longer to ensure mihomo is fully started
logger.info("Waiting for mihomo to start...")
- time.sleep(8)
+ _wait_listener(mixed_port, process)
+ if listener_ports:
+ _wait_listener(listener_ports[0], process, timeout=10)
- # Create proxy info mapping for task generation
mappings = {proxy["name"]: proxy for proxy in nodes}
-
- # Generate tasks for each proxy
- if reader is not None:
- # For locate_by_ipinfo which needs reader parameter
- tasks = [(mappings[name], port, reader) for name, port in records.items() if name in mappings]
- elif api_key:
- # For check_residential with API key
- tasks = [(mappings[name], port, api_key) for name, port in records.items() if name in mappings]
- else:
- # For check_residential without API key
- tasks = [(mappings[name], port) for name, port in records.items() if name in mappings]
-
- # Check proxies using the specified function
- results = utils.multi_thread_run(
- func=func,
- tasks=tasks,
- num_threads=num_threads,
- show_progress=show_progress,
- description=description,
+ tasks = [(mappings[name], port) for name, port in records.items() if name in mappings]
+ results = (
+ utils.multi_thread_run(
+ func=func,
+ tasks=tasks,
+ num_threads=num_threads,
+ show_progress=show_progress,
+ description=description,
+ )
+ or []
)
- return results
+ queried, normalized = set(), []
+ for item in results:
+ if not item:
+ continue
+ normalized.append(item)
+ queried.add(item.proxy.get("name"))
+
+ for proxy in nodes:
+ if proxy["name"] not in queried:
+ normalized.append(ProxyQueryResult(proxy=proxy, result=ProxyInfo(name=proxy["name"]), success=False))
+ return normalized
except Exception as e:
logger.error(f"Error during mihomo check: {str(e)}")
- return [ProxyQueryResult(proxy=proxy, result=ProxyInfo(name=proxy["name"]), success=False) for proxy in nodes]
+ return failed
finally:
- # Always try to kill the mihomo process
+ reservation.close()
if process:
try:
process.terminate()
@@ -1110,13 +1186,16 @@ def batch_query(
pass
-def process_query_results(results: list[ProxyQueryResult], strategy: str) -> tuple[list[dict], list[dict]]:
+def process_query_results(
+ results: list[ProxyQueryResult], strategy: str, score: bool = False
+) -> tuple[list[dict], list[dict]]:
"""
Process proxy query results
Args:
results: List of query results
strategy: Processing strategy ('residential' or 'location')
+ score: Whether to prefix node names with provider and trust score
Returns:
tuple: (list of successful proxies, list of failed proxies)
@@ -1124,28 +1203,36 @@ def process_query_results(results: list[ProxyQueryResult], strategy: str) -> tup
successes, fails = [], []
for item in results:
- if item.success and item.result.country:
+ if not item:
+ continue
+
+ country = utils.trim(item.result.country) if item.result else ""
+ is_cdn = bool(item.result and item.result.is_cdn) or is_cdn_label(country)
+ if is_cdn:
+ _mark_cdn(item.proxy)
+
+ if item.success and country and not is_cdn:
# Copy proxy info to avoid modifying original data
proxy = item.proxy.copy()
if strategy == "residential":
# Residential IP check strategy
- name = item.result.country
+ name = country
if item.result.ip_type == "isp":
name += "家宽"
elif item.result.ip_type == "business":
name += "商宽"
-
- proxy["name"] = name
- successes.append(proxy)
- elif strategy == "location":
- # Location check strategy
- proxy["name"] = item.result.country
- successes.append(proxy)
else:
- # Unknown strategy, use query result directly
- proxy["name"] = item.result.country
- successes.append(proxy)
+ # Location check or unknown strategy
+ name = country
+
+ if score and item.result.score is not None:
+ source = utils.trim(item.result.provider).upper()
+ if source:
+ name = f"[{source}|{str(item.result.score).zfill(3)}] {name}"
+
+ proxy["name"] = name
+ successes.append(proxy)
else:
# Failed query proxies
fails.append(item.proxy)
@@ -1161,36 +1248,46 @@ def regularize(
show_progress: bool = True,
locate: bool = False,
residential: bool = False,
+ ip_library: str = "",
digits: int = 2,
+ score: bool = False,
) -> list[dict]:
if not proxies or not isinstance(proxies, list):
return proxies
# Phase 1: Residential check if necessary
successes, fails = [], []
+ reader = None
if residential:
- logger.info(f"Starting residential check for {len(proxies)} proxies")
-
- # Enable locate if residential check is enabled
locate = True
+ if residential or locate:
+ directory = utils.trim(directory)
+ if not directory:
+ directory = os.path.join(os.path.abspath(os.path.dirname(os.path.dirname(__file__))), "data")
+ reader = load_mmdb(directory=directory, repo="Loyalsoldier/geoip", filename="Country.mmdb", update=update)
+ if not reader:
+ logger.error("cannot load mmdb: Country.mmdb")
+
+ if residential:
+ logger.info(f"Starting residential check for {len(proxies)} proxies")
+
# Get https://api.ipapi.is API key from environment variable
api_key = utils.trim(os.environ.get("IPAPI_IS_API_KEY", ""))
# Use mihomo to check for residential proxies
results = batch_query(
proxies=proxies,
- func=check_residential,
+ func=partial(check_residential, api_key=api_key, ip_library=ip_library, reader=reader),
num_threads=num_threads,
show_progress=show_progress,
description="Checking residential",
digits=digits,
- api_key=api_key,
)
# Process residential check results
- successes, fails = process_query_results(results, "residential")
+ successes, fails = process_query_results(results, "residential", score=score)
logger.info(f"Residential check completed: {len(successes)} successful, {len(fails)} failed")
else:
fails = proxies
@@ -1199,60 +1296,50 @@ def regularize(
if locate and fails:
logger.info(f"Starting location check for {len(fails)} proxies")
- # Initialize reader for locate functionality and load mmdb database if available
- directory = utils.trim(directory)
- if not directory:
- directory = os.path.join(os.path.abspath(os.path.dirname(os.path.dirname(__file__))), "data")
-
- repo, filename = "Loyalsoldier/geoip", "Country.mmdb"
- reader = load_mmdb(directory=directory, repo=repo, filename=filename, update=update)
- if not reader:
- logger.error(f"Skipping location check due to cannot load mmdb: {filename}")
-
unconfirmed = list()
if reader:
# Try local mmdb lookup first
- tasks = [[p, reader] for p in fails if p and isinstance(p, dict)]
+ sources = [p for p in fails if p and isinstance(p, dict)]
+ tasks = [[p, reader] for p in sources]
mmdb_results = utils.multi_thread_run(locate_by_geoip, tasks, num_threads, show_progress, "")
- # Separate confirmed and unconfirmed proxies by regex
- regex = f"中国|{CDN_PATTERN}"
-
- for proxy in mmdb_results:
- if proxy.pop("renamed", False) and not re.search(regex, proxy["name"], flags=re.I):
- # Add to successes list if confirmed by mmdb lookup
- successes.append(proxy)
+ for source, proxy in zip(sources, mmdb_results or []):
+ node = proxy if proxy and isinstance(proxy, dict) else source
+ name = str(node.get("name", ""))
+ cdn = bool(node.get("cdn")) or is_cdn_label(name)
+ if cdn:
+ _mark_cdn(node)
+ if node.pop("renamed", False) and "中国" not in name and not cdn:
+ successes.append(node)
else:
- # Add to unconfirmed list if not confirmed by mmdb lookup
- unconfirmed.append(proxy)
+ unconfirmed.append(node)
else:
# No mmdb available, treat all as unconfirmed
unconfirmed = fails
- # For unconfirmed proxies, use online API services to get location info (fallback)
+ # For unconfirmed proxies, use online API services to get location info
if unconfirmed:
logger.info(f"Using online API services for {len(unconfirmed)} unconfirmed proxies")
# Use mihomo to check IP locations
query_results = batch_query(
proxies=unconfirmed,
- func=locate_by_ipinfo,
+ func=partial(locate_by_ipinfo, reader=reader),
num_threads=num_threads,
show_progress=show_progress,
description="Querying location",
digits=digits,
- reader=reader,
)
# Process location check results and handle CDN proxies
- query_successes, query_fails = process_query_results(query_results, "location")
+ query_successes, query_fails = process_query_results(query_results, "location", score=score)
# Add query successes to final results
successes.extend(query_successes)
- # Handle CDN proxies that failed location check
+ # CDN nodes without a real country fall back to US
for proxy in query_fails:
- if re.search(CDN_PATTERN, proxy["name"], flags=re.I):
+ if _is_cdn_proxy(proxy):
logger.warning(f"Failed to get location for proxy {proxy['name']}, assume it's in US")
proxy["name"] = "美国"
@@ -1264,7 +1351,7 @@ def regularize(
successes.extend(fails)
# Return final results
- return rename(proxies=successes, digits=digits, shuffle=True)
+ return rename(proxies=_remove_temp_flags(successes), digits=digits, shuffle=True)
def rename(proxies: list[dict], digits: int = 2, shuffle: bool = False) -> list[dict]:
diff --git a/subscribe/logger.py b/subscribe/logger.py
index be0122ccb8..c805d761fd 100644
--- a/subscribe/logger.py
+++ b/subscribe/logger.py
@@ -16,7 +16,7 @@
class Logger:
- def __init__(self):
+ def __init__(self) -> None:
self._logger = logging.getLogger()
if not self._logger.handlers:
self.formatter = logging.Formatter(fmt=DEFAULT_LOG_FMT, datefmt=DEFAULT_LOG_DATETIME_FORMAT)
@@ -28,20 +28,20 @@ def __init__(self):
if sys.version_info.major == 2:
logging.getLogger("requests").setLevel(logging.WARNING)
- def _get_file_handler(self, filename):
+ def _get_file_handler(self, filename: str) -> logging.FileHandler:
"""返回一个文件日志handler"""
file_handler = logging.FileHandler(filename=filename, encoding="utf8")
file_handler.setFormatter(self.formatter)
return file_handler
- def _get_console_handler(self):
+ def _get_console_handler(self) -> logging.StreamHandler:
"""返回一个输出到终端日志handler"""
console_handler = logging.StreamHandler(sys.stdout)
console_handler.setFormatter(self.formatter)
return console_handler
@property
- def logger(self):
+ def logger(self) -> logging.Logger:
return self._logger
diff --git a/subscribe/mailtm.py b/subscribe/mailtm.py
index 88492b67cc..dc1c91a17a 100644
--- a/subscribe/mailtm.py
+++ b/subscribe/mailtm.py
@@ -48,16 +48,16 @@ class TemporaryMail(object):
def __init__(self) -> None:
self.api_address = ""
- def get_domains_list(self) -> list:
+ def get_domains_list(self) -> list[str]:
raise NotImplementedError
def get_account(self, retry: int = 3) -> Account:
raise NotImplementedError
- def get_messages(self, account: Account) -> list:
+ def get_messages(self, account: Account) -> list[Message]:
raise NotImplementedError
- def monitor_account(self, account: Account, timeout: int = 300, sleep: int = 3) -> Message:
+ def monitor_account(self, account: Account, timeout: int = 300, sleep: int = 3) -> Message | None:
"""keep waiting for new messages"""
if not account:
return None
@@ -122,7 +122,7 @@ def __init__(self) -> None:
"User-Agent": utils.USER_AGENT,
}
- def get_domains_list(self) -> list:
+ def get_domains_list(self) -> list[str]:
content, count = "", 1
while not content and count <= 3:
count += 1
@@ -180,7 +180,7 @@ def get_account(self, retry: int = 3) -> Account:
except:
return self.get_account(retry=retry - 1)
- def get_messages(self, account: Account) -> list:
+ def get_messages(self, account: Account) -> list[Message]:
if not account:
return []
@@ -261,7 +261,7 @@ class SnapMail(TemporaryMail):
def __init__(self) -> None:
self.api_address = "https://snapmail.cc"
- def get_domains_list(self) -> list:
+ def get_domains_list(self) -> list[str]:
domains = ["snapmail.cc", "lista.cc", "xxxhi.cc"]
# content = utils.http_get(
# url="https://www.snapmail.cc/scripts/controllers/addEmailBox.js", retry=1
@@ -287,7 +287,7 @@ def get_account(self, retry: int = 3) -> Account:
return Account(address=address)
- def get_messages(self, account: Account) -> list:
+ def get_messages(self, account: Account) -> list[Message]:
if not account:
return []
@@ -351,7 +351,7 @@ class LinShiEmail(TemporaryMail):
def __init__(self) -> None:
self.api_address = "https://linshiyouxiang.net"
- def get_domains_list(self) -> list:
+ def get_domains_list(self) -> list[str]:
content = utils.http_get(url=self.api_address)
if not content:
return []
@@ -370,7 +370,7 @@ def get_account(self, retry: int = 3) -> Account:
return Account(address=address)
- def get_messages(self, account: Account) -> list:
+ def get_messages(self, account: Account) -> list[Message]:
if not account:
return []
@@ -415,7 +415,7 @@ def __init__(self) -> None:
self.api_address = "https://api.mail.tm"
self.auth_headers = {}
- def get_domains_list(self) -> list:
+ def get_domains_list(self) -> list[str]:
headers = {"Accept": "application/ld+json"}
try:
content = utils.http_get(url=f"{self.api_address}/domains?page=1", headers=headers)
@@ -450,7 +450,7 @@ def _make_account_request(self, endpoint: str, address: str, password: str, retr
except:
return self._make_account_request(endpoint=endpoint, address=address, password=password, retry=retry - 1)
- def _generate_jwt(self, address: str, password: str, retry: int = 3):
+ def _generate_jwt(self, address: str, password: str, retry: int = 3) -> dict[str, object] | None:
jwt = self._make_account_request(endpoint="token", address=address, password=password, retry=retry)
if not jwt:
logger.error(f"[JWTError] generate jwt token failed, domain: {self.api_address}")
@@ -479,7 +479,7 @@ def get_account(self, retry: int = 3) -> Account:
return account
- def get_messages(self, account: Account) -> list:
+ def get_messages(self, account: Account) -> list[Message]:
"""download a list of messages currently in the account."""
if not account or not self.auth_headers:
return []
@@ -566,7 +566,7 @@ def __init__(self) -> None:
"User-Agent": utils.USER_AGENT,
}
- def get_domains_list(self) -> list:
+ def get_domains_list(self) -> list[str]:
content = utils.http_get(url=self.api_address)
if not content:
return []
@@ -611,7 +611,7 @@ def get_account(self, retry: int = 3) -> Account:
username, domain = address.split("@", maxsplit=1)
return self._make_account_request(username=username, domain=domain, retry=retry)
- def get_messages(self, account: Account) -> list:
+ def get_messages(self, account: Account) -> list[Message]:
if not account:
return []
@@ -654,7 +654,7 @@ def __init__(self, onlygmail: bool = False) -> None:
"Referer": "https://www.emailnator.com/",
}
- def get_domains_list(self) -> list:
+ def get_domains_list(self) -> list[str]:
# unable to obtain the supported email domain through web api
return ["gmail.com", "googlemail.com", "smartnator.com", "psnator.com", "tmpmailtor.com", "mydefipet.live"]
@@ -715,7 +715,7 @@ def get_account(self, retry: int = 3) -> Account:
except:
return self.get_account(retry=retry - 1)
- def get_messages(self, account: Account) -> list:
+ def get_messages(self, account: Account) -> list[Message]:
if not account:
return []
try:
diff --git a/subscribe/outbound/__init__.py b/subscribe/outbound/__init__.py
new file mode 100644
index 0000000000..d72599d120
--- /dev/null
+++ b/subscribe/outbound/__init__.py
@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+
+from outbound.anytls import AnyTLSVerifier
+from outbound.base import proxy_exists, register, verify
+from outbound.builtin import DirectVerifier, DnsVerifier, RejectVerifier, RematchVerifier
+from outbound.common import QuotedStr, endpoint_key, quoted_scalar
+from outbound.hysteria import Hysteria2Verifier, HysteriaVerifier
+from outbound.http import HttpVerifier, Socks5Verifier
+from outbound.mieru import MieruVerifier
+from outbound.overlay import EasyTierVerifier, TailscaleVerifier, ZeroTierVerifier
+from outbound.quic import ShadowQuicVerifier, TuicVerifier
+from outbound.shadowsocks import ShadowsocksRVerifier, ShadowsocksVerifier
+from outbound.snell import SnellVerifier
+from outbound.ssh import SshVerifier
+from outbound.sudoku import SudokuVerifier
+from outbound.trojan import TrojanVerifier
+from outbound.tunnel import GostRelayVerifier, TrustTunnelVerifier
+from outbound.vless import VlessVerifier
+from outbound.vmess import VmessVerifier
+from outbound.vpn import MasqueVerifier, OpenVPNVerifier, WireGuardVerifier
+
+register(
+ ShadowsocksVerifier(),
+ ShadowsocksRVerifier(),
+ SnellVerifier(),
+ VmessVerifier(),
+ VlessVerifier(),
+ TrojanVerifier(),
+ HttpVerifier(),
+ Socks5Verifier(),
+ AnyTLSVerifier(),
+ HysteriaVerifier(),
+ Hysteria2Verifier(),
+ TuicVerifier(),
+ ShadowQuicVerifier(),
+ SshVerifier(),
+ MieruVerifier(),
+ SudokuVerifier(),
+ TrustTunnelVerifier(),
+ GostRelayVerifier(),
+ WireGuardVerifier(),
+ OpenVPNVerifier(),
+ MasqueVerifier(),
+ EasyTierVerifier(),
+ TailscaleVerifier(),
+ ZeroTierVerifier(),
+ DirectVerifier(),
+ DnsVerifier(),
+ RejectVerifier(),
+ RematchVerifier(),
+)
+
+__all__ = ["QuotedStr", "endpoint_key", "proxy_exists", "quoted_scalar", "verify"]
diff --git a/subscribe/outbound/anytls.py b/subscribe/outbound/anytls.py
new file mode 100644
index 0000000000..03821566fc
--- /dev/null
+++ b/subscribe/outbound/anytls.py
@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+from outbound.base import OutboundVerifier, VerifyContext
+from outbound.common import ensure_non_negative_number
+from outbound.tls import verify_optional_str_dict
+
+
+class AnyTLSVerifier(OutboundVerifier):
+ type_name = "anytls"
+ mihomo_only = True
+
+ def verify_fields(self, item: dict[str, object], ctx: VerifyContext) -> bool:
+ for field in ("idle-session-check-interval", "idle-session-timeout", "min-idle-session"):
+ if not ensure_non_negative_number(item, field):
+ return False
+ if not verify_optional_str_dict(item, "shadow-tls-opts"):
+ return False
+ if not verify_optional_str_dict(item, "restls-opts"):
+ return False
+ if not verify_optional_str_dict(item, "jls-opts"):
+ return False
+ return True
diff --git a/subscribe/outbound/base.py b/subscribe/outbound/base.py
new file mode 100644
index 0000000000..b41794f12d
--- /dev/null
+++ b/subscribe/outbound/base.py
@@ -0,0 +1,118 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+import utils
+from outbound.common import (
+ check_common_optional,
+ check_optional_port,
+ check_required_port,
+ endpoint_key,
+ finalize_auth,
+ normalize_name,
+ normalize_server,
+)
+
+
+class VerifyContext:
+ def __init__(self, is_mihomo: bool = True) -> None:
+ self.is_mihomo = is_mihomo
+
+
+class OutboundVerifier(ABC):
+ type_name: str = ""
+ mihomo_only: bool = False
+ require_server: bool = True
+ require_port: bool = True
+
+ def verify(self, item: dict[str, object], ctx: VerifyContext) -> bool:
+ if self.mihomo_only and not ctx.is_mihomo:
+ return False
+
+ item.pop("dialer-proxy", None)
+
+ if not normalize_name(item):
+ return False
+
+ if (self.needs_server(item) or item.get("server")) and not normalize_server(item):
+ return False
+
+ if self.needs_port(item):
+ if not check_required_port(item, item.get("type", self.type_name)):
+ return False
+ elif not check_optional_port(item, item.get("type", self.type_name)):
+ return False
+
+ if not check_common_optional(item):
+ return False
+ if not self.verify_fields(item, ctx):
+ return False
+ return finalize_auth(item, self.auth_field(item))
+
+ def needs_server(self, item: dict[str, object]) -> bool:
+ return self.require_server
+
+ def needs_port(self, item: dict[str, object]) -> bool:
+ return self.require_port
+
+ @abstractmethod
+ def verify_fields(self, item: dict[str, object], ctx: VerifyContext) -> bool:
+ raise NotImplementedError
+
+ def auth_field(self, item: dict[str, object]) -> str | None:
+ return "password"
+
+ def duplicate_key(self, item: dict[str, object]) -> tuple[str, object]:
+ field = self.auth_field(item)
+ secret = item.get(field, "") if field else ""
+ return (self.type_name, secret)
+
+
+VERIFIERS: dict[str, OutboundVerifier] = {}
+
+
+def get_verifier(type_name: str) -> OutboundVerifier | None:
+ return VERIFIERS.get(utils.trim(str(type_name or "")).lower())
+
+
+def register(*verifiers: OutboundVerifier) -> None:
+ for verifier in verifiers:
+ VERIFIERS[verifier.type_name] = verifier
+
+
+def verify(item: dict[str, object], is_mihomo: bool = True) -> bool:
+ if not item or type(item) != dict or "type" not in item:
+ return False
+
+ verifier = get_verifier(str(item.get("type", "")))
+ if verifier is None:
+ return False
+
+ try:
+ return verifier.verify(item, VerifyContext(is_mihomo=is_mihomo))
+ except Exception:
+ return False
+
+
+def proxy_exists(proxy: dict[str, object], hosts: dict[str, list[dict[str, object]]]) -> bool:
+ if not proxy:
+ return True
+ if not hosts:
+ return False
+
+ existing = hosts.get(endpoint_key(proxy), [])
+ if not existing:
+ return False
+
+ protocol = utils.trim(str(proxy.get("type", ""))).lower()
+ if protocol in ("http", "socks5"):
+ return True
+
+ verifier = get_verifier(protocol)
+ if verifier is None:
+ return False
+
+ current = verifier.duplicate_key(proxy)
+ return any(p.get("type") == protocol and verifier.duplicate_key(p) == current for p in existing)
diff --git a/subscribe/outbound/builtin.py b/subscribe/outbound/builtin.py
new file mode 100644
index 0000000000..ec327d0470
--- /dev/null
+++ b/subscribe/outbound/builtin.py
@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+from outbound.base import OutboundVerifier, VerifyContext
+from outbound.common import ensure_str
+
+
+class _BuiltinVerifier(OutboundVerifier):
+ mihomo_only = True
+ require_server = False
+ require_port = False
+
+ def verify_fields(self, item: dict[str, object], ctx: VerifyContext) -> bool:
+ return True
+
+ def auth_field(self, item: dict[str, object]) -> str | None:
+ return None
+
+ def duplicate_key(self, item: dict[str, object]) -> tuple[str, object]:
+ return (self.type_name, item.get("name", ""))
+
+
+class DirectVerifier(_BuiltinVerifier):
+ type_name = "direct"
+
+
+class DnsVerifier(_BuiltinVerifier):
+ type_name = "dns"
+
+
+class RejectVerifier(_BuiltinVerifier):
+ type_name = "reject"
+
+
+class RematchVerifier(_BuiltinVerifier):
+ type_name = "rematch"
+
+ def verify_fields(self, item: dict[str, object], ctx: VerifyContext) -> bool:
+ return ensure_str(item, "target-rematch-name") and ensure_str(item, "target-sub-rule")
diff --git a/subscribe/outbound/common.py b/subscribe/outbound/common.py
new file mode 100644
index 0000000000..d8c74c2818
--- /dev/null
+++ b/subscribe/outbound/common.py
@@ -0,0 +1,231 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+import ipaddress
+import re
+
+import utils
+
+BOOL_FIELDS = ("udp", "tls", "skip-cert-verify", "tfo", "mptcp")
+IP_VERSIONS = ("dual", "ipv4", "ipv6", "ipv4-prefer", "ipv6-prefer")
+CONGESTION_CONTROLLERS = ("cubic", "bbr", "new_reno")
+BBR_PROFILES = ("", "standard", "conservative", "aggressive")
+TRAFFIC_PATTERN = re.compile(r"^\d+(\.\d+)?(\s+)?([kmgt]?bps)?$", re.I)
+
+
+class QuotedStr(str):
+ pass
+
+
+def quoted_scalar(dumper: object, data: object) -> object:
+ return dumper.represent_scalar("tag:yaml.org,2002:str", data, style='"')
+
+
+def wrap(text: object) -> str:
+ if utils.is_number(text):
+ text = str(text)
+ return utils.trim(text)
+
+
+def normalize_name(item: dict[str, object]) -> bool:
+ name = str(item.get("name", "")).strip()
+ if not name:
+ return False
+ item["name"] = name
+ return True
+
+
+def normalize_server(item: dict[str, object]) -> bool:
+ server = str(item.get("server", "")).strip().lower()
+ if not server:
+ return False
+
+ if server.startswith("::"):
+ # ipv6 addresses starting with "::" can break yaml loading
+ try:
+ server = ipaddress.IPv6Address(server).exploded
+ except Exception:
+ return False
+
+ item["server"] = server
+ return True
+
+
+def check_ports(port: object, ranges: object, protocol: str) -> bool:
+ protocol = utils.trim(protocol).lower()
+
+ try:
+ flag = 0 < int(port) <= 65535
+ if not flag or protocol not in ["hysteria", "hysteria2"] or not ranges:
+ return flag
+ except Exception:
+ return False
+
+ return parse_port_ranges(ranges)
+
+
+def parse_port_ranges(ranges: object) -> bool:
+ text = wrap(ranges)
+ if not text:
+ return False
+
+ nums = re.split(r"/|,", text)
+ if not nums:
+ return False
+
+ for num in nums:
+ start, end = num, num
+ if "-" in num:
+ start, end = num.split("-", maxsplit=1)
+ try:
+ start, end = int(start), int(end)
+ if start <= 0 or start > 65535 or end <= 0 or end > 65535 or start > end:
+ return False
+ except Exception:
+ return False
+
+ return True
+
+
+def check_required_port(item: dict[str, object], protocol: str) -> bool:
+ return check_ports(item.get("port", ""), item.get("ports", None), protocol)
+
+
+def check_optional_port(item: dict[str, object], protocol: str) -> bool:
+ if "port" not in item or item.get("port") in (None, ""):
+ return True
+ return check_ports(item.get("port", ""), item.get("ports", None), protocol)
+
+
+def check_common_optional(item: dict[str, object]) -> bool:
+ if "uuid" in item and not utils.verify_uuid(item.get("uuid")):
+ return False
+
+ for attribute in ["servername", "sni"]:
+ if attribute in item and type(item[attribute]) != str:
+ return False
+
+ for attribute in BOOL_FIELDS:
+ if attribute in item and type(item[attribute]) != bool:
+ return False
+
+ if "ip-version" in item:
+ version = utils.trim(str(item.get("ip-version", "")))
+ if version and version not in IP_VERSIONS:
+ return False
+
+ if "alpn" in item and type(item["alpn"]) != list:
+ return False
+
+ return True
+
+
+def finalize_auth(item: dict[str, object], field: str | None) -> bool:
+ if not field:
+ return True
+ if not item.get(field, ""):
+ return False
+ if utils.is_number(item[field]):
+ item[field] = QuotedStr(item[field])
+ return True
+
+
+def quote_numeric_fields(item: dict[str, object], fields: tuple[str, ...]) -> None:
+ for field in fields:
+ if field in item and item.get(field, "") != "" and utils.is_number(item[field]):
+ item[field] = QuotedStr(item[field])
+
+
+def ensure_non_negative_number(item: dict[str, object], field: str) -> bool:
+ if field not in item:
+ return True
+ if not utils.is_number(item[field]) or int(item[field]) < 0:
+ return False
+ return True
+
+
+def ensure_number(item: dict[str, object], field: str) -> bool:
+ if field not in item:
+ return True
+ return utils.is_number(item[field])
+
+
+def ensure_bool(item: dict[str, object], field: str) -> bool:
+ if field not in item:
+ return True
+ return type(item[field]) == bool
+
+
+def ensure_str(item: dict[str, object], field: str) -> bool:
+ if field not in item:
+ return True
+ return type(item[field]) == str
+
+
+def is_valid_ip(text: str) -> bool:
+ text = utils.trim(text)
+ if not text:
+ return False
+ try:
+ ipaddress.ip_address(text)
+ return True
+ except Exception:
+ return False
+
+
+def verify_traffic(item: dict[str, object], field: str) -> bool:
+ if field not in item:
+ return True
+
+ traffic = item.get(field, "")
+ if traffic == "null":
+ item.pop(field)
+ return True
+
+ if traffic and utils.is_number(traffic):
+ traffic = str(traffic)
+ item[field] = traffic
+
+ return bool(TRAFFIC_PATTERN.match(utils.trim(traffic)))
+
+
+def verify_congestion_controller(item: dict[str, object]) -> bool:
+ if "congestion-controller" not in item:
+ return True
+ return item.get("congestion-controller") in CONGESTION_CONTROLLERS
+
+
+def verify_bbr_profile(item: dict[str, object]) -> bool:
+ if "bbr-profile" not in item:
+ return True
+ profile = wrap(item.get("bbr-profile", ""))
+ return profile in BBR_PROFILES
+
+
+def verify_hop_interval(item: dict[str, object]) -> bool:
+ if "hop-interval" not in item:
+ return True
+ value = item.get("hop-interval")
+ if utils.is_number(value):
+ return float(value) >= 0
+
+ text = wrap(value)
+ if not text or "," in text:
+ return False
+ if "-" not in text:
+ return False
+ start, end = text.split("-", maxsplit=1)
+ try:
+ start, end = float(start), float(end)
+ return start > 0 and end >= start
+ except Exception:
+ return False
+
+
+def endpoint_key(item: dict[str, object]) -> str:
+ server = item.get("server")
+ port = item.get("port")
+ if server:
+ return f"{server}:{port}"
+ return f"{item.get('type')}:{item.get('name')}"
diff --git a/subscribe/outbound/http.py b/subscribe/outbound/http.py
new file mode 100644
index 0000000000..f31f3a96c4
--- /dev/null
+++ b/subscribe/outbound/http.py
@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+import utils
+from outbound.base import OutboundVerifier, VerifyContext
+from outbound.common import QuotedStr
+
+
+class _UserPassVerifier(OutboundVerifier):
+ def verify_fields(self, item: dict[str, object], ctx: VerifyContext) -> bool:
+ for field in ("username", "password"):
+ value = item.get(field, None)
+ if not value:
+ continue
+ if not isinstance(value, str) and not utils.is_number(value):
+ return False
+ if utils.is_number(value):
+ item[field] = QuotedStr(value)
+ else:
+ item[field] = utils.trim(value)
+ return True
+
+ def auth_field(self, item: dict[str, object]) -> str | None:
+ return None
+
+ def duplicate_key(self, item: dict[str, object]) -> tuple[str, object]:
+ return (self.type_name,)
+
+
+class HttpVerifier(_UserPassVerifier):
+ type_name = "http"
+
+
+class Socks5Verifier(_UserPassVerifier):
+ type_name = "socks5"
diff --git a/subscribe/outbound/hysteria.py b/subscribe/outbound/hysteria.py
new file mode 100644
index 0000000000..b0668ddd5f
--- /dev/null
+++ b/subscribe/outbound/hysteria.py
@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+from outbound.base import OutboundVerifier, VerifyContext
+from outbound.common import (
+ ensure_bool,
+ ensure_number,
+ ensure_str,
+ parse_port_ranges,
+ verify_bbr_profile,
+ verify_hop_interval,
+ verify_traffic,
+ wrap,
+)
+
+HYSTERIA_PROTOCOLS = ("udp", "wechat-video", "faketcp")
+HYSTERIA2_OBFS = ("salamander", "gecko")
+
+
+class HysteriaVerifier(OutboundVerifier):
+ type_name = "hysteria"
+ mihomo_only = True
+
+ def verify_fields(self, item: dict[str, object], ctx: VerifyContext) -> bool:
+ if not verify_traffic(item, "up") or not verify_traffic(item, "down"):
+ return False
+ for field in ("ca", "ca-str", "auth-str", "auth_str", "obfs"):
+ if not ensure_str(item, field):
+ return False
+ for field in ("disable_mtu_discovery", "fast-open"):
+ if not ensure_bool(item, field):
+ return False
+ if "protocol" in item and wrap(item.get("protocol", "")) not in HYSTERIA_PROTOCOLS:
+ return False
+ if "ports" in item and not parse_port_ranges(item.get("ports")):
+ return False
+ for field in ("recv_window_conn", "recv-window-conn", "recv_window", "recv-window"):
+ if not ensure_number(item, field):
+ return False
+ return True
+
+ def auth_field(self, item: dict[str, object]) -> str | None:
+ return "auth-str" if "auth-str" in item else "auth_str"
+
+ def duplicate_key(self, item: dict[str, object]) -> tuple[str, object]:
+ field = self.auth_field(item)
+ return (self.type_name, item.get(field, ""))
+
+
+class Hysteria2Verifier(OutboundVerifier):
+ type_name = "hysteria2"
+ mihomo_only = True
+
+ def verify_fields(self, item: dict[str, object], ctx: VerifyContext) -> bool:
+ if not verify_traffic(item, "up") or not verify_traffic(item, "down"):
+ return False
+ for field in ("ca", "ca-str", "obfs-password"):
+ if not ensure_str(item, field):
+ return False
+ if "ports" in item and not parse_port_ranges(item.get("ports")):
+ return False
+ if not verify_hop_interval(item) or not verify_bbr_profile(item):
+ return False
+
+ obfs = wrap(item.get("obfs", ""))
+ if obfs:
+ if obfs not in HYSTERIA2_OBFS:
+ return False
+ if not wrap(item.get("obfs-password", "")):
+ return False
+ return True
diff --git a/subscribe/outbound/mieru.py b/subscribe/outbound/mieru.py
new file mode 100644
index 0000000000..aee210ee7e
--- /dev/null
+++ b/subscribe/outbound/mieru.py
@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+from outbound.base import OutboundVerifier, VerifyContext
+from outbound.common import parse_port_ranges, quote_numeric_fields, wrap
+
+MIERU_TRANSPORTS = ("TCP", "UDP")
+MIERU_MULTIPLEXING = (
+ "",
+ "MULTIPLEXING_OFF",
+ "MULTIPLEXING_LOW",
+ "MULTIPLEXING_MIDDLE",
+ "MULTIPLEXING_HIGH",
+)
+MIERU_HANDSHAKE = ("", "HANDSHAKE_STANDARD", "HANDSHAKE_NO_WAIT")
+
+
+class MieruVerifier(OutboundVerifier):
+ type_name = "mieru"
+ mihomo_only = True
+ require_port = False
+
+ def verify_fields(self, item: dict[str, object], ctx: VerifyContext) -> bool:
+ has_port = "port" in item and item.get("port") not in (None, "")
+ has_range = bool(wrap(item.get("port-range", "")))
+ if has_port == has_range:
+ return False
+ if has_range and not parse_port_ranges(item.get("port-range")):
+ return False
+ if not wrap(item.get("username", "")):
+ return False
+ quote_numeric_fields(item, ("username",))
+
+ transport = wrap(item.get("transport", "TCP")).upper() or "TCP"
+ if transport not in MIERU_TRANSPORTS:
+ return False
+ if "multiplexing" in item and wrap(item.get("multiplexing", "")) not in MIERU_MULTIPLEXING:
+ return False
+ if "handshake-mode" in item and wrap(item.get("handshake-mode", "")) not in MIERU_HANDSHAKE:
+ return False
+ return True
+
+ def duplicate_key(self, item: dict[str, object]) -> tuple[str, object]:
+ return (self.type_name, item.get("username", ""), item.get("password", ""))
diff --git a/subscribe/outbound/overlay.py b/subscribe/outbound/overlay.py
new file mode 100644
index 0000000000..d85e8535c6
--- /dev/null
+++ b/subscribe/outbound/overlay.py
@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+from outbound.base import OutboundVerifier, VerifyContext
+from outbound.common import wrap
+
+
+class EasyTierVerifier(OutboundVerifier):
+ type_name = "easytier"
+ mihomo_only = True
+ require_server = False
+ require_port = False
+
+ def verify_fields(self, item: dict[str, object], ctx: VerifyContext) -> bool:
+ if not wrap(item.get("network-name", "")):
+ return False
+ peers = item.get("peers")
+ listeners = item.get("listeners")
+ has_peers = isinstance(peers, list) and any(wrap(p) if not isinstance(p, dict) else True for p in peers)
+ has_listeners = isinstance(listeners, list) and listeners
+ return bool(has_peers or has_listeners)
+
+ def auth_field(self, item: dict[str, object]) -> str | None:
+ return None
+
+ def duplicate_key(self, item: dict[str, object]) -> tuple[str, object]:
+ return (self.type_name, item.get("network-name", ""), item.get("network-secret", ""))
+
+
+class TailscaleVerifier(OutboundVerifier):
+ type_name = "tailscale"
+ mihomo_only = True
+ require_server = False
+ require_port = False
+
+ def verify_fields(self, item: dict[str, object], ctx: VerifyContext) -> bool:
+ return True
+
+ def auth_field(self, item: dict[str, object]) -> str | None:
+ return None
+
+ def duplicate_key(self, item: dict[str, object]) -> tuple[str, object]:
+ return (self.type_name, item.get("auth-key", ""), item.get("hostname", ""), item.get("control-url", ""))
+
+
+class ZeroTierVerifier(OutboundVerifier):
+ type_name = "zerotier"
+ mihomo_only = True
+ require_server = False
+ require_port = False
+
+ def verify_fields(self, item: dict[str, object], ctx: VerifyContext) -> bool:
+ network = wrap(item.get("network", ""))
+ return len(network) == 16
+
+ def auth_field(self, item: dict[str, object]) -> str | None:
+ return None
+
+ def duplicate_key(self, item: dict[str, object]) -> tuple[str, object]:
+ return (self.type_name, item.get("network", ""), item.get("identity-secret", ""))
diff --git a/subscribe/outbound/quic.py b/subscribe/outbound/quic.py
new file mode 100644
index 0000000000..392ed9d065
--- /dev/null
+++ b/subscribe/outbound/quic.py
@@ -0,0 +1,86 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+from outbound.base import OutboundVerifier, VerifyContext
+from outbound.common import (
+ ensure_bool,
+ ensure_number,
+ is_valid_ip,
+ quote_numeric_fields,
+ verify_bbr_profile,
+ verify_congestion_controller,
+ verify_traffic,
+ wrap,
+)
+
+TUIC_UDP_RELAY_MODES = ("native", "quic")
+SHADOWQUIC_VERSIONS = ("v1", "v2")
+
+
+class TuicVerifier(OutboundVerifier):
+ type_name = "tuic"
+ mihomo_only = True
+
+ def verify_fields(self, item: dict[str, object], ctx: VerifyContext) -> bool:
+ token = wrap(item.get("token", ""))
+ uuid = wrap(item.get("uuid", ""))
+ password = wrap(item.get("password", ""))
+
+ if token:
+ if uuid or password:
+ return False
+ item["token"] = token
+ elif uuid:
+ if not password:
+ return False
+ item["password"] = password
+ else:
+ return False
+
+ for field in ("disable-sni", "reduce-rtt", "fast-open"):
+ if not ensure_bool(item, field):
+ return False
+ for field in ("heartbeat-interval", "request-timeout", "max-udp-relay-packet-size", "max-open-streams"):
+ if not ensure_number(item, field):
+ return False
+ if "udp-relay-mode" in item and item["udp-relay-mode"] not in TUIC_UDP_RELAY_MODES:
+ return False
+ if not verify_congestion_controller(item) or not verify_bbr_profile(item):
+ return False
+ if "ip" in item and not is_valid_ip(item.get("ip", "")):
+ return False
+ return True
+
+ def auth_field(self, item: dict[str, object]) -> str | None:
+ return "token" if wrap(item.get("token", "")) else "uuid"
+
+ def duplicate_key(self, item: dict[str, object]) -> tuple[str, object]:
+ if wrap(item.get("token", "")):
+ return (self.type_name, item.get("token", ""))
+ return (self.type_name, item.get("uuid", ""))
+
+
+class ShadowQuicVerifier(OutboundVerifier):
+ type_name = "shadowquic"
+ mihomo_only = True
+
+ def verify_fields(self, item: dict[str, object], ctx: VerifyContext) -> bool:
+ if not wrap(item.get("username", "")):
+ return False
+ quote_numeric_fields(item, ("username",))
+ if not verify_congestion_controller(item) or not verify_bbr_profile(item):
+ return False
+ if not verify_traffic(item, "up") or not verify_traffic(item, "down"):
+ return False
+ if "quic-versions" in item:
+ versions = item.get("quic-versions")
+ if type(versions) != list:
+ return False
+ for version in versions:
+ if wrap(version) not in SHADOWQUIC_VERSIONS:
+ return False
+ return True
+
+ def duplicate_key(self, item: dict[str, object]) -> tuple[str, object]:
+ return (self.type_name, item.get("username", ""), item.get("password", ""))
diff --git a/subscribe/outbound/shadowsocks.py b/subscribe/outbound/shadowsocks.py
new file mode 100644
index 0000000000..091fda5192
--- /dev/null
+++ b/subscribe/outbound/shadowsocks.py
@@ -0,0 +1,161 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+import base64
+import re
+
+import utils
+from outbound.base import OutboundVerifier, VerifyContext
+
+COMMON_SS_SUPPORTED_CIPHERS = [
+ "aes-128-gcm",
+ "aes-192-gcm",
+ "aes-256-gcm",
+ "aes-128-cfb",
+ "aes-192-cfb",
+ "aes-256-cfb",
+ "aes-128-ctr",
+ "aes-192-ctr",
+ "aes-256-ctr",
+ "rc4-md5",
+ "chacha20-ietf",
+ "xchacha20",
+ "chacha20-ietf-poly1305",
+ "xchacha20-ietf-poly1305",
+]
+
+# reference: https://github.com/SagerNet/sing-shadowsocks2/blob/dev/shadowaead_2022/method.go
+MIHOMO_SS_SUPPORTED_CIPHERS_SALT_LEN = {
+ "2022-blake3-aes-128-gcm": 16,
+ "2022-blake3-aes-256-gcm": 32,
+ "2022-blake3-chacha20-poly1305": 32,
+}
+
+MIHOMO_SS_SUPPORTED_CIPHERS = (
+ COMMON_SS_SUPPORTED_CIPHERS
+ + list(MIHOMO_SS_SUPPORTED_CIPHERS_SALT_LEN.keys())
+ + [
+ "aes-128-ccm",
+ "aes-192-ccm",
+ "aes-256-ccm",
+ "aes-128-gcm-siv",
+ "aes-256-gcm-siv",
+ "chacha20",
+ "chacha8-ietf-poly1305",
+ "xchacha8-ietf-poly1305",
+ "lea-128-gcm",
+ "lea-192-gcm",
+ "lea-256-gcm",
+ "rabbit128-poly1305",
+ "aegis-128l",
+ "aegis-256",
+ "aez-384",
+ "deoxys-ii-256-128",
+ "none",
+ ]
+)
+
+SSR_SUPPORTED_CIPHERS = COMMON_SS_SUPPORTED_CIPHERS + ["dummy", "none"]
+SSR_SUPPORTED_OBFS = [
+ "plain",
+ "http_simple",
+ "http_post",
+ "random_head",
+ "tls1.2_ticket_auth",
+ "tls1.2_ticket_fastauth",
+]
+SSR_SUPPORTED_PROTOCOL = [
+ "origin",
+ "auth_sha1_v4",
+ "auth_aes128_md5",
+ "auth_aes128_sha1",
+ "auth_chain_a",
+ "auth_chain_b",
+]
+
+CLASH_SS_PLUGINS = ("", "obfs", "v2ray-plugin")
+MIHOMO_SS_PLUGINS = CLASH_SS_PLUGINS + ("shadow-tls", "restls", "gost-plugin", "kcptun", "jls")
+
+
+def verify_ss_2022_password(cipher: str, password: str) -> bool:
+ password = utils.trim(password)
+ if not password:
+ return False
+
+ words = password.split(":")
+ if cipher == "2022-blake3-chacha20-poly1305" and len(words) > 1:
+ return False
+
+ key_len = MIHOMO_SS_SUPPORTED_CIPHERS_SALT_LEN.get(cipher)
+ if not key_len:
+ return False
+
+ for word in words:
+ if not word or not re.fullmatch(r"[A-Za-z0-9+/]+=*$", word) or len(word) % 4 != 0:
+ return False
+ try:
+ text = base64.b64decode(word, validate=True)
+ except Exception:
+ return False
+ if len(text) != key_len:
+ return False
+
+ return True
+
+
+class ShadowsocksVerifier(OutboundVerifier):
+ type_name = "ss"
+
+ def verify_fields(self, item: dict[str, object], ctx: VerifyContext) -> bool:
+ ciphers = MIHOMO_SS_SUPPORTED_CIPHERS if ctx.is_mihomo else COMMON_SS_SUPPORTED_CIPHERS
+ if item.get("cipher") not in ciphers:
+ return False
+
+ if item["cipher"] in MIHOMO_SS_SUPPORTED_CIPHERS_SALT_LEN:
+ if not verify_ss_2022_password(item["cipher"], str(item.get("password", ""))):
+ return False
+
+ plugin = item.get("plugin", "")
+ plugins = MIHOMO_SS_PLUGINS if ctx.is_mihomo else CLASH_SS_PLUGINS
+ if plugin not in plugins:
+ return False
+ if not plugin:
+ return True
+
+ plugin_opts = item.get("plugin-opts", {})
+ if plugin_opts in (None, ""):
+ plugin_opts = {}
+ if type(plugin_opts) != dict:
+ return False
+
+ mode = plugin_opts.get("mode", "")
+ if plugin == "jls":
+ return all(utils.trim(str(plugin_opts.get(k, ""))) for k in ("host", "username", "password"))
+ if plugin == "restls":
+ return all(utils.trim(str(plugin_opts.get(k, ""))) for k in ("host", "password", "version-hint"))
+ if plugin == "shadow-tls":
+ return bool(utils.trim(str(plugin_opts.get("host", ""))))
+ if plugin == "kcptun":
+ return True
+ if plugin == "obfs":
+ return mode in ("tls", "http")
+ if plugin in ("v2ray-plugin", "gost-plugin"):
+ return mode == "websocket"
+ return False
+
+
+class ShadowsocksRVerifier(OutboundVerifier):
+ type_name = "ssr"
+
+ def verify_fields(self, item: dict[str, object], ctx: VerifyContext) -> bool:
+ if item.get("cipher") not in SSR_SUPPORTED_CIPHERS:
+ return False
+ if item.get("obfs") not in SSR_SUPPORTED_OBFS:
+ return False
+ if item.get("protocol") not in SSR_SUPPORTED_PROTOCOL:
+ return False
+ return True
+
+ def duplicate_key(self, item: dict[str, object]) -> tuple[str, object]:
+ return (self.type_name, str(item.get("protocol-param", "")).lower())
diff --git a/subscribe/outbound/snell.py b/subscribe/outbound/snell.py
new file mode 100644
index 0000000000..f817911d2c
--- /dev/null
+++ b/subscribe/outbound/snell.py
@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+import utils
+from outbound.base import OutboundVerifier, VerifyContext
+from outbound.common import wrap
+
+SNELL_OBFS_MODES = ("", "http", "tls", "shadow-tls", "restls", "jls")
+
+
+class SnellVerifier(OutboundVerifier):
+ type_name = "snell"
+
+ def verify_fields(self, item: dict[str, object], ctx: VerifyContext) -> bool:
+ if "version" in item and not utils.is_number(item["version"]):
+ return False
+
+ version = int(item.get("version", 1))
+ if version < 1 or version > 5:
+ return False
+
+ # v1/v2 do not support UDP; v3/v4/v5 do
+ if version < 3:
+ item.pop("udp", None)
+
+ if "obfs-opts" not in item:
+ return True
+
+ obfs_opts = item.get("obfs-opts", {})
+ if not obfs_opts or type(obfs_opts) != dict:
+ return False
+ if "mode" not in obfs_opts:
+ return True
+
+ mode = wrap(obfs_opts.get("mode", ""))
+ return mode in SNELL_OBFS_MODES
+
+ def auth_field(self, item: dict[str, object]) -> str | None:
+ return "psk"
diff --git a/subscribe/outbound/ssh.py b/subscribe/outbound/ssh.py
new file mode 100644
index 0000000000..4da5fc539e
--- /dev/null
+++ b/subscribe/outbound/ssh.py
@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+from outbound.base import OutboundVerifier, VerifyContext
+from outbound.common import quote_numeric_fields, wrap
+
+
+class SshVerifier(OutboundVerifier):
+ type_name = "ssh"
+ mihomo_only = True
+
+ def verify_fields(self, item: dict[str, object], ctx: VerifyContext) -> bool:
+ if not wrap(item.get("username", "")):
+ return False
+ quote_numeric_fields(item, ("username",))
+ has_password = bool(wrap(item.get("password", "")))
+ has_key = bool(wrap(item.get("private-key", "")) or wrap(item.get("private_key", "")))
+ return has_password or has_key
+
+ def auth_field(self, item: dict[str, object]) -> str | None:
+ if wrap(item.get("password", "")):
+ return "password"
+ return None
+
+ def duplicate_key(self, item: dict[str, object]) -> tuple[str, object]:
+ return (
+ self.type_name,
+ item.get("username", ""),
+ item.get("password", ""),
+ item.get("private-key", "") or item.get("private_key", ""),
+ )
diff --git a/subscribe/outbound/sudoku.py b/subscribe/outbound/sudoku.py
new file mode 100644
index 0000000000..f689d64375
--- /dev/null
+++ b/subscribe/outbound/sudoku.py
@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+import utils
+from outbound.base import OutboundVerifier, VerifyContext
+from outbound.common import ensure_bool, quote_numeric_fields, wrap
+
+SUDOKU_AEAD = ("", "chacha20-poly1305", "aes-128-gcm", "none")
+SUDOKU_TABLES = ("", "prefer_ascii", "prefer_entropy", "up_ascii_down_entropy", "up_entropy_down_ascii")
+SUDOKU_MULTIPLEX = ("", "off", "auto", "on")
+HTTPMASK_MODES = ("", "legacy", "stream", "poll", "auto", "ws")
+
+
+class SudokuVerifier(OutboundVerifier):
+ type_name = "sudoku"
+ mihomo_only = True
+
+ def verify_fields(self, item: dict[str, object], ctx: VerifyContext) -> bool:
+ if not wrap(item.get("key", "")):
+ return False
+ quote_numeric_fields(item, ("key",))
+ if "aead-method" in item and wrap(item.get("aead-method", "")) not in SUDOKU_AEAD:
+ return False
+ if "table-type" in item and wrap(item.get("table-type", "")) not in SUDOKU_TABLES:
+ return False
+ if "multiplex" in item and wrap(item.get("multiplex", "")) not in SUDOKU_MULTIPLEX:
+ return False
+ for field in ("padding-min", "padding-max"):
+ if field in item:
+ if not utils.is_number(item[field]):
+ return False
+ value = int(item[field])
+ if value < 0 or value > 100:
+ return False
+ if "padding-min" in item and "padding-max" in item and int(item["padding-min"]) > int(item["padding-max"]):
+ return False
+ if not ensure_bool(item, "enable-pure-downlink"):
+ return False
+ if "httpmask" in item:
+ httpmask = item.get("httpmask")
+ if type(httpmask) != dict:
+ return False
+ if "mode" in httpmask and wrap(httpmask.get("mode", "")) not in HTTPMASK_MODES:
+ return False
+ return True
+
+ def auth_field(self, item: dict[str, object]) -> str | None:
+ return "key"
diff --git a/subscribe/outbound/tls.py b/subscribe/outbound/tls.py
new file mode 100644
index 0000000000..648722ccb8
--- /dev/null
+++ b/subscribe/outbound/tls.py
@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+import base64
+import re
+
+import utils
+from outbound.common import QuotedStr, wrap
+
+
+def verify_reality_public_key(public_key: str) -> bool:
+ # mihomo uses base64.RawURLEncoding and requires 32 bytes
+ public_key = utils.trim(public_key)
+ if not public_key or not re.fullmatch(r"[A-Za-z0-9_-]+", public_key):
+ return False
+
+ try:
+ decoded = base64.urlsafe_b64decode(public_key + "=" * (-len(public_key) % 4))
+ except Exception:
+ return False
+
+ if len(decoded) != 32:
+ return False
+
+ canonical = base64.urlsafe_b64encode(decoded).decode("utf-8").rstrip("=")
+ return canonical == public_key
+
+
+def verify_reality_opts(item: dict[str, object], required: bool = False) -> bool:
+ if "reality-opts" not in item:
+ return not required
+
+ reality_opts = item.get("reality-opts", {})
+ if not reality_opts or type(reality_opts) != dict:
+ return False
+ if "public-key" not in reality_opts or type(reality_opts["public-key"]) != str:
+ return False
+
+ content = utils.trim(reality_opts["public-key"])
+ if not verify_reality_public_key(content):
+ return False
+ reality_opts["public-key"] = content
+
+ if "short-id" not in reality_opts:
+ return True
+
+ short_id = reality_opts["short-id"]
+ if type(short_id) != str:
+ if utils.is_number(short_id):
+ short_id = str(short_id)
+ else:
+ return False
+
+ if short_id:
+ try:
+ sib = bytes.fromhex(short_id)
+ if len(sib) > 8:
+ return False
+ except ValueError:
+ return False
+
+ reality_opts["short-id"] = QuotedStr(short_id)
+ return True
+
+
+def verify_optional_str_dict(item: dict[str, object], field: str, required_keys: tuple[str, ...] = ()) -> bool:
+ if field not in item:
+ return True
+ opts = item.get(field)
+ if type(opts) != dict:
+ return False
+ for key in required_keys:
+ if key not in opts:
+ return False
+ if type(opts.get(key)) != str or not wrap(opts.get(key)):
+ return False
+ return True
diff --git a/subscribe/outbound/transport.py b/subscribe/outbound/transport.py
new file mode 100644
index 0000000000..8847b11cf6
--- /dev/null
+++ b/subscribe/outbound/transport.py
@@ -0,0 +1,301 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+import utils
+from outbound.common import wrap
+
+# mihomo ParseRange uses strconv.Atoi
+XHTTP_RANGE_MAX = 2**63 - 1
+XHTTP_RANGE_FIELDS = (
+ "sc-max-each-post-bytes",
+ "sc-min-posts-interval-ms",
+ "x-padding-bytes",
+ "uplink-chunk-size",
+ "session-length",
+)
+XHTTP_RANGE_POSITIVE_MAX = set(["sc-max-each-post-bytes", "sc-min-posts-interval-ms"])
+XHTTP_REUSE_RANGE_FIELDS = (
+ "max-concurrency",
+ "max-connections",
+ "c-max-reuse-times",
+ "h-max-request-times",
+ "h-max-reusable-secs",
+)
+XHTTP_MODES = ("auto", "stream-one", "stream-up", "packet-up")
+MKCP_HEADERS = ("", "none", "srtp", "utp", "wechat-video", "dtls", "wireguard")
+
+
+def parse_xhttp_range_bound(text: str) -> int | None:
+ text = utils.trim(text)
+ if not text:
+ return None
+
+ if text.isdigit() or (text.startswith("-") and text[1:].isdigit()):
+ try:
+ value = int(text)
+ except Exception:
+ return None
+ else:
+ try:
+ number = float(text)
+ except Exception:
+ return None
+ if number != number or number == float("inf") or number == float("-inf"):
+ return None
+ value = int(number)
+ if value != number:
+ return None
+
+ if abs(value) > XHTTP_RANGE_MAX:
+ return None
+ return value
+
+
+def normalize_xhttp_range(value: object, allow_negative: bool = False) -> int | None:
+ if isinstance(value, bool) or value is None:
+ return None
+ if isinstance(value, int):
+ if (value < 0 and not allow_negative) or abs(value) > XHTTP_RANGE_MAX:
+ return None
+ return str(value)
+ if isinstance(value, float):
+ if value != value or value == float("inf") or value == float("-inf"):
+ return None
+ if value < 0 and not allow_negative:
+ return None
+ number = int(value)
+ if number != value or abs(number) > XHTTP_RANGE_MAX:
+ return None
+ return str(number)
+
+ text = utils.trim(str(value))
+ if not text:
+ return ""
+
+ bound = parse_xhttp_range_bound(text)
+ if bound is not None:
+ if bound < 0 and not allow_negative:
+ return None
+ return str(bound)
+
+ if text.count("-") != 1:
+ return None
+ left, right = text.split("-", 1)
+ min_val, max_val = parse_xhttp_range_bound(left), parse_xhttp_range_bound(right)
+ if min_val is None or max_val is None or max_val < min_val:
+ return None
+ if min_val < 0 and not allow_negative:
+ return None
+ if min_val == max_val:
+ return str(min_val)
+ return f"{min_val}-{max_val}"
+
+
+def apply_xhttp_range_field(
+ container: dict[str, object], key: str, min_positive: bool = False, max_positive: bool = False
+) -> bool:
+ if key not in container:
+ return True
+
+ value = container[key]
+ if value is None or (isinstance(value, str) and not utils.trim(value)):
+ container.pop(key, None)
+ return True
+
+ normalized = normalize_xhttp_range(value)
+ if normalized is None:
+ return False
+ if not normalized:
+ container.pop(key, None)
+ return True
+
+ parts = normalized.split("-")
+ min_val, max_val = int(parts[0]), int(parts[-1])
+ if min_positive and min_val <= 0:
+ return False
+ if max_positive and max_val <= 0:
+ return False
+
+ container[key] = min_val if len(parts) == 1 else normalized
+ return True
+
+
+def verify_xhttp_reuse_settings(settings: dict[str, object]) -> bool:
+ if type(settings) != dict:
+ return False
+
+ for key in XHTTP_REUSE_RANGE_FIELDS:
+ if not apply_xhttp_range_field(settings, key):
+ return False
+
+ if "h-keep-alive-period" not in settings:
+ return True
+
+ value = settings["h-keep-alive-period"]
+ if value is None or (isinstance(value, str) and not utils.trim(value)):
+ settings.pop("h-keep-alive-period", None)
+ return True
+
+ # wiki allows negatives such as -1 to disable keepalive
+ normalized = normalize_xhttp_range(value, allow_negative=True)
+ if not normalized or "-" in normalized[1:]:
+ return False
+
+ settings["h-keep-alive-period"] = int(normalized)
+ return True
+
+
+def verify_ws_opts(item: dict[str, object], network: str) -> bool:
+ if "ws-opts" not in item:
+ return True
+ if network != "ws":
+ return False
+
+ ws_opts = item.get("ws-opts", {})
+ if not ws_opts or type(ws_opts) != dict:
+ return False
+ if "path" in ws_opts and type(ws_opts["path"]) != str:
+ return False
+ if "headers" in ws_opts and type(ws_opts["headers"]) != dict:
+ return False
+ return True
+
+
+def verify_grpc_opts(item: dict[str, object], network: str) -> bool:
+ if "grpc-opts" not in item:
+ return True
+ if network != "grpc":
+ return False
+
+ grpc_opts = item.get("grpc-opts", {})
+ if not grpc_opts or type(grpc_opts) != dict:
+ return False
+ if "grpc-service-name" in grpc_opts and type(grpc_opts["grpc-service-name"]) != str:
+ return False
+ return True
+
+
+def verify_http_opts(item: dict[str, object], network: str) -> bool:
+ if "http-opts" not in item:
+ return True
+ if network != "http":
+ return False
+
+ http_opts = item.get("http-opts", {})
+ if not http_opts or type(http_opts) != dict:
+ return False
+ if "path" in http_opts and type(http_opts["path"]) != list:
+ return False
+ if "headers" in http_opts:
+ headers = http_opts.get("headers", {})
+ if not isinstance(headers, dict):
+ return False
+ for key, value in headers.items():
+ if not isinstance(key, str):
+ return False
+ if key.lower() == "host" and not isinstance(value, list):
+ return False
+ return True
+
+
+def verify_h2_opts(item: dict[str, object], network: str) -> bool:
+ if "h2-opts" not in item:
+ return True
+ if network != "h2":
+ return False
+
+ h2_opts = item.get("h2-opts", {})
+ if not h2_opts or type(h2_opts) != dict:
+ return False
+ if "host" in h2_opts and type(h2_opts["host"]) != list:
+ return False
+ return True
+
+
+def verify_mkcp_opts(item: dict[str, object], network: str) -> bool:
+ if "mkcp-opts" not in item:
+ return True
+ if network not in ("mkcp", "kcp"):
+ return False
+
+ mkcp_opts = item.get("mkcp-opts", {})
+ if type(mkcp_opts) != dict:
+ return False
+ if "header" in mkcp_opts:
+ header = wrap(mkcp_opts.get("header", ""))
+ if header not in MKCP_HEADERS:
+ return False
+ return True
+
+
+def verify_mekya_opts(item: dict[str, object], network: str) -> bool:
+ if "mekya-opts" not in item:
+ return True
+ if network != "mekya":
+ return False
+
+ mekya_opts = item.get("mekya-opts", {})
+ if not mekya_opts or type(mekya_opts) != dict:
+ return False
+ if "url" in mekya_opts and type(mekya_opts["url"]) != str:
+ return False
+ if "kcp" in mekya_opts and type(mekya_opts["kcp"]) != dict:
+ return False
+ return True
+
+
+def verify_xhttp_opts(item: dict[str, object], network: str) -> bool:
+ if "xhttp-opts" not in item:
+ return True
+ if network != "xhttp":
+ return False
+
+ xhttp_opts = item.get("xhttp-opts", {})
+ if not xhttp_opts or type(xhttp_opts) != dict:
+ return False
+ if "path" in xhttp_opts and type(xhttp_opts["path"]) != str:
+ return False
+ if "host" in xhttp_opts and type(xhttp_opts["host"]) != str:
+ return False
+
+ if "mode" in xhttp_opts:
+ xhttp_mode = wrap(xhttp_opts.get("mode", ""))
+ if xhttp_mode and xhttp_mode not in XHTTP_MODES:
+ return False
+ if "headers" in xhttp_opts and type(xhttp_opts["headers"]) != dict:
+ return False
+
+ for key in XHTTP_RANGE_FIELDS:
+ min_positive = key == "session-length"
+ max_positive = key in XHTTP_RANGE_POSITIVE_MAX
+ if not apply_xhttp_range_field(xhttp_opts, key, min_positive=min_positive, max_positive=max_positive):
+ return False
+
+ if "reuse-settings" in xhttp_opts and not verify_xhttp_reuse_settings(xhttp_opts.get("reuse-settings")):
+ return False
+ if "download-settings" in xhttp_opts:
+ download_settings = xhttp_opts.get("download-settings")
+ if type(download_settings) != dict:
+ return False
+ if "reuse-settings" in download_settings and not verify_xhttp_reuse_settings(
+ download_settings.get("reuse-settings")
+ ):
+ return False
+ return True
+
+
+def normalize_httpupgrade(item: dict[str, object]) -> str:
+ network = wrap(item.get("network", ""))
+ if network != "httpupgrade":
+ return network
+
+ item["network"] = "ws"
+ ws_opts = item.get("ws-opts")
+ if ws_opts is None or ws_opts == "":
+ ws_opts = {}
+ item["ws-opts"] = ws_opts
+ if type(ws_opts) != dict:
+ return "ws"
+ ws_opts["v2ray-http-upgrade"] = True
+ return "ws"
diff --git a/subscribe/outbound/trojan.py b/subscribe/outbound/trojan.py
new file mode 100644
index 0000000000..b9ce0cef91
--- /dev/null
+++ b/subscribe/outbound/trojan.py
@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+from outbound.base import OutboundVerifier, VerifyContext
+from outbound.common import wrap
+from outbound.tls import verify_reality_opts
+from outbound.transport import verify_grpc_opts, verify_ws_opts
+
+
+class TrojanVerifier(OutboundVerifier):
+ type_name = "trojan"
+
+ def verify_fields(self, item: dict[str, object], ctx: VerifyContext) -> bool:
+ network = wrap(item.get("network", ""))
+ if network and network not in ("tcp", "ws", "grpc"):
+ return False
+
+ # Trojan outbound no longer has flow; drop leftover XTLS values
+ item.pop("flow", None)
+
+ if not verify_ws_opts(item, network):
+ return False
+ if not verify_grpc_opts(item, network):
+ return False
+ if ctx.is_mihomo and not verify_reality_opts(item):
+ return False
+
+ if "ss-opts" in item:
+ ss_opts = item.get("ss-opts")
+ if type(ss_opts) != dict:
+ return False
+ return True
diff --git a/subscribe/outbound/tunnel.py b/subscribe/outbound/tunnel.py
new file mode 100644
index 0000000000..93b5a3e2ba
--- /dev/null
+++ b/subscribe/outbound/tunnel.py
@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+from outbound.base import OutboundVerifier, VerifyContext
+from outbound.common import (
+ quote_numeric_fields,
+ verify_bbr_profile,
+ verify_congestion_controller,
+ wrap,
+)
+
+
+class TrustTunnelVerifier(OutboundVerifier):
+ type_name = "trusttunnel"
+ mihomo_only = True
+
+ def verify_fields(self, item: dict[str, object], ctx: VerifyContext) -> bool:
+ if not wrap(item.get("username", "")):
+ return False
+ quote_numeric_fields(item, ("username",))
+ if not verify_congestion_controller(item) or not verify_bbr_profile(item):
+ return False
+ return True
+
+ def duplicate_key(self, item: dict[str, object]) -> tuple[str, object]:
+ return (self.type_name, item.get("username", ""), item.get("password", ""))
+
+
+class GostRelayVerifier(OutboundVerifier):
+ type_name = "gost-relay"
+ mihomo_only = True
+
+ def verify_fields(self, item: dict[str, object], ctx: VerifyContext) -> bool:
+ quote_numeric_fields(item, ("username", "password"))
+ return True
+
+ def auth_field(self, item: dict[str, object]) -> str | None:
+ return None
+
+ def duplicate_key(self, item: dict[str, object]) -> tuple[str, object]:
+ return (self.type_name, item.get("username", ""), item.get("password", ""))
diff --git a/subscribe/outbound/vless.py b/subscribe/outbound/vless.py
new file mode 100644
index 0000000000..0244664d24
--- /dev/null
+++ b/subscribe/outbound/vless.py
@@ -0,0 +1,94 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+import base64
+import re
+
+from outbound.base import OutboundVerifier, VerifyContext
+from outbound.common import wrap
+from outbound.tls import verify_reality_opts
+from outbound.transport import (
+ verify_grpc_opts,
+ verify_h2_opts,
+ verify_http_opts,
+ verify_ws_opts,
+ verify_xhttp_opts,
+)
+
+VLESS_MLKEM_X25519_PLUS_PREFIX = "mlkem768x25519plus"
+VLESS_MLKEM_X25519_PLUS_MODES = ("native", "xorpub", "random")
+VLESS_MLKEM_X25519_PLUS_RTTS = ("1rtt", "0rtt")
+VLESS_MLKEM_X25519_PLUS_PADDING_LIMIT = 20
+VLESS_MLKEM_X25519_PLUS_KEY_SIZES = (32, 1184)
+VLESS_VISION_FLOW = "xtls-rprx-vision"
+
+
+def verify_vless_encryption(encryption: str) -> bool:
+ if not encryption or encryption == "none":
+ return True
+
+ parts = encryption.split(".")
+ if (
+ len(parts) < 4
+ or parts[0] != VLESS_MLKEM_X25519_PLUS_PREFIX
+ or parts[1] not in VLESS_MLKEM_X25519_PLUS_MODES
+ or parts[2] not in VLESS_MLKEM_X25519_PLUS_RTTS
+ ):
+ return False
+
+ for key in parts[3:]:
+ if len(key) < VLESS_MLKEM_X25519_PLUS_PADDING_LIMIT:
+ continue
+ if not re.fullmatch(r"[A-Za-z0-9_-]+", key):
+ return False
+
+ try:
+ content = key + "=" * (-len(key) % 4)
+ decoded = base64.urlsafe_b64decode(content)
+ except Exception:
+ return False
+
+ if len(decoded) not in VLESS_MLKEM_X25519_PLUS_KEY_SIZES:
+ return False
+
+ return True
+
+
+class VlessVerifier(OutboundVerifier):
+ type_name = "vless"
+ mihomo_only = True
+
+ def verify_fields(self, item: dict[str, object], ctx: VerifyContext) -> bool:
+ encryption = wrap(item.get("encryption", ""))
+ if not verify_vless_encryption(encryption):
+ return False
+
+ network = wrap(item.get("network", "tcp")) or "tcp"
+ if network not in ("ws", "tcp", "grpc", "http", "h2", "xhttp"):
+ return False
+
+ if "flow" in item:
+ flow = wrap(item.get("flow", ""))
+ if flow:
+ # mihomo truncates flow to 16 chars, so vision-udp443 is valid
+ if flow[:16] != VLESS_VISION_FLOW:
+ return False
+ item["flow"] = VLESS_VISION_FLOW
+
+ if not verify_ws_opts(item, network):
+ return False
+ if not verify_grpc_opts(item, network):
+ return False
+ if not verify_http_opts(item, network):
+ return False
+ if not verify_h2_opts(item, network):
+ return False
+ if not verify_reality_opts(item):
+ return False
+ if not verify_xhttp_opts(item, network):
+ return False
+ return True
+
+ def auth_field(self, item: dict[str, object]) -> str | None:
+ return "uuid"
diff --git a/subscribe/outbound/vmess.py b/subscribe/outbound/vmess.py
new file mode 100644
index 0000000000..b38b903012
--- /dev/null
+++ b/subscribe/outbound/vmess.py
@@ -0,0 +1,73 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+import utils
+from outbound.base import OutboundVerifier, VerifyContext
+from outbound.tls import verify_reality_opts
+from outbound.transport import (
+ normalize_httpupgrade,
+ verify_grpc_opts,
+ verify_h2_opts,
+ verify_http_opts,
+ verify_mekya_opts,
+ verify_mkcp_opts,
+ verify_ws_opts,
+)
+
+VMESS_CIPHERS = ["auto", "aes-128-gcm", "chacha20-poly1305", "none"]
+VMESS_NETWORKS = ("tcp", "ws", "h2", "http", "grpc", "mkcp", "kcp", "mekya")
+
+
+class VmessVerifier(OutboundVerifier):
+ type_name = "vmess"
+
+ def verify_fields(self, item: dict[str, object], ctx: VerifyContext) -> bool:
+ network = normalize_httpupgrade(item)
+ if not network:
+ for key, value in (
+ ("ws-opts", "ws"),
+ ("http-opts", "http"),
+ ("h2-opts", "h2"),
+ ("grpc-opts", "grpc"),
+ ("mkcp-opts", "mkcp"),
+ ("mekya-opts", "mekya"),
+ ):
+ if key in item:
+ network = value
+ break
+ else:
+ network = "tcp" if ctx.is_mihomo else "ws"
+
+ allowed = VMESS_NETWORKS if ctx.is_mihomo else ("ws", "h2", "http", "grpc")
+ if network not in allowed:
+ return False
+
+ ciphers = VMESS_CIPHERS + ["zero"] if ctx.is_mihomo else VMESS_CIPHERS
+ if item.get("cipher") not in ciphers:
+ return False
+ if "alterId" not in item or not utils.is_number(item["alterId"]):
+ return False
+
+ if not verify_h2_opts(item, network):
+ return False
+ if not verify_http_opts(item, network):
+ return False
+ if not verify_ws_opts(item, network):
+ return False
+ if "grpc-opts" in item:
+ if not ctx.is_mihomo:
+ return False
+ if not verify_grpc_opts(item, network):
+ return False
+ if ctx.is_mihomo:
+ if not verify_mkcp_opts(item, network):
+ return False
+ if not verify_mekya_opts(item, network):
+ return False
+ if not verify_reality_opts(item):
+ return False
+ return True
+
+ def auth_field(self, item: dict[str, object]) -> str | None:
+ return "uuid"
diff --git a/subscribe/outbound/vpn.py b/subscribe/outbound/vpn.py
new file mode 100644
index 0000000000..81d3a6645c
--- /dev/null
+++ b/subscribe/outbound/vpn.py
@@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+from outbound.base import OutboundVerifier, VerifyContext
+from outbound.common import wrap
+
+OPENVPN_PROTOS = ("", "udp", "tcp")
+OPENVPN_CIPHERS = (
+ "",
+ "AES-128-GCM",
+ "AES-192-GCM",
+ "AES-256-GCM",
+ "AES-128-CBC",
+ "AES-192-CBC",
+ "AES-256-CBC",
+ "AES-CBC",
+ "CHACHA20-POLY1305",
+)
+OPENVPN_AUTH = ("", "MD5", "SHA1", "SHA256", "SHA384", "SHA512")
+OPENVPN_COMP = ("", "yes", "no", "adaptive")
+OPENVPN_CIPHERS_UPPER = {c.upper() for c in OPENVPN_CIPHERS}
+MASQUE_NETWORKS = ("", "quic", "h2", "h3-l4proxy")
+
+
+class WireGuardVerifier(OutboundVerifier):
+ type_name = "wireguard"
+ mihomo_only = True
+
+ def needs_server(self, item: dict[str, object]) -> bool:
+ peers = item.get("peers")
+ return not (isinstance(peers, list) and peers)
+
+ def needs_port(self, item: dict[str, object]) -> bool:
+ return self.needs_server(item)
+
+ def verify_fields(self, item: dict[str, object], ctx: VerifyContext) -> bool:
+ if not wrap(item.get("private-key", "")):
+ return False
+ peers = item.get("peers")
+ if isinstance(peers, list) and peers:
+ for peer in peers:
+ if type(peer) != dict:
+ return False
+ if not wrap(peer.get("public-key", "")):
+ return False
+ if "allowed-ips" not in peer:
+ return False
+ return True
+ return bool(wrap(item.get("public-key", "")))
+
+ def auth_field(self, item: dict[str, object]) -> str | None:
+ return None
+
+ def duplicate_key(self, item: dict[str, object]) -> tuple[str, object]:
+ return (self.type_name, item.get("private-key", ""), item.get("public-key", ""))
+
+
+class OpenVPNVerifier(OutboundVerifier):
+ type_name = "openvpn"
+ mihomo_only = True
+
+ def verify_fields(self, item: dict[str, object], ctx: VerifyContext) -> bool:
+ if not wrap(item.get("ca", "")):
+ return False
+ if "proto" in item and wrap(item.get("proto", "")).lower() not in OPENVPN_PROTOS:
+ return False
+ if "cipher" in item and wrap(item.get("cipher", "")).upper() not in OPENVPN_CIPHERS_UPPER:
+ return False
+ if "auth" in item and wrap(item.get("auth", "")).upper() not in OPENVPN_AUTH:
+ return False
+ if "comp-lzo" in item and wrap(item.get("comp-lzo", "")) not in OPENVPN_COMP:
+ return False
+
+ has_user = bool(wrap(item.get("username", "")) and wrap(item.get("password", "")))
+ has_cert = bool(wrap(item.get("cert", "")) and wrap(item.get("key", "")))
+ return has_user or has_cert
+
+ def auth_field(self, item: dict[str, object]) -> str | None:
+ if wrap(item.get("password", "")):
+ return "password"
+ return None
+
+ def duplicate_key(self, item: dict[str, object]) -> tuple[str, object]:
+ return (
+ self.type_name,
+ item.get("username", ""),
+ item.get("password", ""),
+ item.get("cert", ""),
+ item.get("key", ""),
+ )
+
+
+class MasqueVerifier(OutboundVerifier):
+ type_name = "masque"
+ mihomo_only = True
+
+ def verify_fields(self, item: dict[str, object], ctx: VerifyContext) -> bool:
+ if not wrap(item.get("private-key", "")) or not wrap(item.get("public-key", "")):
+ return False
+ network = wrap(item.get("network", ""))
+ if network not in MASQUE_NETWORKS:
+ return False
+ return True
+
+ def auth_field(self, item: dict[str, object]) -> str | None:
+ return None
+
+ def duplicate_key(self, item: dict[str, object]) -> tuple[str, object]:
+ return (self.type_name, item.get("private-key", ""), item.get("public-key", ""))
diff --git a/subscribe/pipeline.py b/subscribe/pipeline.py
new file mode 100644
index 0000000000..7b6b1a0567
--- /dev/null
+++ b/subscribe/pipeline.py
@@ -0,0 +1,234 @@
+# -*- coding: utf-8 -*-
+
+import os
+import random
+import subprocess
+import time
+
+import utils
+import workflow
+import yaml
+from airport import AirPort
+from config.models import GroupConfig, NodeInput, SiteConfig
+from logger import logger
+from origin import Origin
+from workflow import TaskConfig, exists
+
+import clash
+import subconverter
+
+
+def assign_sites(
+ sites: list[SiteConfig],
+ groups: dict[str, GroupConfig],
+ retry: int,
+ bin_name: str,
+ allow_gmail_alias: bool = False,
+ special_protocols: bool | None = None,
+) -> tuple[list[TaskConfig], dict[str, list[int]]]:
+ tasks, grouped = [], {}
+ retry, globalid = max(1, retry), 0
+ if special_protocols is None:
+ special_protocols = AirPort.enable_special_protocols()
+
+ for site in sites or []:
+ if not isinstance(site, SiteConfig) or not site.enable:
+ continue
+ name = utils.trim(site.name).lower()
+ domain = utils.trim(site.domain).lower()
+ subscribe = site.nodes.subscribe_list()
+ if len(subscribe) >= 2:
+ subscribe = list(dict.fromkeys(subscribe))
+
+ count = min(max(1, int(site.count)), 10)
+ if subscribe:
+ count = len(subscribe)
+ if site.renew and site.renew.accounts:
+ count = len(site.renew.accounts)
+
+ source = site.origin
+ if not source:
+ source = Origin.TEMPORARY.name if not domain else Origin.OWNED.name
+ site.origin = source
+ if source != Origin.TEMPORARY.name:
+ site.errors = max(site.errors, 0) + 1
+ if name:
+ site.name = name.rsplit("-", maxsplit=1)[0]
+
+ if not name or (not domain and site.nodes.empty()) or count <= 0:
+ continue
+
+ if site.nodes.uris or site.nodes.proxies:
+ globalid += 1
+ task = TaskConfig(
+ name=name,
+ taskid=globalid,
+ domain=domain,
+ nodes=NodeInput(
+ subscribe=list(subscribe),
+ uris=list(site.nodes.uris),
+ proxies=list(site.nodes.proxies),
+ ),
+ index=-1,
+ retry=retry,
+ max_rate=site.max_rate,
+ bin_name=bin_name,
+ renew=(
+ site.renew.jobs(coupon=site.coupon, api_prefix=site.api_prefix)[0]
+ if site.renew and site.renew.accounts
+ else None
+ ),
+ rename=site.rename,
+ exclude=site.exclude,
+ include=site.include,
+ check_alive=site.check_alive,
+ coupon=site.coupon,
+ require_tls=site.require_tls,
+ ignore_default_exclude=site.ignore_default_exclude,
+ allow_gmail_alias=allow_gmail_alias,
+ skip_captcha=site.skip_captcha,
+ special_protocols=special_protocols,
+ invite_code=site.invite_code,
+ api_prefix=site.api_prefix,
+ )
+ if not exists(tasks=tasks, task=task):
+ tasks.append(task)
+ for push_name in site.push_to:
+ grouped.setdefault(push_name, []).append(globalid)
+ continue
+
+ for index in range(count):
+ globalid += 1
+ sub = subscribe[index] if index < len(subscribe) else ""
+ renew = None
+ if site.renew:
+ jobs = site.renew.jobs(coupon=site.coupon, api_prefix=site.api_prefix)
+ if index < len(jobs):
+ renew = jobs[index]
+ task = TaskConfig(
+ name=name,
+ taskid=globalid,
+ domain=domain,
+ nodes=NodeInput(subscribe=sub),
+ index=-1 if count == 1 else index + 1,
+ retry=retry,
+ max_rate=site.max_rate,
+ bin_name=bin_name,
+ renew=renew,
+ rename=site.rename,
+ exclude=site.exclude,
+ include=site.include,
+ check_alive=site.check_alive,
+ coupon=site.coupon,
+ require_tls=site.require_tls,
+ ignore_default_exclude=site.ignore_default_exclude,
+ allow_gmail_alias=allow_gmail_alias,
+ skip_captcha=site.skip_captcha,
+ special_protocols=special_protocols,
+ invite_code=site.invite_code,
+ api_prefix=site.api_prefix,
+ )
+ if exists(tasks=tasks, task=task):
+ continue
+ tasks.append(task)
+ for push_name in site.push_to:
+ if push_name not in groups:
+ logger.error(f"cannot found push config, name=[{push_name}]\tsite=[{name}]")
+ continue
+ grouped.setdefault(push_name, []).append(globalid)
+
+ return tasks, grouped
+
+
+def execute_tasks(tasks: list[TaskConfig]) -> list[tuple[int, list[dict[str, object]]]]:
+ return utils.multi_process_run(func=workflow.executewrapper, tasks=tasks)
+
+
+def check_alive_proxies(
+ proxies: list[dict[str, object]],
+ clash_bin: str,
+ workspace: str,
+ filename: str = "config.yaml",
+ timeout: int = 5000,
+ test_url: str = "https://www.google.com/generate_204",
+ delay: int = 5000,
+ num_threads: int = 64,
+ display: bool = True,
+ skip: bool = False,
+ group: str = "",
+) -> list[dict[str, object]]:
+ if not proxies:
+ return []
+
+ proxies = clash.generate_config(workspace, proxies, filename)
+ if skip:
+ return proxies
+
+ checks, nochecks = workflow.liveness_fillter(proxies=proxies)
+ if not checks:
+ return nochecks
+
+ binpath = os.path.join(workspace, clash_bin)
+ utils.chmod(binpath)
+ logger.info(f"startup clash now, workspace: {workspace}, config: {filename}")
+ process = subprocess.Popen([binpath, "-d", workspace, "-f", os.path.join(workspace, filename)])
+ logger.info(f"clash start success, begin check proxies, group: {group}\tcount: {len(checks)}")
+ time.sleep(random.randint(5, 8))
+ params = [
+ [item, clash.EXTERNAL_CONTROLLER, timeout, test_url, delay, False] for item in checks if isinstance(item, dict)
+ ]
+ masks = utils.multi_thread_run(func=clash.check, tasks=params, num_threads=num_threads, show_progress=display)
+ try:
+ process.terminate()
+ except Exception:
+ logger.error(f"terminate clash process error, group: {group}")
+
+ availables = [checks[i] for i in range(len(checks)) if masks[i]]
+ nochecks.extend(availables)
+ logger.info(
+ f"proxies check finished, total: {len(checks)}, alive: {len(availables)}, dead: {len(checks) - len(availables)}"
+ )
+ return nochecks
+
+
+def convert_proxies(
+ proxies: list[dict[str, object]],
+ subconverter_bin: str,
+ workspace: str,
+ source_file: str,
+ dest_file: str,
+ artifact: str,
+ target: str,
+ emoji: bool = True,
+ list_only: bool = True,
+ ignore_exclude: bool = False,
+) -> str:
+ filepath = os.path.join(workspace, source_file)
+ with open(filepath, "w+", encoding="utf8") as handle:
+ yaml.add_representer(clash.QuotedStr, clash.quoted_scalar)
+ yaml.dump({"proxies": proxies}, handle, allow_unicode=True)
+
+ generate_conf = os.path.join(workspace, "generate.ini")
+ if os.path.exists(generate_conf) and os.path.isfile(generate_conf):
+ os.remove(generate_conf)
+
+ success = subconverter.generate_conf(
+ filepath=generate_conf,
+ name=artifact,
+ source=source_file,
+ dest=dest_file,
+ target=target,
+ emoji=emoji,
+ list_only=list_only,
+ ignore_exclude=ignore_exclude,
+ )
+ if not success:
+ return ""
+ if not subconverter.convert(binname=subconverter_bin, artifact=artifact):
+ return ""
+
+ converted = os.path.join(workspace, dest_file)
+ if not os.path.exists(converted) or not os.path.isfile(converted):
+ return ""
+ with open(converted, "r", encoding="utf8") as handle:
+ return handle.read()
diff --git a/subscribe/process.py b/subscribe/process.py
index cc98014986..2df8b06377 100644
--- a/subscribe/process.py
+++ b/subscribe/process.py
@@ -5,29 +5,24 @@
import argparse
import base64
-import copy
import itertools
import json
import os
-import random
import re
-import subprocess
import sys
import time
import traceback
-from copy import deepcopy
-from dataclasses import dataclass, field
-import crawl
import executable
import location
+import pipeline
import push
import utils
import workflow
import yaml
from airport import AirPort
+from config.models import NodeInput, ProcessConfig, StorageItem
from logger import logger
-from origin import Origin
from workflow import TaskConfig
import clash
@@ -36,25 +31,9 @@
PATH = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
-@dataclass
-class ProcessConfig(object):
- # task list
- tasks: list[dict] = field(default_factory=list)
-
- # crawl config
- crawl: dict = field(default_factory=dict)
-
- # persist config
- storage: dict = field(default_factory=dict)
-
- # groups config
- groups: dict[dict] = field(default_factory=dict)
-
- # update config
- update: dict = field(default_factory=dict)
-
- # max acceptable delay
- delay: int = 5000
+def parse_workflow_mode(raw: str | None = None) -> int:
+ text = utils.trim(raw if raw is not None else os.environ.get("WORKFLOW_MODE", "0"))
+ return 0 if not text.isdigit() else min(max(int(text), 0), 2)
def load_configs(
@@ -62,256 +41,56 @@ def load_configs(
only_check: bool = False,
num_threads: int = 0,
display: bool = True,
+ retry: int = 3,
+ mode: int = 0,
) -> ProcessConfig:
- def parse_config(config: dict) -> None:
- tasks.extend(config.get("domains", []))
- groups.update(config.get("groups", {}))
- update_conf.update(config.get("update", {}))
- crawl_conf.update(config.get("crawl", {}))
- storage.update(config.get("storage", {}))
-
- push_conf = deepcopy(storage)
- push_conf.pop("items", None)
-
- nonlocal delay
- delay = min(delay, max(config.get("delay", sys.maxsize), 50))
-
- if only_check:
- return
-
- # global exclude
- params["exclude"] = crawl_conf.get("exclude", "")
-
- # persistence configuration
- persist = {k: storage.get("items", {}).get(v, {}) for k, v in crawl_conf.get("persist", {}).items()}
- params["storage"] = {"items": persist, **push_conf}
-
- params["config"] = crawl_conf.get("config", {})
- params["enable"] = crawl_conf.get("enable", True)
- params["singlelink"] = crawl_conf.get("singlelink", False)
-
- threshold = max(crawl_conf.get("threshold", 1), 1)
- params["threshold"] = threshold
- spiders = deepcopy(crawl_conf)
-
- # spider's config for telegram
- telegram_conf = spiders.get("telegram", {})
- users = telegram_conf.pop("users", {})
- telegram_conf["pages"] = max(telegram_conf.get("pages", 1), 1)
- if telegram_conf.pop("enable", True) and users:
- enabled_users, common_exclude = {}, telegram_conf.pop("exclude", "")
- for k, v in users.items():
- exclude = v.get("exclude", "").strip()
- v["exclude"] = f"{exclude}|{common_exclude}".removeprefix("|") if common_exclude else exclude
- v["push_to"] = list(set(v.get("push_to", [])))
-
- enabled_users[k] = v
- telegram_conf["users"] = enabled_users
- params["telegram"] = telegram_conf
-
- # spider's config for google
- google_conf = spiders.get("google", {})
- push_to = list(set(google_conf.get("push_to", [])))
- if google_conf.pop("enable", True) and push_to:
- google_conf["push_to"] = push_to
- params["google"] = google_conf
-
- # spider's config for yandex
- yandex_conf = spiders.get("yandex", {})
- push_to = list(set(yandex_conf.get("push_to", [])))
- if yandex_conf.pop("enable", True) and push_to:
- yandex_conf["push_to"] = push_to
- params["yandex"] = yandex_conf
-
- # spider's config for github
- github_conf = spiders.get("github", {})
- push_to = list(set(github_conf.get("push_to", [])))
- spams = list(set(github_conf.get("spams", [])))
- if github_conf.pop("enable", True) and push_to:
- github_conf["pages"] = max(github_conf.get("pages", 1), 1)
- github_conf["push_to"] = push_to
- github_conf["spams"] = spams
- params["github"] = github_conf
-
- # spider's config for twitter
- twitter_conf = spiders.get("twitter", {})
- users = twitter_conf.pop("users", {})
- if twitter_conf.pop("enable", True) and users:
- enabled_users = {}
- for k, v in users.items():
- if utils.isblank(k) or not v or type(v) != dict or not v.pop("enable", True):
- continue
-
- v["push_to"] = list(set(v.get("push_to", [])))
- enabled_users[k] = v
-
- params["twitter"] = enabled_users
-
- # spider's config for github's repositories
- repo_conf, repositories = spiders.get("repositories", []), {}
- for repo in repo_conf:
- enable = repo.pop("enable", True)
- username = repo.get("username", "").strip()
- repo_name = repo.get("repo_name", "").strip()
- if not enable or not username or not repo_name:
- continue
-
- key = "/".join([username, repo_name])
- push_to = list(set(repo.get("push_to", [])))
- repo["username"] = username
- repo["repo_name"] = repo_name
- repo["commits"] = max(repo.get("commits", 3), 1)
- repo["push_to"] = push_to
-
- repositories[key] = repo
- params["repositories"] = repositories
-
- # spider's config for specified page
- pages_conf, pages = spiders.get("pages", []), {}
- for page in pages_conf:
- enable = page.pop("enable", True)
- url = page.get("url", "")
- push_to = list(set(page.get("push_to", [])))
- if not enable or not url or not push_to:
- continue
-
- multiple = page.pop("multiple", False)
- if not multiple:
- page["push_to"] = push_to
- if isinstance(url, str):
- pages[url] = page
- elif isinstance(url, list):
- for u in url:
- u = utils.trim(u)
- if u:
- pages[u] = page
- else:
- placeholder = utils.trim(page.pop("placeholder", ""))
- if not placeholder or placeholder not in url or not isinstance(url, str):
- continue
-
- # page number range
- start, end = -1, -1
- try:
- start = int(page.pop("start", 1))
- end = int(page.pop("end", 1))
- except:
- pass
-
- if start < 0 or end < start:
- continue
-
- for i in range(start, end + 1):
- copypage = deepcopy(page)
- link = url.replace(placeholder, str(i))
- copypage["url"] = link
- copypage["push_to"] = push_to
- pages[link] = copypage
-
- params["pages"] = pages
-
- # spider's config for scripts
- scripts_conf, scripts = spiders.get("scripts", []), {}
-
- for script in scripts_conf:
- enable = script.pop("enable", True)
- path = script.pop("script", "").strip()
- if not enable or not path:
- continue
-
- task_conf = script.get("params", {})
- if not isinstance(task_conf, dict):
- task_conf = {}
-
- # record storge
- task_conf["storage"] = {"items": task_conf.pop("persist", {}), **push_conf}
-
- scripts[path] = task_conf
- params["scripts"] = scripts
-
- def verify(storage: dict, groups: dict) -> bool:
- if not isinstance(storage, dict) or not isinstance(groups, dict):
- return False
-
- pushtool = push.get_instance(config=push.PushConfig.from_dict(storage))
- if not isinstance(storage.get("items", {}), dict):
- logger.error(f"cannot found any valid storage config")
- return False
-
- items = pushtool.filter_push(config=storage.get("items", {}))
- for name, group in groups.items():
- name = utils.trim(name)
-
- if not name or not isinstance(group, dict):
- logger.error(f"invalid group config, name: {name}")
- return False
-
- targets = group.get("targets", {})
- if not targets or not isinstance(targets, dict):
- logger.error(f"group {name} should contain at least one type conversion")
- return False
-
- for category, storage_name in targets.items():
- category = utils.trim(category).lower()
- if category not in subconverter.CONVERT_TARGETS:
- logger.error(f"group {name} contains unsupported conversion type: {category}")
- return False
-
- storage_name = utils.trim(storage_name)
- if storage_name not in items:
- logger.error(f"missing storage configuration for group {name} to convert type to {category}")
- return False
-
- return True
-
- tasks, delay, storage, groups = [], sys.maxsize, {}, {}
- params, crawl_conf, update_conf = {}, {}, {}
+ from crawl.engine import run
+ raw = {}
try:
if re.match(
r"^(https?:\/\/(([a-zA-Z0-9]+-?)+[a-zA-Z0-9]+\.)+[a-zA-Z]+)(:\d+)?(\/.*)?(\?.*)?(#.*)?$",
url,
):
headers = {"User-Agent": utils.USER_AGENT, "Referer": url}
- content = utils.http_get(url=url, headers=headers)
+ content = utils.http_get(url=url, headers=headers, retry=max(retry, 1), timeout=120)
if not content:
logger.error(f"cannot fetch config from remote, url: {utils.hide(url=url)}")
else:
os.environ["SUBSCRIBE_CONF"] = url
- parse_config(json.loads(content))
+ raw = json.loads(content)
else:
localfile = os.path.abspath(url)
if os.path.exists(localfile) and os.path.isfile(localfile):
- config = json.loads(open(localfile, "r", encoding="utf8").read())
+ raw = json.loads(open(localfile, "r", encoding="utf8").read())
os.environ["SUBSCRIBE_CONF"] = localfile
- parse_config(config)
- # check configuration
- if not verify(storage=storage, groups=groups):
- raise ValueError(f"there are some errors in the configuration, please check and confirm")
-
- # execute crawl tasks
- if params:
- result = crawl.batch_crawl(conf=params, num_threads=num_threads, display=display)
- tasks.extend(result)
+ config = ProcessConfig.parse(raw or {})
+ pushtool = push.get_instance(config.storage)
+ config.verify(pushtool)
+ crawl_enabled = bool(config.crawl and config.crawl.enable)
+ if not only_check:
+ if mode == 1 and not crawl_enabled:
+ logger.warning("exit process because mode=1 and crawling task is disabled")
+ sys.exit(0)
+ if crawl_enabled:
+ config.sites.extend(
+ run(config.crawl, storage=config.storage, num_threads=num_threads, display=display, mode=mode)
+ )
+ if mode == 1:
+ sys.exit(0)
+ return config
except SystemExit as e:
if e.code != 0:
logger.error("parse configuration failed due to process abnormally exits")
-
sys.exit(e.code)
- except:
+ except ValueError as e:
+ logger.error(f"invalid configuration: {e}")
+ sys.exit(0)
+ except Exception:
logger.error(f"occur error when load task config:\n{traceback.format_exc()}")
sys.exit(0)
- return ProcessConfig(
- tasks=tasks,
- crawl=crawl_conf,
- storage=storage,
- groups=groups,
- update=update_conf,
- delay=delay,
- )
-
def assign(
pc: ProcessConfig,
@@ -319,188 +98,54 @@ def assign(
bin_name: str,
remain: bool,
pushtool: push.PushTo,
- only_check=False,
+ only_check: bool = False,
rigid: bool = True,
-) -> tuple[list[TaskConfig], dict, list]:
- if not isinstance(pc, ProcessConfig):
- return [], {}, []
-
- tasks, groups, arrays = [], {}, []
- retry, globalid = max(1, retry), 0
-
- # 是否允许特殊协议
- special_protocols = AirPort.enable_special_protocols()
-
- sites = [] if not isinstance(pc.tasks, list) else deepcopy(pc.tasks)
- for site in sites:
- if not site:
- continue
-
- name = site.get("name", "").strip().lower()
- domain = site.get("domain", "").strip().lower()
-
- # 订阅地址，支持单个或多个
- subscribe = site.get("sub", "")
- if isinstance(subscribe, str):
- subscribe = [subscribe.strip()]
- subscribe = [s for s in subscribe if s.strip() != ""]
- if len(subscribe) >= 2:
- subscribe = list(set(subscribe))
-
- # 节点倍率超过该值将会被丢弃
- rate = float(site.get("rate", 3.0))
-
- # 需要注册账号的个数
- num = min(max(1, int(site.get("count", 1))), 10)
-
- # 如果订阅链接不为空，num为订阅链接数
- num = len(subscribe) if subscribe else num
-
- # 组名列表
- push_names = site.get("push_to", [])
-
- # 失败次数，超过该值将不再尝试注册
- errors = max(site.get("errors", 0), 0) + 1
-
- # 来源类别
- source = site.get("origin", "")
-
- # 重命名规则，正常正则表达式
- rename = site.get("rename", "")
-
- # 排除匹配到的节点，支持正则表达式
- exclude = site.get("exclude", "").strip()
-
- # 仅保留匹配到的节点，支持正则表达式
- include = site.get("include", "").strip()
-
- # 是否检查 ChatGPT 的连通性
- chatgpt = site.get("chatgpt", {})
-
- # 是否对节点测活
- liveness = site.get("liveness", True)
-
- # 拒绝跳过证书验证
- disable_insecure = site.get("secure", False)
-
- # 优惠码
- coupon = utils.trim(site.get("coupon", ""))
+ special_protocols: bool | None = None,
+) -> tuple[list[TaskConfig], dict[str, list[int]]]:
+ if not isinstance(pc, ProcessConfig) or not isinstance(pushtool, push.PushTo):
+ return [], {}
- # 邀请码
- invite_code = utils.trim(site.get("invite_code", ""))
+ special_protocols = AirPort.enable_special_protocols(special_protocols)
- # 覆盖subconverter默认exclude规则
- ignoreder = site.get("ignorede", False)
-
- # 需要人机验证时是否直接放弃
- chuck = site.get("chuck", False)
-
- # 接口地址前缀
- api_prefix = site.get("api_prefix", "")
-
- if not source:
- source = Origin.TEMPORARY.name if not domain else Origin.OWNED.name
- site["origin"] = source
-
- if source != Origin.TEMPORARY.name:
- site["errors"] = errors
-
- site["name"] = name.rsplit(crawl.SEPARATOR, maxsplit=1)[0]
- arrays.append(site)
-
- renews = copy.deepcopy(site.get("renew", {}))
- accounts = renews.pop("account", [])
-
- # 如果renew不为空，num为配置的renew账号数
- num = len(accounts) if accounts else num
-
- if not site.get("enable", True) or "" == name or ("" == domain and not subscribe) or num <= 0:
- continue
-
- for i in range(num):
- index = -1 if num == 1 else i + 1
- sub = subscribe[i] if subscribe else ""
- renew = {"coupon_code": coupon} if coupon else {}
-
- globalid += 1
- if accounts:
- renew.update(accounts[i])
- renew.update(renews)
-
- if renew and api_prefix:
- renew["api_prefix"] = api_prefix
-
- task = TaskConfig(
- name=name,
- taskid=globalid,
- domain=domain,
- sub=sub,
- index=index,
- retry=retry,
- rate=rate,
- bin_name=bin_name,
- renew=renew,
- rename=rename,
- exclude=exclude,
- include=include,
- chatgpt=chatgpt,
- liveness=liveness,
- coupon=coupon,
- disable_insecure=disable_insecure,
- ignorede=ignoreder,
- rigid=rigid,
- chuck=chuck,
- special_protocols=special_protocols,
- invite_code=invite_code,
- api_prefix=api_prefix,
- )
- found = workflow.exists(tasks=tasks, task=task)
- if found:
- continue
-
- tasks.append(task)
- for push_name in push_names:
- if push_name not in pc.groups:
- logger.error(f"cannot found push config, name=[{push_name}]\tsite=[{name}]")
- continue
-
- taskids = groups.get(push_name, [])
- taskids.append(globalid)
- groups[push_name] = taskids
+ tasks, groups = pipeline.assign_sites(
+ sites=pc.sites,
+ groups=pc.groups,
+ retry=retry,
+ bin_name=bin_name,
+ allow_gmail_alias=not rigid,
+ special_protocols=special_protocols,
+ )
if (remain or only_check) and pc.groups:
if only_check:
- # clean all extra tasks
- tasks, groups, globalid = [], {k: [] for k in groups.keys()}, 0
+ tasks, groups, globalid = [], {k: [] for k in pc.groups.keys()}, 0
+ else:
+ globalid = max([task.taskid for task in tasks], default=0)
- for k, v in pc.groups.items():
- taskids = groups.get(k, [])
- targets = v.get("targets", {})
- if not targets:
+ for name, group in pc.groups.items():
+ taskids = groups.get(name, [])
+ if not group.targets:
continue
-
- # get the first conversion configuration for each group
- values = list(targets.values())
- config = pc.storage.get("items", {}).get(values[0], {})
- subscribe = pushtool.raw_url(config=config)
- if k not in groups or not subscribe:
+ values = list(group.targets.values())
+ item = pc.storage.items.get(values[0])
+ subscribe = pushtool.raw_url(item=item) if item else ""
+ if name not in groups or not subscribe:
continue
-
globalid += 1
tasks.append(
TaskConfig(
- name=f"remains-{k}",
+ name=f"remains-{name}",
taskid=globalid,
- sub=subscribe,
+ nodes=NodeInput(subscribe=subscribe),
index=-1,
- retry=retry,
+ retry=max(1, retry),
bin_name=bin_name,
special_protocols=special_protocols,
)
)
taskids.append(globalid)
- groups[k] = taskids
- return tasks, groups, arrays
+ groups[name] = taskids
+ return tasks, groups
def aggregate(args: argparse.Namespace) -> None:
@@ -509,6 +154,7 @@ def aggregate(args: argparse.Namespace) -> None:
clash_bin, subconverter_bin = executable.which_bin()
display = not args.invisible
+ retry = min(max(1, args.retry), 10)
# parse config
server = utils.trim(args.server) or os.environ.get("SUBSCRIBE_CONF", "").strip()
@@ -517,14 +163,14 @@ def aggregate(args: argparse.Namespace) -> None:
only_check=args.check,
num_threads=args.num,
display=display,
+ retry=retry,
+ mode=args.mode,
)
- storages = process_config.storage or {}
- pushtool = push.get_instance(config=push.PushConfig.from_dict(storages))
- retry = min(max(1, args.retry), 10)
+ pushtool = push.get_instance(process_config.storage)
# generate tasks
- tasks, groups, sites = assign(
+ tasks, groups = assign(
pc=process_config,
retry=retry,
bin_name=subconverter_bin,
@@ -532,6 +178,7 @@ def aggregate(args: argparse.Namespace) -> None:
pushtool=pushtool,
only_check=args.check,
rigid=not args.flexible,
+ special_protocols=args.special_protocols,
)
if not tasks:
logger.error("cannot found any valid config, exit")
@@ -550,8 +197,9 @@ def aggregate(args: argparse.Namespace) -> None:
data = results[i]
if not data or data[0] < 0 or not data[1]:
# not contain any proxy
- if tasks[i] and tasks[i].sub:
- subscribes[tasks[i].sub] = False
+ if tasks[i]:
+ for url in tasks[i].nodes.subscribe_list():
+ subscribes[url] = False
continue
datasets[data[0]] = data[1]
@@ -568,59 +216,21 @@ def aggregate(args: argparse.Namespace) -> None:
continue
workspace = os.path.join(PATH, "clash")
- binpath = os.path.join(workspace, clash_bin)
filename = "config.yaml"
- proxies = clash.generate_config(workspace, proxies, filename)
-
- # filer
- skip = utils.trim(os.environ.get("SKIP_ALIVE_CHECK", "false")).lower() in ["true", "1"]
- nochecks, starttime = proxies, time.time()
-
- if not skip:
- checks, nochecks = workflow.liveness_fillter(proxies=proxies)
- if checks:
- # executable
- utils.chmod(binpath)
-
- logger.info(f"startup clash now, workspace: {workspace}, config: {filename}")
- process = subprocess.Popen(
- [
- binpath,
- "-d",
- workspace,
- "-f",
- os.path.join(workspace, filename),
- ]
- )
-
- logger.info(f"clash start success, begin check proxies, group: {k}\tcount: {len(checks)}")
- time.sleep(random.randint(5, 8))
-
- params = [
- [p, clash.EXTERNAL_CONTROLLER, args.timeout, args.url, process_config.delay, False]
- for p in checks
- if isinstance(p, dict)
- ]
-
- # check
- masks = utils.multi_thread_run(
- func=clash.check,
- tasks=params,
- num_threads=args.num,
- show_progress=display,
- )
-
- # close clash client
- try:
- process.terminate()
- except:
- logger.error(f"terminate clash process error, group: {k}")
-
- availables = [checks[i] for i in range(len(checks)) if masks[i]]
- nochecks.extend(availables)
-
- dead = len(checks) - len(availables)
- logger.info(f"proxies check finished, total: {len(checks)}, alive: {len(availables)}, dead: {dead}")
+ starttime = time.time()
+ nochecks = pipeline.check_alive_proxies(
+ proxies=proxies,
+ clash_bin=clash_bin,
+ workspace=workspace,
+ filename=filename,
+ timeout=args.timeout,
+ test_url=args.url,
+ delay=process_config.delay,
+ num_threads=args.num,
+ display=display,
+ skip=args.skip_alive_check,
+ group=k,
+ )
for item in nochecks:
item.pop("sub", "")
@@ -629,26 +239,22 @@ def aggregate(args: argparse.Namespace) -> None:
logger.error(f"cannot fetch any proxy, group=[{k}], cost: {time.time()-starttime:.2f}s")
continue
- group_conf = process_config.groups.get(k, {})
- emoji = group_conf.get("emoji", True)
- list_only = group_conf.get("list", True)
-
- regularize = group_conf.get("regularize", {})
- if regularize and isinstance(regularize, dict) and regularize.get("enable", False):
- locate = regularize.get("locate", False)
- residential = regularize.get("residential", False)
- try:
- bits = max(1, int(regularize.get("bits", 2)))
- except:
- bits = 2
+ group = process_config.groups.get(k)
+ if not group:
+ continue
+ emoji = group.emoji
+ list_only = group.list_only
+ if group.regularize and group.regularize.enable:
nochecks = location.regularize(
proxies=nochecks,
num_threads=args.num,
show_progress=display,
- locate=locate,
- residential=residential,
- digits=bits,
+ locate=group.regularize.locate and not args.skip_alive_check,
+ residential=group.regularize.residential and not args.skip_alive_check,
+ ip_library=group.regularize.library,
+ digits=max(1, group.regularize.digits),
+ score=group.regularize.score,
)
source_file, data = "config.yaml", {"proxies": nochecks}
@@ -657,7 +263,7 @@ def aggregate(args: argparse.Namespace) -> None:
yaml.add_representer(clash.QuotedStr, clash.quoted_scalar)
yaml.dump(data, f, allow_unicode=True)
- targets = group_conf.get("targets", {})
+ targets = group.targets
for target, storage_name in targets.items():
persisted, content = False, " "
@@ -703,7 +309,7 @@ def aggregate(args: argparse.Namespace) -> None:
# save to remote server
persisted = pushtool.push_to(
content=content,
- config=process_config.storage.get("items", {}).get(storage_name, {}),
+ item=process_config.storage.items.get(storage_name, StorageItem()),
group=f"{k}::{target}",
)
@@ -720,21 +326,27 @@ def aggregate(args: argparse.Namespace) -> None:
cost = "{:.2f}s".format(time.time() - starttime)
logger.info(f"group [{k}] process finished, count: {len(nochecks)}, cost: {cost}")
- config = {
- "domains": sites,
- "crawl": process_config.crawl,
- "groups": process_config.groups,
- "storage": process_config.storage,
- "update": process_config.update,
- }
- skip_remark = utils.trim(os.environ.get("SKIP_REMARK", "false")).lower() in ["true", "1"]
-
- workflow.refresh(config=config, push=pushtool, alives=dict(subscribes), skip_remark=skip_remark)
+ workflow.refresh(
+ config=process_config,
+ push=pushtool,
+ alives=dict(subscribes),
+ skip_remark=args.skip_remark,
+ )
if __name__ == "__main__":
- parser = argparse.ArgumentParser()
+ env_parser = argparse.ArgumentParser(add_help=False)
+ env_parser.add_argument(
+ "-e",
+ "--environment",
+ type=str,
+ default=".env",
+ help="environment file name",
+ )
+ env_args, _ = env_parser.parse_known_args()
+ utils.load_dotenv(env_args.environment)
+ parser = argparse.ArgumentParser(parents=[env_parser])
parser.add_argument(
"-c",
"--check",
@@ -744,15 +356,6 @@ def aggregate(args: argparse.Namespace) -> None:
help="only check proxies are alive",
)
- parser.add_argument(
- "-e",
- "--environment",
- type=str,
- required=False,
- default=".env",
- help="environment file name",
- )
-
parser.add_argument(
"-f",
"--flexible",
@@ -771,6 +374,15 @@ def aggregate(args: argparse.Namespace) -> None:
help="don't show check progress bar",
)
+ parser.add_argument(
+ "-m",
+ "--mode",
+ type=int,
+ choices=[0, 1, 2],
+ default=parse_workflow_mode(),
+ help="workflow mode: 0=crawl+aggregate, 1=crawl only, 2=aggregate only",
+ )
+
parser.add_argument(
"-n",
"--num",
@@ -803,10 +415,31 @@ def aggregate(args: argparse.Namespace) -> None:
"--server",
type=str,
required=False,
- default="",
+ default=utils.trim(os.environ.get("SUBSCRIBE_CONF", "")),
help="remote config file",
)
+ parser.add_argument(
+ "--skip-alive-check",
+ action=argparse.BooleanOptionalAction,
+ default=utils.env_bool("SKIP_ALIVE_CHECK", False),
+ help="skip proxy liveness check",
+ )
+
+ parser.add_argument(
+ "--skip-remark",
+ action=argparse.BooleanOptionalAction,
+ default=utils.env_bool("SKIP_REMARK", False),
+ help="skip remark update for crawled subscriptions",
+ )
+
+ parser.add_argument(
+ "--special-protocols",
+ action=argparse.BooleanOptionalAction,
+ default=utils.env_bool("ENABLE_SPECIAL_PROTOCOLS", True),
+ help="include special protocols such as vless and hysteria",
+ )
+
parser.add_argument(
"-t",
"--timeout",
@@ -826,6 +459,4 @@ def aggregate(args: argparse.Namespace) -> None:
)
args = parser.parse_args()
- utils.load_dotenv(args.environment)
-
aggregate(args=args)
diff --git a/subscribe/push.py b/subscribe/push.py
index 89b5558663..6a42e4e776 100644
--- a/subscribe/push.py
+++ b/subscribe/push.py
@@ -8,10 +8,10 @@
import traceback
import urllib
import urllib.request
-from dataclasses import dataclass
from http.client import HTTPResponse
import utils
+from config.models import StorageConfig, StorageItem
from logger import logger
from urlvalidator import isurl
@@ -48,7 +48,7 @@ def _storage(self, content: str, filename: str, folder: str = "") -> bool:
except:
return False
- def push_file(self, filepath: str, config: dict, group: str = "", retry: int = 5) -> bool:
+ def push_file(self, filepath: str, item: StorageItem, group: str = "", retry: int = 5) -> bool:
if not os.path.exists(filepath) or not os.path.isfile(filepath):
logger.error(f"[PushFileError] file {filepath} not found")
return False
@@ -57,17 +57,17 @@ def push_file(self, filepath: str, config: dict, group: str = "", retry: int = 5
with open(filepath, "r", encoding="utf8") as f:
content = f.read()
- return self.push_to(content=content, config=config, group=group, retry=retry)
+ return self.push_to(content=content, item=item, group=group, retry=retry)
- def push_to(self, content: str, config: dict, group: str = "", retry: int = 5, **kwargs) -> bool:
- if not self.validate(config=config):
+ def push_to(self, content: str, item: StorageItem, group: str = "", retry: int = 5, **kwargs: object) -> bool:
+ if not self.validate(item=item):
logger.error(f"[PushError] push config is invalidate, domain: {self.name}")
return False
- if config.get("local", ""):
- self._storage(content=content, filename=config.get("local"))
+ if item.local:
+ self._storage(content=content, filename=item.local)
- url, data, headers = self._generate_payload(content=content, config=config)
+ url, data, headers = self._generate_payload(content=content, item=item)
payload = kwargs.get("payload", None)
if payload and isinstance(payload, dict):
try:
@@ -108,26 +108,26 @@ def push_to(self, content: str, config: dict, group: str = "", retry: int = 5, *
retry -= 1
if retry > 0:
- return self.push_to(content, config, group, retry)
+ return self.push_to(content, item, group, retry)
return False
def _is_success(self, response: HTTPResponse) -> bool:
return response and response.getcode() == 200
- def _generate_payload(self, content: str, config: dict) -> tuple[str, str, dict]:
+ def _generate_payload(self, content: str, item: StorageItem) -> tuple[str, str, dict]:
raise NotImplementedError
def _error_handler(self, group: str = "") -> None:
logger.error(f"[PushError]: group=[{group}], name: {self.name}, error message: \n{traceback.format_exc()}")
- def validate(self, config: dict) -> bool:
+ def validate(self, item: StorageItem) -> bool:
raise NotImplementedError
- def filter_push(self, config: dict) -> dict:
+ def filter_push(self, items: dict[str, StorageItem]) -> dict[str, StorageItem]:
raise NotImplementedError
- def raw_url(self, config: dict) -> str:
+ def raw_url(self, item: StorageItem) -> str:
raise NotImplementedError
@@ -150,18 +150,18 @@ def __init__(self, token: str, base: str = "", domain: str = "") -> None:
self.domain = domain
self.api_address = f"{base}/v1/pastes"
- def validate(self, config: dict) -> bool:
- if not config or type(config) != dict:
+ def validate(self, item: StorageItem) -> bool:
+ if not isinstance(item, StorageItem):
return False
- folderid = config.get("folderid", "")
- fileid = config.get("fileid", "")
+ folder_id = item.folder_id
+ file_id = item.file_id
- return "" != self.token.strip() and "" != folderid.strip() and "" != fileid.strip()
+ return "" != self.token.strip() and "" != folder_id.strip() and "" != file_id.strip()
- def _generate_payload(self, content: str, config: dict) -> tuple[str, str, dict]:
- folderid = config.get("folderid", "")
- fileid = config.get("fileid", "")
+ def _generate_payload(self, content: str, item: StorageItem) -> tuple[str, str, dict]:
+ folder_id = item.folder_id
+ file_id = item.file_id
headers = {
"Authorization": f"Key {self.token}",
@@ -169,7 +169,7 @@ def _generate_payload(self, content: str, config: dict) -> tuple[str, str, dict]
"User-Agent": utils.USER_AGENT,
}
data = json.dumps({"content": {"format": "text", "value": content}}).encode("UTF8")
- url = f"{self.api_address}/{folderid}/files/{fileid}"
+ url = f"{self.api_address}/{folder_id}/files/{file_id}"
return url, data, headers
@@ -179,26 +179,26 @@ def _is_success(self, response: HTTPResponse) -> bool:
def _error_handler(self, group: str = "") -> None:
logger.error(f"[PushError]: group=[{group}], name: {self.name}, error message: \n{traceback.format_exc()}")
- def filter_push(self, config: dict) -> dict:
+ def filter_push(self, items: dict[str, StorageItem]) -> dict[str, StorageItem]:
records = {}
- for k, v in config.items():
- if self.token and v.get("folderid", "") and v.get("fileid", "") and v.get("username", ""):
+ for k, v in items.items():
+ if self.token and v.folder_id and v.file_id and v.username:
records[k] = v
return records
- def raw_url(self, config: dict) -> str:
- if not config or type(config) != dict:
+ def raw_url(self, item: StorageItem) -> str:
+ if not isinstance(item, StorageItem):
return ""
- fileid = config.get("fileid", "")
- folderid = config.get("folderid", "")
- username = config.get("username", "")
+ file_id = item.file_id
+ folder_id = item.folder_id
+ username = item.username
- if not fileid or not folderid or not username:
+ if not file_id or not folder_id or not username:
return ""
- return f"{self.domain}/p/{username}/{folderid}/files/{fileid}/raw"
+ return f"{self.domain}/p/{username}/{folder_id}/files/{file_id}/raw"
class PushToDevbin(PushToPasteGG):
@@ -215,23 +215,23 @@ def __init__(self, token: str, base: str = "") -> None:
self.domain = base
self.api_address = f"{base}/api/v3/paste"
- def validate(self, config: dict) -> bool:
- if not config or type(config) != dict:
+ def validate(self, item: StorageItem) -> bool:
+ if not isinstance(item, StorageItem):
return False
- fileid = config.get("fileid", "")
- return "" != self.token.strip() and "" != fileid.strip()
+ file_id = item.file_id
+ return "" != self.token.strip() and "" != file_id.strip()
- def filter_push(self, config: dict) -> dict:
+ def filter_push(self, items: dict[str, StorageItem]) -> dict[str, StorageItem]:
records = {}
- for k, v in config.items():
- if v.get("fileid", "") and self.token:
+ for k, v in items.items():
+ if v.file_id and self.token:
records[k] = v
return records
- def _generate_payload(self, content: str, config: dict) -> tuple[str, str, dict]:
- fileid = config.get("fileid", "")
+ def _generate_payload(self, content: str, item: StorageItem) -> tuple[str, str, dict]:
+ file_id = item.file_id
headers = {
"Authorization": self.token,
@@ -239,19 +239,19 @@ def _generate_payload(self, content: str, config: dict) -> tuple[str, str, dict]
"Accept": "*/*",
}
data = json.dumps({"content": content, "syntaxName": "auto"}).encode("UTF8")
- url = f"{self.api_address}/{fileid}"
+ url = f"{self.api_address}/{file_id}"
return url, data, headers
def _is_success(self, response: HTTPResponse) -> bool:
return response and response.getcode() == 201
- def raw_url(self, config: dict) -> str:
- if not config or type(config) != dict or not config.get("fileid", ""):
+ def raw_url(self, item: StorageItem) -> str:
+ if not isinstance(item, StorageItem) or not item.file_id:
return ""
- fileid = config.get("fileid", "")
- return f"{self.domain}/Raw/{fileid}"
+ file_id = item.file_id
+ return f"{self.domain}/Raw/{file_id}"
class PushToPastefy(PushToDevbin):
@@ -269,8 +269,8 @@ def __init__(self, token: str, base: str = "") -> None:
self.domain = base
self.api_address = f"{base}/api/v2/paste"
- def _generate_payload(self, content: str, config: dict) -> tuple[str, str, dict]:
- fileid = config.get("fileid", "")
+ def _generate_payload(self, content: str, item: StorageItem) -> tuple[str, str, dict]:
+ file_id = item.file_id
headers = {
"Authorization": f"Bearer {self.token}",
@@ -279,7 +279,7 @@ def _generate_payload(self, content: str, config: dict) -> tuple[str, str, dict]
"User-Agent": utils.USER_AGENT,
}
data = json.dumps({"content": content}).encode("UTF8")
- url = f"{self.api_address}/{fileid}"
+ url = f"{self.api_address}/{file_id}"
return url, data, headers
@@ -295,15 +295,15 @@ def _is_success(self, response: HTTPResponse) -> bool:
def _error_handler(self, group: str = "") -> None:
logger.error(f"[PushError]: group=[{group}], name: {self.name}, error message: \n{traceback.format_exc()}")
- def raw_url(self, config: dict) -> str:
- if not config or type(config) != dict:
+ def raw_url(self, item: StorageItem) -> str:
+ if not isinstance(item, StorageItem):
return ""
- fileid = utils.trim(config.get("fileid", ""))
- if not fileid:
+ file_id = utils.trim(item.file_id)
+ if not file_id:
return ""
- return f"{self.domain}/{fileid}/raw"
+ return f"{self.domain}/{file_id}/raw"
class PushToImperial(PushToPasteGG):
@@ -325,30 +325,30 @@ def __init__(self, token: str, base: str = "", domain: str = "") -> None:
self.domain = domain
self.api_address = f"{base}/v1/document"
- def raw_url(self, config: dict) -> str:
- if not self.validate(config):
+ def raw_url(self, item: StorageItem) -> str:
+ if not self.validate(item):
return ""
- fileid = config.get("fileid", "")
- return f"{self.domain}/r/{fileid}"
+ file_id = item.file_id
+ return f"{self.domain}/r/{file_id}"
- def validate(self, config: dict) -> bool:
- if not config or type(config) != dict:
+ def validate(self, item: StorageItem) -> bool:
+ if not isinstance(item, StorageItem):
return False
- fileid = config.get("fileid", "")
- return "" != self.token.strip() and "" != fileid.strip()
+ file_id = item.file_id
+ return "" != self.token.strip() and "" != file_id.strip()
- def filter_push(self, config: dict) -> dict:
+ def filter_push(self, items: dict[str, StorageItem]) -> dict[str, StorageItem]:
records = {}
- for k, v in config.items():
- if v.get("fileid", "") and self.token:
+ for k, v in items.items():
+ if v.file_id and self.token:
records[k] = v
return records
- def _generate_payload(self, content: str, config: dict) -> tuple[str, str, dict]:
- fileid = config.get("fileid", "")
+ def _generate_payload(self, content: str, item: StorageItem) -> tuple[str, str, dict]:
+ file_id = item.file_id
headers = {
"Authorization": self.token,
@@ -357,7 +357,7 @@ def _generate_payload(self, content: str, config: dict) -> tuple[str, str, dict]
"User-Agent": utils.USER_AGENT,
}
- data = json.dumps({"id": fileid, "content": content}).encode("UTF8")
+ data = json.dumps({"id": file_id, "content": content}).encode("UTF8")
return self.api_address, data, headers
def _is_success(self, response: HTTPResponse) -> bool:
@@ -375,27 +375,27 @@ def __init__(self) -> None:
super().__init__(token="")
self.name = "local"
- def validate(self, config: dict) -> bool:
- return config is not None and config.get("fileid", "")
+ def validate(self, item: StorageItem) -> bool:
+ return isinstance(item, StorageItem) and bool(item.file_id)
- def push_to(self, content: str, config: dict, group: str = "", retry: int = 5) -> bool:
- folder = config.get("folderid", "")
- filename = config.get("fileid", "")
+ def push_to(self, content: str, item: StorageItem, group: str = "", retry: int = 5) -> bool:
+ folder = item.folder_id
+ filename = item.file_id
success = self._storage(content=content, filename=filename, folder=folder)
message = "successed" if success else "failed"
logger.info(f"[PushInfo] push subscribes information to {self.name} {message}, group=[{group}]")
return success
- def filter_push(self, config: dict) -> dict:
- return {k: v for k, v in config.items() if v.get("fileid", "")}
+ def filter_push(self, items: dict[str, StorageItem]) -> dict[str, StorageItem]:
+ return {k: v for k, v in items.items() if isinstance(v, StorageItem) and v.file_id}
- def raw_url(self, config: dict) -> str:
- if not config or type(config) != dict:
+ def raw_url(self, item: StorageItem) -> str:
+ if not isinstance(item, StorageItem):
return ""
- fileid = config.get("fileid", "")
- folderid = config.get("folderid", "")
- filepath = os.path.abspath(os.path.join(folderid, fileid))
+ file_id = item.file_id
+ folder_id = item.folder_id
+ filepath = os.path.abspath(os.path.join(folder_id, file_id))
return f"{utils.FILEPATH_PROTOCAL}{filepath}"
@@ -408,20 +408,20 @@ def __init__(self, token: str) -> None:
self.domain = "https://gist.githubusercontent.com"
self.method = "PATCH"
- def validate(self, config: dict) -> bool:
- if not isinstance(config, dict):
+ def validate(self, item: StorageItem) -> bool:
+ if not isinstance(item, StorageItem):
return False
- gistid = config.get("gistid", "")
- filename = config.get("filename", "")
+ gist_id = item.gist_id
+ filename = item.filename
- return "" != self.token.strip() and "" != gistid.strip() and "" != filename.strip()
+ return "" != self.token.strip() and "" != gist_id.strip() and "" != filename.strip()
- def _generate_payload(self, content: str, config: dict) -> tuple[str, str, dict]:
- gistid = config.get("gistid", "")
- filename = config.get("filename", "")
+ def _generate_payload(self, content: str, item: StorageItem) -> tuple[str, str, dict]:
+ gist_id = item.gist_id
+ filename = item.filename
- url = f"{self.api_address}/{gistid}"
+ url = f"{self.api_address}/{gist_id}"
headers = {
"Accept": "application/vnd.github+json",
"Authorization": f"Bearer {self.token}",
@@ -436,29 +436,25 @@ def _generate_payload(self, content: str, config: dict) -> tuple[str, str, dict]
def _is_success(self, response: HTTPResponse) -> bool:
return response and response.getcode() == 200
- def filter_push(self, config: dict) -> dict:
- if not self.token or not isinstance(config, dict):
+ def filter_push(self, items: dict[str, StorageItem]) -> dict[str, StorageItem]:
+ if not self.token or not isinstance(items, dict):
return {}
- return {
- k: v
- for k, v in config.items()
- if k and isinstance(v, dict) and v.get("gistid", "") and v.get("filename", "")
- }
+ return {k: v for k, v in items.items() if k and isinstance(v, StorageItem) and v.gist_id and v.filename}
- def raw_url(self, config: dict) -> str:
- if not config or type(config) != dict:
+ def raw_url(self, item: StorageItem) -> str:
+ if not isinstance(item, StorageItem):
return ""
- username = utils.trim(config.get("username", ""))
- gistid = utils.trim(config.get("gistid", ""))
- revision = utils.trim(config.get("revision", ""))
- filename = utils.trim(config.get("filename", ""))
+ username = utils.trim(item.username)
+ gist_id = utils.trim(item.gist_id)
+ revision = utils.trim(item.revision)
+ filename = utils.trim(item.filename)
- if not username or not gistid or not filename:
+ if not username or not gist_id or not filename:
return ""
- prefix = f"{self.domain}/{username}/{gistid}"
+ prefix = f"{self.domain}/{username}/{gist_id}"
if revision:
return f"{prefix}/raw/{revision}/{filename}"
@@ -480,17 +476,17 @@ def __init__(self, token: str, base: str = "") -> None:
self.domain = base
self.api_address = f"{base}/save"
- def validate(self, config: dict) -> bool:
- if not config or type(config) != dict:
+ def validate(self, item: StorageItem) -> bool:
+ if not isinstance(item, StorageItem):
return False
- fileid = config.get("fileid", "")
- return "" != self.token.strip() and "" != utils.trim(fileid)
+ file_id = item.file_id
+ return "" != self.token.strip() and "" != utils.trim(file_id)
- def _generate_payload(self, content: str, config: dict) -> tuple[str, str, dict]:
- fileid = config.get("fileid", "")
- password = config.get("password", "")
- expire = config.get("expire", 0)
+ def _generate_payload(self, content: str, item: StorageItem) -> tuple[str, str, dict]:
+ file_id = item.file_id
+ password = item.password
+ expire = item.expire
headers = {
"Cookie": f"token={self.token}",
@@ -501,7 +497,7 @@ def _generate_payload(self, content: str, config: dict) -> tuple[str, str, dict]
if isinstance(expire, int) and expire > 0:
headers["x-expire"] = str(expire)
- url = f"{self.api_address}/{fileid}"
+ url = f"{self.api_address}/{file_id}"
if password:
url = f"{url}/{password}"
@@ -517,25 +513,25 @@ def _is_success(self, response: HTTPResponse) -> bool:
except:
return False
- def filter_push(self, config: dict) -> dict:
+ def filter_push(self, items: dict[str, StorageItem]) -> dict[str, StorageItem]:
records = {}
- for k, v in config.items():
- if v.get("fileid", "") and self.token:
+ for k, v in items.items():
+ if v.file_id and self.token:
records[k] = v
return records
- def raw_url(self, config: dict) -> str:
- if not config or type(config) != dict:
+ def raw_url(self, item: StorageItem) -> str:
+ if not isinstance(item, StorageItem):
return ""
- fileid = utils.trim(config.get("fileid", ""))
- password = utils.trim(config.get("password", ""))
+ file_id = utils.trim(item.file_id)
+ password = utils.trim(item.password)
- if not fileid:
+ if not file_id:
return ""
- url = f"{self.domain}/r/{fileid}"
+ url = f"{self.domain}/r/{file_id}"
if password:
url = f"{url}/{password}"
@@ -545,52 +541,22 @@ def raw_url(self, config: dict) -> str:
SUPPORTED_ENGINES = set(["gist", "imperial", "pastefy", "pastegg", "qbin"] + [LOCAL_STORAGE])
-@dataclass
-class PushConfig(object):
- # storage type
- engine: str = ""
-
- # storage token
- token: str = ""
-
- # storage base address
- base: str = ""
-
- # storage domain address
- domain: str = ""
-
- @classmethod
- def from_dict(cls, data: dict) -> "PushConfig":
- if not data or type(data) != dict:
- return None
-
- engine = utils.trim(data.get("engine", ""))
- if engine not in SUPPORTED_ENGINES:
- return None
-
- token = utils.trim(data.get("token", ""))
- base = utils.trim(data.get("base", ""))
- domain = utils.trim(data.get("domain", ""))
-
- return cls(engine=engine, token=token, base=base, domain=domain)
-
-
-def get_instance(config: PushConfig) -> PushTo:
- if not config or not isinstance(config, PushConfig):
- raise ValueError("[PushError] invalid push config")
+def get_instance(storage: StorageConfig) -> PushTo:
+ if not isinstance(storage, StorageConfig):
+ raise ValueError("[PushError] invalid storage config")
- engine = utils.trim(config.engine)
- if not engine:
+ engine = utils.trim(storage.engine)
+ if engine not in SUPPORTED_ENGINES:
raise ValueError(f"[PushError] unknown storge type: {engine}")
- token = utils.trim(config.token or os.environ.get("PUSH_TOKEN", ""))
+ token = utils.trim(storage.token or os.environ.get("PUSH_TOKEN", ""))
if engine != LOCAL_STORAGE and not token:
- raise ValueError(f"[PushError] not found 'PUSH_TOKEN' in environment variables, please check it and try again")
+ raise ValueError("[PushError] not found 'PUSH_TOKEN' in environment variables, please check it and try again")
if engine == "gist":
return PushToGist(token=token)
- base, domain = utils.trim(config.base), utils.trim(config.domain)
+ base, domain = utils.trim(storage.base), utils.trim(storage.domain)
if engine == "imperial":
return PushToImperial(token=token, base=base, domain=domain)
elif engine == "pastefy":
diff --git a/subscribe/renewal.py b/subscribe/renewal.py
index 1fc42cc3ac..5f264abb2b 100644
--- a/subscribe/renewal.py
+++ b/subscribe/renewal.py
@@ -19,6 +19,7 @@
from datetime import datetime
import utils
+from config.models import RenewJob
from logger import logger
warnings.filterwarnings("ignore")
@@ -86,7 +87,9 @@ def get_cookies(
return utils.extract_cookie(text), authorization
-def generate_headers(domain: str, cookies: str, authorization: str, headers: dict = None) -> dict:
+def generate_headers(
+ domain: str, cookies: str, authorization: str, headers: dict[str, str] | None = None
+) -> dict[str, str]:
if not headers:
headers = {"user-agent": utils.USER_AGENT}
@@ -100,7 +103,9 @@ def generate_headers(domain: str, cookies: str, authorization: str, headers: dic
return headers
-def login(url: str, params: dict, headers: dict, retry: int = 3, jsonify: bool = False) -> tuple[str, str]:
+def login(
+ url: str, params: dict[str, object], headers: dict[str, str], retry: int = 3, jsonify: bool = False
+) -> tuple[str, str]:
if not params:
logger.error("[RenewalError] cannot login because parameters is empty")
return "", ""
@@ -137,7 +142,7 @@ def login(url: str, params: dict, headers: dict, retry: int = 3, jsonify: bool =
return "", ""
-def order(url: str, params: dict, headers: dict, retry: int = 3, jsonify: bool = False) -> str:
+def order(url: str, params: dict[str, object], headers: dict[str, str], retry: int = 3, jsonify: bool = False) -> str:
try:
if jsonify:
headers["Content-Type"] = "application/json"
@@ -165,7 +170,7 @@ def order(url: str, params: dict, headers: dict, retry: int = 3, jsonify: bool =
logger.error("[OrderError] URL: {}".format(utils.extract_domain(url)))
-def fetch(url: str, headers: dict, retry: int = 3) -> str:
+def fetch(url: str, headers: dict[str, str], retry: int = 3) -> str:
try:
request = urllib.request.Request(url, headers=headers, method="GET")
response = urllib.request.urlopen(request, timeout=10, context=utils.CTX)
@@ -189,7 +194,9 @@ def fetch(url: str, headers: dict, retry: int = 3) -> str:
logger.error("[FetchError] URL: {}".format(utils.extract_domain(url)))
-def payment(url: str, params: dict, headers: dict, retry: int = 3, jsonify: bool = False) -> bool:
+def payment(
+ url: str, params: dict[str, object], headers: dict[str, str], retry: int = 3, jsonify: bool = False
+) -> bool:
try:
data = urllib.parse.urlencode(params).encode(encoding="UTF8")
if jsonify:
@@ -220,13 +227,13 @@ def payment(url: str, params: dict, headers: dict, retry: int = 3, jsonify: bool
def checkout(
domain: str,
coupon: str,
- headers: dict,
+ headers: dict[str, str],
planid: int = -1,
retry: int = 3,
link: str = "",
api_prefix: str = "",
jsonify: bool = False,
-) -> dict:
+) -> dict[str, object]:
if utils.isblank(domain) or utils.isblank(coupon):
return {}
@@ -274,7 +281,7 @@ def checkout(
def get_payment_method(
domain: str, cookies: str, authorization: str = "", retry: int = 3, api_prefix: str = ""
-) -> list:
+) -> list[dict[str, object]]:
if not domain or (not cookies and not authorization):
logger.error(f"query payment method error, cookies and authorization is empty, domain: {domain}")
return []
@@ -294,7 +301,7 @@ def get_payment_method(
return []
-def unclosed_ticket(domain: str, headers: dict, api_prefix: str = "") -> tuple[int, int, str]:
+def unclosed_ticket(domain: str, headers: dict[str, str], api_prefix: str = "") -> tuple[int, int, str]:
if utils.isblank(domain) or not headers:
logger.info(f"[TicketError] cannot fetch tickets because invalidate arguments, domain: {domain}")
return -1, -1, ""
@@ -319,7 +326,7 @@ def unclosed_ticket(domain: str, headers: dict, api_prefix: str = "") -> tuple[i
def close_ticket(
- domain: str, tid: int, headers: dict, retry: int = 3, api_prefix: str = "", jsonify: bool = False
+ domain: str, tid: int, headers: dict[str, str], retry: int = 3, api_prefix: str = "", jsonify: bool = False
) -> bool:
if utils.isblank(domain) or tid < 0 or not headers or retry <= 0:
logger.info(f"[TicketError] cannot close ticket because invalidate arguments, domain: {domain}, tid: {tid}")
@@ -359,7 +366,7 @@ def close_ticket(
def submit_ticket(
domain: str,
cookies: str,
- ticket: dict,
+ ticket: dict[str, object],
authorization: str = "",
retry: int = 3,
api_prefix: str = "",
@@ -514,7 +521,7 @@ def get_free_plan(
return None
-def isfree(planid: str, package: str, price: float, discount: dict) -> bool:
+def isfree(planid: str, package: str, price: float, discount: dict[str, object]) -> bool:
# 不存在的套餐
if utils.isblank(planid) or utils.isblank(package) or price is None:
return False
@@ -610,7 +617,7 @@ def get_subscribe_info(
def flow(
domain: str,
- params: dict,
+ params: dict[str, object],
reset: bool = False,
retry: int = 3,
cookies: str = "",
@@ -697,18 +704,18 @@ def flow(
return success
-def add_traffic_flow(domain: str, params: dict, jsonify: bool = False) -> str:
- if not domain or not params:
- logger.error(f"[RenewalError] invalidate arguments")
+def add_traffic_flow(domain: str, job: RenewJob, jsonify: bool = False) -> str:
+ if not domain or not isinstance(job, RenewJob) or not job.account:
+ logger.error("[RenewalError] invalidate arguments")
return ""
try:
- email = base64.b64decode(params.get("email", "")).decode()
- password = base64.b64decode(params.get("passwd", "")).decode()
+ email = base64.b64decode(job.account.email).decode() if job.account.email else ""
+ password = base64.b64decode(job.account.password).decode() if job.account.password else ""
if utils.isblank(email) or utils.isblank(password):
logger.info(f"[RenewalError] email or password cannot be empty, domain: {domain}")
return ""
- api_prefix = params.get("api_prefix", "")
+ api_prefix = job.api_prefix
cookies, authorization = get_cookies(
domain=domain,
username=email,
@@ -726,10 +733,10 @@ def add_traffic_flow(domain: str, params: dict, jsonify: bool = False) -> str:
logger.info(f"[RenewalError] cannot fetch subscribe information")
return ""
- plan_id = params.get("plan_id", subscribe.plan_id)
- package = params.get("package", subscribe.package)
- coupon_code = params.get("coupon_code", "")
- method = params.get("method", -1)
+ plan_id = job.plan_id if job.plan_id is not None else subscribe.plan_id
+ package = job.package or subscribe.package
+ coupon_code = job.coupon_code
+ method = job.method if job.method is not None else -1
if method <= 0:
methods = get_payment_method(
domain=domain,
@@ -753,7 +760,7 @@ def add_traffic_flow(domain: str, params: dict, jsonify: bool = False) -> str:
"jsonify": jsonify,
}
- renew = params.get("enable", True)
+ renew = job.enable
if renew and subscribe.reset_enable and subscribe.used_rate >= 0.8:
success = flow(
domain=domain,
@@ -789,20 +796,17 @@ def add_traffic_flow(domain: str, params: dict, jsonify: bool = False) -> str:
f"skip renew traffic plan, domain: {domain}\trenew: {renew}\tenable: {subscribe.renew_enable}\texpired-days: {subscribe.expired_days}"
)
- # 提交工单重置流量
- ticket = params.get("ticket", {})
- if ticket and type(ticket) == dict:
- enable = ticket.pop("enable", True)
- autoreset = ticket.pop("autoreset", False)
+ ticket = job.account.ticket
+ if ticket:
# 过期时间 <= 5 或者 流量使用例 >= 0.8 或者 重置日期 <= 1 且不会自动重置时提交工单
- if enable and (
+ if ticket.enable and (
(subscribe.expired_days <= 5 or subscribe.used_rate >= 0.8)
- or (not autoreset and subscribe.reset_day <= 1)
+ or (not ticket.auto_reset and subscribe.reset_day <= 1)
):
success = submit_ticket(
domain=domain,
cookies=cookies,
- ticket=ticket,
+ ticket=ticket.to_dict(),
authorization=authorization,
api_prefix=api_prefix,
jsonify=jsonify,
diff --git a/subscribe/scripts/__init__.py b/subscribe/scripts/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/subscribe/scripts/commons.py b/subscribe/scripts/commons.py
deleted file mode 100644
index 84b9119312..0000000000
--- a/subscribe/scripts/commons.py
+++ /dev/null
@@ -1,21 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# @Author : wzdnzd
-# @Time : 2022-11-12
-
-import json
-
-import push
-from logger import logger
-
-
-def persist(config: push.PushConfig, data: dict, persist: dict, meta: str = "") -> None:
- try:
- pushtool = push.get_instance(config=config)
- if data is None or type(data) != dict or not pushtool.validate(config=persist):
- logger.debug(f"[{meta}] skip persist subscibes because fileid or data is empty")
- return
-
- pushtool.push_to(content=json.dumps(data), config=persist, group="subscribes")
- except:
- logger.error(f"[{meta}] occur error when persist subscribes")
diff --git a/subscribe/scripts/purefast.py b/subscribe/scripts/purefast.py
deleted file mode 100644
index eefaf167a7..0000000000
--- a/subscribe/scripts/purefast.py
+++ /dev/null
@@ -1,430 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# @Author : wzdnzd
-# @Time : 2022-12-07
-
-import concurrent.futures
-import gzip
-import json
-import os
-import random
-import re
-import sys
-import time
-import urllib
-import urllib.parse
-import urllib.request
-import warnings
-from base64 import b64decode
-from http.client import HTTPResponse
-from typing import Any
-from urllib.error import HTTPError
-from urllib.request import OpenerDirector
-
-import utils
-from logger import logger
-
-warnings.filterwarnings("ignore")
-from http import cookiejar
-
-
-def login(
- url: str,
- opener: OpenerDirector,
- cookies: cookiejar.CookieJar,
- params: dict,
- headers: dict,
- endtime: int,
- retry: int = 3,
-) -> tuple[bool, dict]:
- if utils.isblank(url) or not params or retry <= 0 or checkconn(opener, cookies):
- logger.error(f"[PFVPNLoginError] cannot login, url: {url}, retry: {retry}")
- return False, {}
-
- if not headers:
- headers = {
- "user-agent": utils.USER_AGENT,
- "accept": "application/json, text/javascript, */*; q=0.01",
- "accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
- "content-type": "application/x-www-form-urlencoded; charset=UTF-8",
- "accept-encoding": "gzip, deflate",
- "referer": url,
- "x-requested-with": "XMLHttpRequest",
- }
-
- try:
- successed, skip, count = False, False, 25
- while not successed and count > 0 and time.time() < endtime:
- count -= 1
- data = urllib.parse.urlencode(params).encode(encoding="UTF8")
- request = urllib.request.Request(url, data=data, headers=headers, method="POST")
- response = opener.open(request, timeout=10)
- cookie = response.getheader("Set-Cookie")
-
- if response.getcode() == 200:
- if not skip and not utils.isblank(specified_cookie(cookie, "ge_ua_p", False)):
- skip, headers = bypass(
- url=url,
- opener=opener,
- cookies=cookies,
- endtime=endtime,
- content=read(response),
- headers=headers,
- retry=3,
- starttime=time.time(),
- )
- continue
-
- cookie = get_cookie(cookie)
- successed = not utils.isblank(cookie)
- if successed:
- cookie = add_or_replace(headers.get("cookie", ""), cookie)
- headers["cookie"] = cookie
- break
-
- time.sleep(random.randint(5, 15) / 10)
- return successed, headers
- except:
- return login(url, opener, cookies, params, headers, endtime, retry - 1)
-
-
-def checkin(
- url: str,
- opener: OpenerDirector,
- cookies: cookiejar.CookieJar,
- headers: dict,
- endtime: int,
- retry: int = 3,
-) -> bool:
- if utils.isblank(url) or not headers or retry <= 0 or checkconn(opener, cookies):
- logger.error(f"[PFVPNError] cannot checkin, url: {url}, retry: {retry}")
- return False
- try:
- successed, skip, count = False, False, 25
- while not skip and not successed and count > 0 and time.time() < endtime:
- count -= 1
- request = urllib.request.Request(url, headers=headers, method="POST")
- response = opener.open(request, timeout=10)
-
- if response.getcode() == 200:
- if not utils.isblank(specified_cookie(response.getheader("Set-Cookie"), "ge_ua_p", False)):
- skip, headers = bypass(
- url=url,
- opener=opener,
- cookies=cookies,
- endtime=endtime,
- content=read(response),
- headers=headers,
- retry=3,
- starttime=time.time(),
- )
- continue
-
- content = read(response)
- try:
- data = json.loads(content)
- successed = data.get("ret", 0) == 1
- if successed:
- message = data.get("msg", "")
- logger.info(f"[PFVPN] checkin successed, message: {message}")
- break
- except:
- logger.error(f"[PFVPNError] checkin failed, message: {content}")
-
- time.sleep(random.randint(5, 15) / 10)
- return successed
- except HTTPError as e:
- if e.status == 307:
- cookie = specified_cookie(e.headers["Set-Cookie"], "WAF_VALIDATOR_ID", True)
- headers["cookie"] = add_or_replace(source=headers.get("cookie", ""), dest=cookie)
- headers["x-cache"] = "BYPASS"
- return checkin(url, opener, cookies, headers, endtime, retry - 1)
- except:
- return checkin(url, opener, cookies, headers, endtime, retry - 1)
-
-
-def get_cookie(text: str) -> str:
- regex = "(__cfduid|uid|email|key|ip|expire_in)=(.+?);"
- if utils.isblank(text):
- return ""
-
- content = re.findall(regex, text)
- cookie = ";".join(["=".join(x) for x in content]).strip()
-
- return cookie
-
-
-def run(domain: str, params: dict, timeout: int) -> bool:
- domain = utils.extract_domain(url=domain, include_protocal=True)
- if not domain:
- logger.error(f"[PFVPNError] cannot checkin because domain: {domain} is invalidate")
- return False
-
- login_url = domain + params.get("login", "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/auth/login")
- checkin_url = domain + params.get("checkin", "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/user/checkin")
-
- email = params.get("username", "").strip()
- username = email.split("@", maxsplit=1)[0]
- passwd = params.get("password", "").strip()
- try:
- if utils.isblank(email) or utils.isblank(passwd):
- logger.error(f"[PFVPNError] skip checkin for username=[{username}]")
- return False
-
- passwd = b64decode(passwd)
- except:
- logger.error(f"[PFVPNError] username=[{username}], password error, please encoding it with base64")
- return False
-
- user_info = {"email": email, "passwd": passwd, "code": ""}
- opener, cookies = build_opener()
- starttime, endtime = time.time(), time.time() + max(timeout, 5)
-
- successed, headers = login(login_url, opener, cookies, user_info, None, endtime, 3)
- if not successed:
- logger.error(f"[PFVPNError] login failed, skip checkin, username: {username}")
- return successed
-
- headers["referer"] = domain + "/user"
- headers["content-type"] = "application/json"
- successed = checkin(checkin_url, opener, cookies, headers, endtime, 5)
-
- cost = round(time.time() - starttime, 2)
- logger.info(f"[PFVPNInfo] finished checkin, username: {username}, result: {successed}, cost: {cost}s")
-
- return successed
-
-
-def calsum(cpk: str, nonce: int) -> int:
- if utils.isblank(cpk):
- return -1
-
- num = 0
- for i in range(len(cpk)):
- c = cpk[i]
- if c.isalnum():
- num += ord(c) * (nonce + i)
- return num
-
-
-def aboartable_run(domain: str, params: dict, timeout: int = 180) -> None:
- timeout = max(timeout, 0)
- with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
- try:
- future = executor.submit(run, domain, params, timeout)
- future.result(timeout=timeout)
- except concurrent.futures.TimeoutError:
- username = params.get("username", "").split("@", maxsplit=1)[0]
- logger.error(f"[PFVPNError] checkin task aborting due to timeout {timeout}s, username: {username}")
- executor.shutdown(wait=False, cancel_futures=True)
-
-
-def bypass(
- url: str,
- opener: OpenerDirector,
- cookies: cookiejar.CookieJar,
- endtime: int,
- content: str = "",
- headers: dict = None,
- retry: int = 3,
- starttime: int = -1,
-) -> tuple[bool, dict]:
- if utils.isblank(url) or retry <= 0 or checkconn(opener=opener, cookies=cookies):
- return False, headers
-
- if not headers:
- headers = {
- "user-agent": utils.USER_AGENT,
- "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
- "accept-language": "zh-CN,zh;q=0.9",
- "content-type": "application/x-www-form-urlencoded; charset=UTF-8",
- "accept-encoding": "gzip, deflate",
- "referer": url,
- "x-requested-with": "XMLHttpRequest",
- }
-
- try:
- if utils.isblank(content):
- request = urllib.request.Request(url, headers=headers)
- response = opener.open(request, timeout=10)
- starttime = time.time()
- if response.getcode() >= 400:
- return False, headers
- content = read(response)
-
- groups = re.findall(r'var\s+cpk(?:\s+)?=(?:\s+)?"(.*)"', content, re.I)
- cpkname = groups[0] if groups else "ge_ua_p"
-
- groups = re.findall(r'var\s+step(?:\s+)?=(?:\s+)?"(.*)"', content, re.I)
- step = groups[0] if groups else "prev"
-
- groups = re.findall(r"var\s+nonce(?:\s+)?=(?:\s+)?(\d+);", content, re.I)
- nonce = int(groups[0]) if groups else -1
-
- cpkvalue = specified_cookie(cookies, cpkname, False)
- if utils.isblank(cpkvalue) or utils.isblank(step) or nonce < 0:
- return False, headers
-
- time.sleep(max(5 - (time.time() - starttime), 0))
- sumval = calsum(cpk=cpkvalue, nonce=nonce)
- data = urllib.parse.urlencode({"sum": sumval, "nonce": nonce}).encode(encoding="UTF8")
- headers["x-ge-ua-step"] = step
- headers["cookie"] = add_or_replace(source=headers.get("cookie", ""), dest=f"{cpkname}={cpkvalue}")
- request = urllib.request.Request(url, data=data, headers=headers, method="POST")
-
- successed, count = False, 20
- while not successed and count > 0 and time.time() < endtime:
- count -= 1
-
- response = opener.open(request, timeout=10)
- if response.getcode() == 200:
- text = read(response)
- if not utils.isblank(text):
- successed = json.loads(text).get("ok", False)
- if successed:
- break
-
- time.sleep(random.randint(3, 8) / 10)
-
- headers.pop("x-ge-ua-step", "")
- # remove ge_ua_p from cookie
- cookie = add_or_replace(source=headers.get("cookie", ""), dest=f"{cpkname}=")
- headers["cookie"] = cookie
-
- if successed:
- # add or replace ge_ua_key and lang
- guk = specified_cookie(response.getheader("Set-Cookie"), "ge_ua_key", concat=True)
- cookie = add_or_replace(source=cookie, dest=f"{guk}; lang=zh-cn")
- headers["cookie"] = cookie
- return successed, headers
- except:
- return bypass(
- url=url,
- opener=opener,
- cookies=cookies,
- endtime=endtime,
- content=content,
- headers=headers,
- retry=retry - 1,
- starttime=starttime,
- )
-
-
-def specified_cookie(items: Any, key: str, concat: bool = False) -> str:
- value = ""
-
- if not items or utils.isblank(key):
- return value
-
- if type(items) == cookiejar.CookieJar:
- for cookie in items:
- if key == cookie.name:
- value = cookie.value
- break
-
- elif type(items) == str:
- for cookie in items.split(";"):
- words = cookie.split("=", maxsplit=1)
- if len(words) != 2:
- continue
- if key == words[0]:
- value = words[1]
- break
-
- return f"{key}={value}" if concat and not utils.isblank(value) else value
-
-
-def add_or_replace(source: str, dest: str) -> str:
- def covertto(content: str) -> dict:
- targets = {}
- if utils.isblank(content):
- return targets
-
- for text in content.split(";"):
- text = text.strip()
- words = text.split("=", maxsplit=1)
- if len(words) != 2 or utils.isblank(words[0]):
- continue
- targets[words[0]] = words[1]
-
- return targets
-
- if utils.isblank(dest):
- return source
-
- raws, others = covertto(source), covertto(dest)
- raws.update(others)
- items = [f"{k}={v}" for k, v in raws.items() if not utils.isblank(v)]
- return "; ".join(items)
-
-
-def read(response: HTTPResponse) -> str:
- if not response or type(response) != HTTPResponse:
- return ""
- try:
- content = response.read()
- try:
- content = gzip.decompress(content).decode("utf8")
- except:
- content = str(content, encoding="utf8")
- return content
- except:
- return ""
-
-
-def loadconf(filename: str = "") -> dict:
- domain = os.environ.get("PFVPN_DOMAIN", "https://purefast.net")
- ustr = os.environ.get("PFVPN_USERNAMES", "")
- pstr = os.environ.get("PFVPN_PASSWORDS", "")
-
- configs = {}
- try:
- if utils.isblank(domain) or utils.isblank(ustr) or utils.isblank(pstr):
- if not utils.isblank(filename) and os.path.exists(filename) and os.path.isfile(filename):
- configs = json.loads(open(filename, "r").read())
- else:
- configs["domain"] = domain
- usernames, passwords = ustr.split(";"), pstr.split(";")
- if len(usernames) != len(passwords):
- logger.warning(f"[PFVPNError] the number of usernames and the number of passwords do not match")
- else:
- accounts = []
- for i in range(len(usernames)):
- accounts.append(
- {
- "username": usernames[i].strip(),
- "password": passwords[i].strip(),
- }
- )
- configs["accounts"] = accounts
- except:
- logger.error(f"[PFVPNError] loading config error, filename: {filename}")
-
- return configs
-
-
-def build_opener() -> tuple[OpenerDirector, cookiejar.CookieJar]:
- cookies = cookiejar.CookieJar()
- cookie_handle = urllib.request.HTTPCookieProcessor(cookies)
- http_handle = urllib.request.HTTPHandler()
- https_handle = urllib.request.HTTPSHandler()
- opener = urllib.request.build_opener(http_handle, https_handle, cookie_handle)
-
- return opener, cookies
-
-
-def checkconn(opener: OpenerDirector, cookies: cookiejar.CookieJar) -> bool:
- return opener is None or type(opener) != OpenerDirector or cookies is None or type(cookies) != cookiejar.CookieJar
-
-
-def main(filepath: str) -> None:
- config = loadconf(filename=filepath)
- domain = config.get("domain", "https://purefast.net")
- accounts = config.get("accounts", [])
-
- if utils.isblank(domain) or not accounts:
- logger.error(f"[PFVPNError] skip checkin because cannot found any valid config, exit")
- sys.exit(1)
-
- params = [[domain, x, 300] for x in accounts if x]
- utils.multi_thread_run(func=aboartable_run, tasks=params, show_progress=True)
diff --git a/subscribe/subconverter.py b/subscribe/subconverter.py
index fa4b552ad0..72266f24a7 100644
--- a/subscribe/subconverter.py
+++ b/subscribe/subconverter.py
@@ -7,31 +7,13 @@
from threading import Lock
import utils
+from config.models import CONVERT_TARGETS
from logger import logger
PATH = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
FILE_LOCK = Lock()
-CONVERT_TARGETS = [
- "clash",
- "v2ray",
- "singbox",
- "mixed",
- "clashr",
- "quan",
- "quanx",
- "loon",
- "ss",
- "sssub",
- "ssd",
- "ssr",
- "surfboard",
- "surge",
- # "surge&ver=2",
- # "surge&ver=3",
-]
-
def get_filename(target: str) -> str:
target = utils.trim(target).lower()
@@ -103,7 +85,7 @@ def generate_conf(
lines.extend(["emoji=false", "add_emoji=false"])
if ignore_exclude:
- lines.append("exclude=流量|过期|剩余|时间|Expire|Traffic")
+ lines.append("exclude=[到过]期|Expire|Traffic|剩余流量|时间|官网|产品|联系")
lines.append("\n")
content = "\n".join(lines)
diff --git a/subscribe/utils.py b/subscribe/utils.py
index 957731a715..cf488518f4 100644
--- a/subscribe/utils.py
+++ b/subscribe/utils.py
@@ -35,7 +35,7 @@
CTX.verify_mode = ssl.CERT_NONE
USER_AGENT = (
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"
)
@@ -43,16 +43,28 @@
FILEPATH_PROTOCAL = "file:///"
-# ChatGPT 标识
-CHATGPT_FLAG = "-GPT"
-
-
DEFAULT_HTTP_HEADERS = {
"User-Agent": USER_AGENT,
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
}
+_SUSPICIOUS_URL_RE = re.compile(
+ r"(?i)("
+ r"(?:^|//)(?:[^/]*\.)?(?:speedtest|librespeed|fast\.com|cachefly\.net|thinkbroadband\.com|speed\.cloudflare\.com)"
+ r"|/__down(?:\?|$)"
+ r"|\.(?:zip|iso|exe|mp4|mkv|avi|tar|tgz|gz|7z|rar|bin|img|dmg|apk|msi|pdf|"
+ r"css|js|png|jpe?g|gif|svg|webp|woff2?|ico|map|torrent)(?:\?|$)"
+ r")"
+)
+
+
+def is_suspicious_url(url: str) -> bool:
+ if not url or not isinstance(url, str):
+ return False
+ return bool(_SUSPICIOUS_URL_RE.search(url.strip()))
+
+
def random_chars(length: int, punctuation: bool = False) -> str:
length = max(length, 1)
if punctuation:
@@ -65,14 +77,14 @@ def random_chars(length: int, punctuation: bool = False) -> str:
def http_get(
url: str,
- headers: dict = None,
- params: dict = None,
+ headers: dict[str, str] | None = None,
+ params: dict[str, str] | None = None,
retry: int = 3,
proxy: str = "",
interval: float = 0,
timeout: float = 10,
trace: bool = False,
- max_size=None,
+ max_size: int | None = None,
) -> str:
if not isurl(url=url):
logger.error(f"invalid url: {url}")
@@ -190,7 +202,7 @@ def extract_cookie(text: str) -> str:
return cookie
-def cmd(command: list, output: bool = False) -> tuple[bool, str]:
+def cmd(command: list[str], output: bool = False) -> tuple[bool, str]:
if command is None or len(command) == 0:
return False, ""
@@ -288,7 +300,7 @@ def encoding_url(url: str) -> str:
return url
-def write_file(filename: str, lines: list) -> bool:
+def write_file(filename: str, lines: list[str] | str) -> bool:
if not filename or not lines:
logger.error(f"filename or lines is empty, filename: {filename}")
return False
@@ -356,6 +368,13 @@ def load_dotenv(enviroment: str = ".env") -> None:
os.environ[k] = v
+def env_bool(name: str, default: bool = False) -> bool:
+ raw = trim(os.environ.get(name, ""))
+ if not raw:
+ return default
+ return raw.lower() in ["true", "1"]
+
+
def hide(url: str) -> str:
# len('http://') equals 7
if isblank(url) or len(url) < 7:
@@ -413,12 +432,12 @@ def http_error_302(
def http_post(
url: str,
- headers: dict = None,
- params: dict = {},
+ headers: dict[str, str] | None = None,
+ params: dict[str, object] | None = None,
retry: int = 3,
timeout: float = 6,
allow_redirects: bool = True,
-) -> HTTPResponse:
+) -> HTTPResponse | None:
if params is None or type(params) != dict or retry <= 0:
return None
@@ -481,7 +500,7 @@ def url_complete(url: str, secret: bool = False) -> str:
return url
-def load_emoji_pattern(filepath: str = "") -> dict:
+def load_emoji_pattern(filepath: str = "") -> dict[str, str]:
filepath = trim(filepath)
if not filepath:
workspace = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
@@ -509,7 +528,7 @@ def load_emoji_pattern(filepath: str = "") -> dict:
return patterns
-def get_emoji(text: str, patterns: dict, default: str = "") -> str:
+def get_emoji(text: str, patterns: dict[str, str], default: str = "") -> str:
if not patterns or type(patterns) != dict or not text or type(text) != str:
return default
@@ -530,7 +549,7 @@ def get_subpath(api_prefix: str, default: str = "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/api/v1/") -> str:
return path
-def multi_process_run(func: typing.Callable, tasks: list) -> list:
+def multi_process_run(func: typing.Callable[..., object], tasks: list[object]) -> list[object]:
if not func or not isinstance(func, typing.Callable):
logger.error(f"skip execute due to func is not callable")
return []
@@ -566,12 +585,12 @@ def multi_process_run(func: typing.Callable, tasks: list) -> list:
def multi_thread_run(
- func: typing.Callable,
- tasks: list,
- num_threads: int = None,
+ func: typing.Callable[..., object],
+ tasks: list[object],
+ num_threads: int | None = None,
show_progress: bool = False,
description: str = "",
-) -> list:
+) -> list[object]:
if not func or not tasks or not isinstance(tasks, list):
return []
diff --git a/subscribe/workflow.py b/subscribe/workflow.py
index 83f4a4ef85..56d142f1ba 100644
--- a/subscribe/workflow.py
+++ b/subscribe/workflow.py
@@ -6,11 +6,12 @@
import json
import os
import re
-from dataclasses import dataclass
+from dataclasses import dataclass, field
import renewal
import utils
from airport import ANOTHER_API_PREFIX, AirPort
+from config.models import NodeInput, ProcessConfig, RenewJob, SiteConfig
from logger import logger
from origin import Origin
from push import PushTo
@@ -18,83 +19,41 @@
@dataclass
class TaskConfig:
- # 任务名
name: str
-
- # subconverter程序名
bin_name: str
-
- # 任务编号
taskid: int = -1
-
- # 网址域名
domain: str = ""
-
- # 订阅地址
- sub: str = ""
-
- # 任务编号
+ nodes: NodeInput = field(default_factory=NodeInput)
index: int = 1
-
- # 失败重试次数
retry: int = 3
-
- # 最高允许倍率
- rate: float = 20.0
-
- # 套餐续期配置
- renew: dict = None
-
- # 优惠码
+ max_rate: float = 20.0
+ renew: RenewJob | None = None
coupon: str = ""
-
- # 节点重命名规则
rename: str = ""
-
- # 节点排除规则
exclude: str = ""
include: str = ""
-
- # ChatGPT连通性测试节点过滤规则
- chatgpt: dict = None
-
- # 是否检测节点存活状态
- liveness: bool = True
-
- # 是否强制开启 tls 及阻止跳过证书验证
- disable_insecure: bool = False
-
- # 覆盖subconverter默认exclude规则
- ignorede: bool = False
-
- # 是否允许特殊协议
+ check_alive: bool = True
+ require_tls: bool = False
+ ignore_default_exclude: bool = False
special_protocols: bool = False
-
- # 对于具有邮箱域名白名单且需要验证码的情况，是否使用 Gmail 别名邮箱尝试，为 True 时表示不使用
- rigid: bool = True
-
- # 是否丢弃可能需要人机验证的站点
- chuck: bool = False
-
- # 邀请码
+ allow_gmail_alias: bool = False
+ skip_captcha: bool = False
invite_code: str = ""
-
- # 接口地址前缀，如 /api/v1/ 或 /api?scheme=
api_prefix: str = "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/api/v1/"
-def execute(task_conf: TaskConfig) -> list:
+def execute(task_conf: TaskConfig) -> list[dict[str, object]]:
if not task_conf or not isinstance(task_conf, TaskConfig):
return []
obj = AirPort(
name=task_conf.name,
site=task_conf.domain,
- sub=task_conf.sub,
+ nodes=task_conf.nodes,
rename=task_conf.rename,
exclude=task_conf.exclude,
include=task_conf.include,
- liveness=task_conf.liveness,
+ check_alive=task_conf.check_alive,
coupon=task_conf.coupon,
api_prefix=task_conf.api_prefix,
)
@@ -105,30 +64,32 @@ def execute(task_conf: TaskConfig) -> list:
if task_conf.renew:
sub_url = renewal.add_traffic_flow(
domain=obj.ref,
- params=task_conf.renew,
+ job=task_conf.renew,
jsonify=obj.api_prefix == ANOTHER_API_PREFIX,
)
if sub_url and not obj.registed:
obj.registed = True
- obj.sub = sub_url
-
- cookie, authorization = obj.get_subscribe(
- retry=task_conf.retry,
- rigid=task_conf.rigid,
- chuck=task_conf.chuck,
- invite_code=task_conf.invite_code,
- )
+ obj.nodes.subscribe = sub_url
+
+ cookie, authorization = "", ""
+ if task_conf.nodes.empty() and task_conf.domain:
+ cookie, authorization = obj.get_subscribe(
+ retry=task_conf.retry,
+ allow_gmail_alias=task_conf.allow_gmail_alias,
+ skip_captcha=task_conf.skip_captcha,
+ invite_code=task_conf.invite_code,
+ )
proxies = obj.parse(
cookie=cookie,
auth=authorization,
retry=task_conf.retry,
- rate=task_conf.rate,
+ rate=task_conf.max_rate,
bin_name=task_conf.bin_name,
- disable_insecure=task_conf.disable_insecure,
- ignore_exclude=task_conf.ignorede,
- chatgpt=task_conf.chatgpt,
+ require_tls=task_conf.require_tls,
+ ignore_exclude=task_conf.ignore_default_exclude,
special_protocols=task_conf.special_protocols,
+ nodes=task_conf.nodes,
)
logger.info(
@@ -138,7 +99,7 @@ def execute(task_conf: TaskConfig) -> list:
return proxies
-def executewrapper(task_conf: TaskConfig) -> tuple[int, list]:
+def executewrapper(task_conf: TaskConfig) -> tuple[int, list[dict[str, object]]]:
if not task_conf:
return (-1, [])
@@ -147,7 +108,7 @@ def executewrapper(task_conf: TaskConfig) -> tuple[int, list]:
return (taskid, proxies)
-def liveness_fillter(proxies: list) -> tuple[list, list]:
+def liveness_fillter(proxies: list[dict[str, object]]) -> tuple[list[dict[str, object]], list[dict[str, object]]]:
if not list:
return [], []
@@ -161,13 +122,12 @@ def liveness_fillter(proxies: list) -> tuple[list, list]:
checks.append(p)
else:
p.pop("sub", "")
- p.pop("chatgpt", False)
nochecks.append(p)
return checks, nochecks
-def cleanup(filepath: str = "", filenames: list = []) -> None:
+def cleanup(filepath: str = "", filenames: list[str] | None = None) -> None:
if not filepath or not filenames:
return
@@ -177,7 +137,7 @@ def cleanup(filepath: str = "", filenames: list = []) -> None:
os.remove(filename)
-def dedup_task(tasks: list) -> list:
+def dedup_task(tasks: list[TaskConfig]) -> list[TaskConfig]:
if not tasks:
return []
items = []
@@ -188,7 +148,7 @@ def dedup_task(tasks: list) -> list:
return items
-def exists(tasks: list, task: TaskConfig) -> bool:
+def exists(tasks: list[TaskConfig], task: TaskConfig) -> bool:
if not isinstance(task, TaskConfig):
logger.error(f"[DedupError] need type 'TaskConfig' but got type '{type(task)}'")
return True
@@ -197,8 +157,10 @@ def exists(tasks: list, task: TaskConfig) -> bool:
found = False
for item in tasks:
- if task.sub != "":
- if task.sub == item.sub:
+ left = task.nodes.subscribe_list()
+ right = item.nodes.subscribe_list()
+ if left:
+ if left == right:
found = True
else:
if task.domain == item.domain and task.index == item.index:
@@ -211,83 +173,79 @@ def exists(tasks: list, task: TaskConfig) -> bool:
item.exclude = "|".join([item.exclude, task.exclude]).removeprefix("|")
if task.include:
item.include = "|".join([item.include, task.include]).removeprefix("|")
- break
+ break
return found
-def merge_config(configs: list) -> list:
- def judge_exists(raw: dict, target: dict) -> bool:
- if not raw or not target:
- return False
+def _subscribe_key(site: SiteConfig) -> str | list[str]:
+ subs = site.nodes.subscribe_list()
+ if len(subs) <= 1:
+ return subs[0] if subs else ""
+ return subs
+
- rsub = raw.get("sub").strip()
- tsub = target.get("sub", "")
- if not tsub:
+def merge_config(sites: list[SiteConfig]) -> list[SiteConfig]:
+ def judge_exists(raw: SiteConfig, target: SiteConfig) -> bool:
+ rsubs = raw.nodes.subscribe_list()
+ tsubs = target.nodes.subscribe_list()
+ rsub = rsubs[0] if rsubs else ""
+ if not tsubs:
if rsub:
return False
- return raw.get("domain", "").strip() == target.get("domain", "").strip()
- if isinstance(tsub, str):
- return rsub == tsub.strip()
- for sub in tsub:
- if rsub == sub.strip():
- return True
- return False
+ return utils.trim(raw.domain) == utils.trim(target.domain)
+ return rsub in tsubs
- if not configs:
+ if not sites:
return []
items = []
- for conf in configs:
- if not isinstance(conf, dict):
- logger.error(f"[MergeError] need type 'dict' but got type '{type(conf)}'")
+ for site in sites:
+ if not isinstance(site, SiteConfig):
+ logger.error(f"[MergeError] need type 'SiteConfig' but got type '{type(site)}'")
continue
- sub = conf.get("sub", "")
- if isinstance(sub, list) and len(sub) <= 1:
- sub = sub[0] if sub else ""
+ sub = _subscribe_key(site)
+ if isinstance(sub, str):
+ site.nodes.subscribe = sub
- # 人工维护配置，无需合并
- if isinstance(sub, list) or conf.get("renew", {}):
- items.append(conf)
+ if isinstance(sub, list) or site.renew:
+ items.append(site)
continue
found = False
- conf["sub"] = sub
for item in items:
- found = judge_exists(raw=conf, target=item)
+ found = judge_exists(raw=site, target=item)
if found:
- if conf.get("errors", 0) > item.get("errors", 0):
- item["errors"] = conf.get("errors", 0)
- if item.get("debut", False):
- item["debut"] = conf.get("debut", False)
- if not item.get("rename", ""):
- item["rename"] = conf.get("rename", "")
- if conf.get("exclude", ""):
- item["exclude"] = "|".join([item.get("exclude", ""), conf.get("exclude", "")]).removeprefix("|")
- if conf.get("include", ""):
- item["include"] = "|".join([item.get("include", ""), conf.get("include", "")]).removeprefix("|")
-
+ if site.errors > item.errors:
+ item.errors = site.errors
+ if item.debut:
+ item.debut = site.debut
+ if not item.rename:
+ item.rename = site.rename
+ if site.exclude:
+ item.exclude = "|".join([item.exclude, site.exclude]).removeprefix("|")
+ if site.include:
+ item.include = "|".join([item.include, site.include]).removeprefix("|")
break
-
if not found:
- items.append(conf)
-
+ items.append(site)
return items
-def refresh(config: dict, push: PushTo, alives: dict, filepath: str = "", skip_remark: bool = False) -> None:
- if not config or not push:
+def refresh(
+ config: ProcessConfig, push: PushTo, alives: dict[str, bool] | None, filepath: str = "", skip_remark: bool = False
+) -> None:
+ if not isinstance(config, ProcessConfig) or not isinstance(push, PushTo):
logger.error("[UpdateError] cannot update remote config because content is empty")
return
- # mark invalid crawled subscription
invalidsubs = None if (skip_remark or not alives) else [k for k, v in alives.items() if not v]
- if invalidsubs:
- crawledsub = config.get("crawl", {}).get("persist", {}).get("subs", "")
- threshold = max(config.get("threshold", 1), 1)
- pushconf = config.get("groups", {}).get(crawledsub, {})
- if push.validate(config=pushconf):
- url = push.raw_url(config=pushconf)
+ if invalidsubs and config.crawl:
+ crawledsub = config.crawl.persist.subscribe
+ threshold = max(config.crawl.max_fails, 1)
+ pushconf = config.storage.items.get(crawledsub)
+ if push.validate(item=pushconf):
+ url = push.raw_url(item=pushconf)
content = utils.http_get(url=url)
try:
data, count = json.loads(content), 0
@@ -295,57 +253,56 @@ def refresh(config: dict, push: PushTo, alives: dict, filepath: str = "", skip_r
record = data.pop(sub, None)
if not record:
continue
-
- defeat = record.get("defeat", 0) + 1
+ errors = record.get("errors", 0) + 1
count += 1
- if defeat <= threshold and standard_sub(url=sub):
- record["defeat"] = defeat
+ if errors <= threshold and standard_sub(url=sub):
+ record["errors"] = errors
data[sub] = record
-
if count > 0:
content = json.dumps(data)
- push.push_to(content=content, config=pushconf, group="crawled-remark")
+ push.push_to(content=content, item=pushconf, group="crawled-remark")
logger.info(f"[UpdateInfo] found {count} invalid crawled subscriptions")
- except:
- logger.error(f"[UpdateError] remark invalid crawled subscriptions failed")
+ except Exception:
+ logger.error("[UpdateError] remark invalid crawled subscriptions failed")
- update_conf = config.get("update", {})
- if not update_conf.get("enable", False):
+ if not config.update.enable:
logger.debug("[UpdateError] skip update remote config because enable=[False]")
return
- if not push.validate(config=update_conf):
- logger.error(f"[UpdateError] update config is invalidate")
+ if not push.validate(item=config.update.item):
+ logger.error("[UpdateError] update config is invalidate")
return
- domains = merge_config(configs=config.get("domains", []))
+ sites_conf = merge_config(sites=config.sites)
if alives:
sites = []
- for item in domains:
- source = item.get("origin", "")
- sub = item.get("sub", "")
- if isinstance(sub, list) and len(sub) <= 1:
- sub = sub[0] if sub else ""
- if source in [Origin.TEMPORARY.name, Origin.OWNED.name] or isinstance(sub, list) or alives.get(sub, False):
- item.pop("errors", None)
- item.pop("debut", None)
+ for item in sites_conf:
+ if not item.enable:
+ sites.append(item)
+ continue
+ sub = _subscribe_key(item)
+ source = item.origin
+ if (
+ source in [Origin.TEMPORARY.name, Origin.OWNED.name]
+ or isinstance(sub, list)
+ or (isinstance(sub, str) and alives.get(sub, False))
+ ):
+ item.errors = 0
+ item.debut = False
sites.append(item)
continue
-
- errors = item.get("errors", 1)
expire = Origin.get_expire(source)
- if errors < expire and not item.get("debut", False):
- item.pop("debut", None)
+ if item.errors < expire and not item.debut:
+ item.debut = False
sites.append(item)
+ config.sites = sites
+ sites_conf = sites
- config["domains"] = sites
- domains = config.get("domains", [])
-
- if not domains:
- logger.error("[UpdateError] skip update remote config because domians is empty")
+ if not sites_conf:
+ logger.error("[UpdateError] skip update remote config because sites is empty")
return
- content = json.dumps(config)
+ content = json.dumps(config.to_dict())
if filepath:
directory = os.path.abspath(os.path.dirname(filepath))
os.makedirs(directory, exist_ok=True)
@@ -353,7 +310,7 @@ def refresh(config: dict, push: PushTo, alives: dict, filepath: str = "", skip_r
f.write(content)
f.flush()
- push.push_to(content=content, config=update_conf, group="update")
+ push.push_to(content=content, item=config.update.item, group="update")
def standard_sub(url: str) -> bool:
diff --git a/tools/clean.py b/tools/clean.py
index dc7301cdb0..21f5c481ca 100644
--- a/tools/clean.py
+++ b/tools/clean.py
@@ -223,7 +223,7 @@ def main(args: argparse.Namespace) -> None:
elif country:
name = country
else:
- print("cannot get geolocation and rename because IP address is faked")
+ print("cannot get geolocation and name because IP address is faked")
item["name"] = name
except Exception:
diff --git a/tools/proxy-gen.py b/tools/proxy-gen.py
new file mode 100644
index 0000000000..26a275d610
--- /dev/null
+++ b/tools/proxy-gen.py
@@ -0,0 +1,335 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""Generate dynamic HTTP/SOCKS proxy nodes from CLI options"""
+
+from __future__ import annotations
+
+import argparse
+import ipaddress
+import json
+import os
+import re
+import secrets
+import string
+import sys
+from dataclasses import dataclass
+
+SCHEMES = ("socks5", "socks5h", "http", "https")
+FORMATS = ("uri", "clash")
+NAMING_MODES = ("sid", "index", "none")
+SID_CHARS = string.ascii_letters + string.digits
+KEYWORD_PATTERN = re.compile(r"^[A-Za-z0-9_-]+$")
+DOMAIN_LABEL = re.compile(r"^[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?$")
+POSITIVE_INT = re.compile(r"^[1-9]\d*$")
+ILLEGAL_CHARS = set(":@#/?%")
+ISO_CODES = frozenset("""
+ AD AE AF AG AI AL AM AO AQ AR AS AT AU AW AX AZ
+ BA BB BD BE BF BG BH BI BJ BL BM BN BO BQ BR BS BT BV BW BY BZ
+ CA CC CD CF CG CH CI CK CL CM CN CO CR CU CV CW CX CY CZ
+ DE DJ DK DM DO DZ
+ EC EE EG EH ER ES ET
+ FI FJ FK FM FO FR
+ GA GB GD GE GF GG GH GI GL GM GN GP GQ GR GS GT GU GW GY
+ HK HM HN HR HT HU
+ ID IE IL IM IN IO IQ IR IS IT
+ JE JM JO JP
+ KE KG KH KI KM KN KP KR KW KY KZ
+ LA LB LC LI LK LR LS LT LU LV LY
+ MA MC MD ME MF MG MH MK ML MM MN MO MP MQ MR MS MT MU MV MW MX MY MZ
+ NA NC NE NF NG NI NL NO NP NR NU NZ
+ OM
+ PA PE PF PG PH PK PL PM PN PR PS PT PW PY
+ QA
+ RE RO RS RU RW
+ SA SB SC SD SE SG SH SI SJ SK SL SM SN SO SR SS ST SV SX SY SZ
+ TC TD TF TG TH TJ TK TL TM TN TO TR TT TV TW TZ
+ UA UG UM US UY UZ
+ VA VC VE VG VI VN VU
+ WF WS
+ XK
+ YE YT
+ ZA ZM ZW
+ """.split())
+
+
+class ValidationError(ValueError):
+ pass
+
+
+@dataclass
+class Config:
+ scheme: str
+ username: str
+ password: str
+ host: str
+ port: int
+ count: int
+ region_key: str
+ country: str
+ state_key: str
+ state: str
+ session_key: str
+ sid_length: int
+ duration_key: str
+ duration: int
+ naming_mode: str
+ output_format: str
+ output_path: str
+
+
+def trim(value: object | None) -> str:
+ return "" if value is None else str(value).strip()
+
+
+def require_text(value: object | None, name: str) -> str:
+ text = trim(value)
+ if not text:
+ raise ValidationError(f"{name} is required")
+ if any(ch.isspace() for ch in text):
+ raise ValidationError(f"{name} must not contain whitespace")
+ if any(ch in ILLEGAL_CHARS for ch in text):
+ raise ValidationError(f"{name} contains illegal character")
+ return text
+
+
+def require_keyword(value: object | None, name: str, required: bool = True) -> str:
+ text = trim(value)
+ if not text:
+ if required:
+ raise ValidationError(f"{name} is required")
+ return ""
+ if any(ch.isspace() for ch in text):
+ raise ValidationError(f"{name} must not contain whitespace")
+ if not KEYWORD_PATTERN.fullmatch(text):
+ raise ValidationError(f"{name} may contain only letters, digits, '-' and '_'")
+ return text
+
+
+def require_integer(value: object | None, name: str, maximum: int | None = None) -> int:
+ text = trim(value)
+ if not POSITIVE_INT.fullmatch(text):
+ raise ValidationError(f"{name} must be an integer > 0")
+ number = int(text)
+ if maximum is not None and number > maximum:
+ raise ValidationError(f"{name} must be <= {maximum}")
+ return number
+
+
+def require_host(value: object | None) -> str:
+ host = require_text(value, "host")
+ try:
+ ipaddress.IPv4Address(host)
+ return host
+ except ValueError:
+ pass
+ hostname = host[:-1] if host.endswith(".") else host
+ if not hostname or len(hostname) > 253:
+ raise ValidationError("host must be a domain or IPv4 address")
+ labels = hostname.split(".")
+ if not all(DOMAIN_LABEL.fullmatch(label) for label in labels):
+ raise ValidationError("host must be a domain or IPv4 address")
+ return host
+
+
+def require_country(value: object | None) -> str:
+ text = trim(value)
+ if not text:
+ return ""
+ country = text.upper()
+ if country not in ISO_CODES:
+ raise ValidationError("country must be an ISO 3166-1 alpha-2 code")
+ return country
+
+
+def require_choice(value: object | None, name: str, options: tuple[str, ...], default: str) -> str:
+ text = trim(value).lower()
+ if not text:
+ return default
+ if text not in options:
+ raise ValidationError(f"{name} must be one of {', '.join(options)}")
+ return text
+
+
+def generate_sids(count: int, length: int) -> list[str]:
+ capacity = len(SID_CHARS) ** length
+ if count > capacity:
+ raise ValidationError(f"count {count} exceeds unique sid space {capacity} for sid-length {length}")
+ randomizer = secrets.SystemRandom()
+ if length == 1:
+ return randomizer.sample(list(SID_CHARS), count)
+ seen: set[str] = set()
+ sids: list[str] = []
+ while len(sids) < count:
+ sid = "".join(randomizer.choice(SID_CHARS) for _ in range(length))
+ if sid in seen:
+ continue
+ seen.add(sid)
+ sids.append(sid)
+ return sids
+
+
+def build_username(config: Config, sid: str) -> str:
+ parts = [config.username]
+ if config.country:
+ parts.extend((config.region_key, config.country))
+ if config.state:
+ parts.extend((config.state_key, config.state))
+ parts.extend((config.session_key, sid, config.duration_key, str(config.duration)))
+ return "-".join(parts)
+
+
+def build_node_name(config: Config, sid: str, index: int, index_width: int) -> str:
+ if config.naming_mode == "none":
+ return ""
+ suffix = sid if config.naming_mode == "sid" else f"{index:0{index_width}d}"
+ return f"{config.country}-{suffix}" if config.country else suffix
+
+
+def format_uri(config: Config, username: str, node_name: str) -> str:
+ uri = f"{config.scheme}://{username}:{config.password}@{config.host}:{config.port}"
+ return f"{uri}#{node_name}" if node_name else uri
+
+
+def format_clash(config: Config, username: str, node_name: str) -> str:
+ proxy_type = "http" if config.scheme in ("http", "https") else "socks5"
+ proxy = {
+ "name": node_name,
+ "type": proxy_type,
+ "server": config.host,
+ "port": config.port,
+ "username": username,
+ "password": config.password,
+ }
+ if config.scheme in ("socks5", "socks5h"):
+ proxy["udp"] = True
+ proxy["tls"] = True
+ proxy["skip-cert-verify"] = False
+ elif config.scheme == "https":
+ proxy["tls"] = True
+ proxy["skip-cert-verify"] = False
+ return json.dumps(proxy, ensure_ascii=False, separators=(",", ":"))
+
+
+def parse_config(args: argparse.Namespace) -> Config:
+ scheme = require_choice(args.scheme, "scheme", SCHEMES, "socks5")
+ username = require_text(args.username, "username")
+ password = require_text(args.password, "password")
+ host = require_host(args.host)
+ port = require_integer(args.port, "port", maximum=65535)
+ count = require_integer(args.count, "count")
+ region_key = require_keyword(args.region_key, "region-key")
+ country = require_country(args.country)
+ state_key = require_keyword(args.state_key, "state-key", required=False)
+ state = require_keyword(args.state, "state", required=False)
+ session_key = require_keyword(args.session_key, "session-key")
+ sid_length = require_integer(args.sid_length, "sid-length", maximum=64)
+ duration_key = require_keyword(args.duration_key, "duration-key")
+ duration = require_integer(args.duration, "duration")
+ naming_mode = require_choice(args.naming, "naming", NAMING_MODES, "sid")
+ output_format = require_choice(args.format, "format", FORMATS, "uri")
+ output_path = trim(args.output)
+
+ if bool(state_key) != bool(state):
+ raise ValidationError("state-key and state must be provided together")
+ if (state_key or state) and not country:
+ raise ValidationError("state requires country")
+ if naming_mode == "none" and output_format != "uri":
+ raise ValidationError("naming=none is only supported when format=uri")
+
+ return Config(
+ scheme=scheme,
+ username=username,
+ password=password,
+ host=host,
+ port=port,
+ count=count,
+ region_key=region_key,
+ country=country,
+ state_key=state_key,
+ state=state,
+ session_key=session_key,
+ sid_length=sid_length,
+ duration_key=duration_key,
+ duration=duration,
+ naming_mode=naming_mode,
+ output_format=output_format,
+ output_path=output_path,
+ )
+
+
+def generate_nodes(config: Config) -> list[str]:
+ sids = generate_sids(config.count, config.sid_length)
+ index_width = len(str(config.count))
+ nodes: list[str] = []
+ for index, sid in enumerate(sids, start=1):
+ username = build_username(config, sid)
+ node_name = build_node_name(config, sid, index, index_width)
+ if config.output_format == "clash":
+ nodes.append(f" - {format_clash(config, username, node_name)}")
+ else:
+ nodes.append(format_uri(config, username, node_name))
+ if config.output_format == "clash":
+ return ["proxies:", *nodes]
+ return nodes
+
+
+def write_output(nodes: list[str], output_path: str) -> None:
+ content = "\n".join(nodes)
+ if content:
+ content += "\n"
+ if not output_path:
+ sys.stdout.write(content)
+ return
+ path = os.path.abspath(output_path)
+ directory = os.path.dirname(path)
+ if directory and not os.path.isdir(directory):
+ raise ValidationError(f"output directory not found: {directory}")
+ with open(path, "w", encoding="utf-8", newline="\n") as output_file:
+ output_file.write(content)
+
+
+def build_parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(
+ description="Generate dynamic HTTP/SOCKS proxy nodes",
+ formatter_class=argparse.RawTextHelpFormatter,
+ epilog=(
+ "example:\n"
+ " python proxies-gen.py --username alice --password secret --host gate.example.com --port 1080 --count 10 "
+ "--session-key sid --sid-length 8 --duration-key t --country US --state-key st --state california"
+ ),
+ allow_abbrev=False,
+ )
+ parser.add_argument("--scheme", default="socks5", help="socks5, socks5h, http or https")
+ parser.add_argument("--username", required=True, help="base username")
+ parser.add_argument("--password", required=True, help="password")
+ parser.add_argument("--host", required=True, help="domain or IPv4 address")
+ parser.add_argument("--port", required=True, help="port, 1-65535")
+ parser.add_argument("--count", required=True, help="number of nodes")
+ parser.add_argument("--region-key", default="region", help="region keyword, default: region")
+ parser.add_argument("--country", default="", help="ISO 3166-1 alpha-2 country code")
+ parser.add_argument("--state-key", default="", help="state or city keyword, e.g. st")
+ parser.add_argument("--state", default="", help="state or city name")
+ parser.add_argument("--session-key", required=True, help="session keyword, e.g. sid")
+ parser.add_argument("--sid-length", required=True, help="random sid length")
+ parser.add_argument("--duration-key", required=True, help="session duration keyword, e.g. t")
+ parser.add_argument("--duration", default="5", help="session duration, default: 5")
+ parser.add_argument("--naming", default="sid", help="node name style: sid, index or none, none only for uri")
+ parser.add_argument("--format", default="uri", help="uri or clash")
+ parser.add_argument("--output", default="", help="output file, default: stdout")
+ return parser
+
+
+def main() -> None:
+ parser = build_parser()
+ try:
+ config = parse_config(parser.parse_args())
+ write_output(generate_nodes(config), config.output_path)
+ except ValidationError as error:
+ parser.error(str(error))
+ except OSError as error:
+ parser.error(str(error))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tools/socks-checker.py b/tools/socks-checker.py
index c7bbb26388..7f8e4a8b37 100644
--- a/tools/socks-checker.py
+++ b/tools/socks-checker.py
@@ -11,13 +11,15 @@
import argparse
import asyncio
+import html
import ipaddress
+import json
import re
import sys
from dataclasses import dataclass
from datetime import datetime
-from typing import Dict, List, Optional, Tuple
-from urllib.parse import urlparse
+from typing import Any, Callable, Dict, List, Optional, Tuple
+from urllib.parse import quote, urlparse
import aiohttp
import yaml
@@ -276,6 +278,97 @@
}
+def country_flag_emoji(country_code: str) -> str:
+ if not country_code or len(country_code) != 2:
+ return ""
+
+ code = country_code.upper()
+ if not code.isalpha():
+ return ""
+
+ return chr(0x1F1E6 + ord(code[0]) - ord("A")) + chr(0x1F1E6 + ord(code[1]) - ord("A"))
+
+
+def country_name_zh(country_code: str) -> str:
+ if not country_code:
+ return ""
+
+ return COUNTRY_NAME_ZH.get(country_code.upper(), "")
+
+
+CHINA_PROVINCE_SUFFIXES = (
+ "特别行政区",
+ "维吾尔自治区",
+ "壮族自治区",
+ "回族自治区",
+ "自治区",
+ "省",
+ "市",
+)
+CHINA_PROVINCE_ALIASES = {
+ "anhui": "安徽",
+ "beijing": "北京",
+ "chongqing": "重庆",
+ "fujian": "福建",
+ "gansu": "甘肃",
+ "guangdong": "广东",
+ "guangxi": "广西",
+ "guangxizhuang": "广西",
+ "guizhou": "贵州",
+ "hainan": "海南",
+ "hebei": "河北",
+ "heilongjiang": "黑龙江",
+ "henan": "河南",
+ "hubei": "湖北",
+ "hunan": "湖南",
+ "innermongolia": "内蒙古",
+ "jiangsu": "江苏",
+ "jiangxi": "江西",
+ "jilin": "吉林",
+ "liaoning": "辽宁",
+ "neimenggu": "内蒙古",
+ "neimongol": "内蒙古",
+ "ningxia": "宁夏",
+ "ningxiahuizu": "宁夏",
+ "qinghai": "青海",
+ "shaanxi": "陕西",
+ "shandong": "山东",
+ "shanghai": "上海",
+ "shanxi": "山西",
+ "sichuan": "四川",
+ "tianjin": "天津",
+ "tibet": "西藏",
+ "xinjiang": "新疆",
+ "xinjianguygur": "新疆",
+ "xinjianguyghur": "新疆",
+ "xizang": "西藏",
+ "yunnan": "云南",
+ "zhejiang": "浙江",
+}
+CHINA_MAINLAND_PROVINCES = frozenset(CHINA_PROVINCE_ALIASES.values())
+CHINA_PROVINCE_ALIAS_PREFIXES = tuple(
+ sorted(CHINA_PROVINCE_ALIASES.items(), key=lambda item: len(item[0]), reverse=True)
+)
+CHINA_PROVINCE_EN_SUFFIXES = (
+ "autonomousregion",
+ "municipality",
+ "province",
+ "region",
+ "sheng",
+ "city",
+)
+
+REGION_KEYS = set(["province", "province_name", "region", "region_name", "state", "state_name"])
+
+
+def short_company_name(value: str) -> str:
+ if not value:
+ return "UNKNOWN"
+
+ parts = [part for part in re.split(r"[\s,\.\-_@;:]+", value.strip()) if part]
+ return parts[0].upper() if parts else "UNKNOWN"
+
+
@dataclass
class ProxyInfo:
protocol: str
@@ -309,6 +402,820 @@ def from_proxy(cls, proxy_info: ProxyInfo) -> "TestResult":
)
+@dataclass
+class IpLookupResult:
+ ip: Optional[str]
+ data: Optional[Dict]
+ error: Optional[str] = None
+
+
+class IPLibrary:
+ name: str = ""
+
+ def __init__(self):
+ self._caches: Dict[str, str] = {}
+
+ async def lookup(
+ self, session: aiohttp.ClientSession, proxy_info: ProxyInfo, retries: int, timeout: int
+ ) -> IpLookupResult:
+ data, error = await self._fetch(session, proxy_info, retries, timeout)
+ if not data:
+ host = "" if not proxy_info else proxy_info.host
+ return IpLookupResult(None, None, error or f"Failed to get IP info from {self.name}, host: {host}")
+
+ return self._verify(data, self.name)
+
+ async def build_remark(
+ self,
+ session: aiohttp.ClientSession,
+ ip: str,
+ data: Dict,
+ include_asn_name: bool,
+ retries: int,
+ timeout: int,
+ ) -> str:
+ raise NotImplementedError
+
+ async def _fetch(
+ self, session: aiohttp.ClientSession, proxy_info: ProxyInfo, retries: int, timeout: int
+ ) -> Tuple[Optional[Dict], Optional[str]]:
+ raise NotImplementedError
+
+ @staticmethod
+ def _build_headers(url: str) -> Dict[str, str]:
+ result = urlparse(url)
+ base = f"{result.scheme}://{result.netloc}" if result.scheme and result.netloc else ""
+
+ return {
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36",
+ "Accept": "application/json, text/plain, */*",
+ "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+ "Cache-Control": "no-cache",
+ "Pragma": "no-cache",
+ "Connection": "close",
+ "Referer": f"{base}/" if base else url,
+ "Origin": base if base else url,
+ }
+
+ async def _make_request(
+ self,
+ session: aiohttp.ClientSession,
+ url: str,
+ retries: int,
+ timeout: int,
+ headers: Optional[Dict[str, str]] = None,
+ deserialize: bool = True,
+ parser: Optional[Callable[[str], Any]] = None,
+ ) -> Tuple[Optional[Any], Optional[str]]:
+ default_headers = self._build_headers(url)
+ if headers and isinstance(headers, dict):
+ default_headers.update({k: v for k, v in headers.items() if k and v is not None})
+
+ error = None
+ for attempt in range(1, retries + 1):
+ try:
+ async with session.get(
+ url,
+ headers=default_headers,
+ timeout=aiohttp.ClientTimeout(total=timeout),
+ ) as response:
+ if response.status == 200:
+ content = await response.text()
+
+ if parser is not None:
+ data = parser(content)
+ if data:
+ return data, None
+
+ error = "Invalid response payload"
+ elif deserialize:
+ try:
+ data = json.loads(content)
+ except Exception:
+ data = None
+
+ if isinstance(data, dict):
+ return data, None
+
+ error = "Invalid JSON response"
+ else:
+ return content, None
+
+ else:
+ error = f"HTTP {response.status}"
+ except asyncio.TimeoutError:
+ error = "Timeout"
+ except Exception as e:
+ error = str(e)[:100]
+
+ if attempt < retries:
+ await asyncio.sleep(attempt)
+
+ return None, error
+
+ @staticmethod
+ def _verify(data: Dict, source: str) -> IpLookupResult:
+ address = (data.get("ip") or "").strip()
+ if not address:
+ return IpLookupResult(None, None, f"Invalid IP from {source}")
+
+ try:
+ ipaddress.ip_address(address)
+ except ValueError:
+ return IpLookupResult(None, None, f"Invalid IP from {source}, ip: {address}")
+
+ return IpLookupResult(address, data, None)
+
+ @staticmethod
+ def _format_remark(
+ country_code: str,
+ country: str,
+ label: str,
+ include_asn_name: bool,
+ company_name: str,
+ detail: str = "",
+ ) -> str:
+ flag = country_flag_emoji(country_code)
+ base = f"{flag} {country}{label}".strip()
+
+ if include_asn_name and company_name:
+ if detail:
+ return f"{base} [{company_name}::{detail}]".strip()
+
+ return f"{base} [{company_name}]".strip()
+
+ return base
+
+ async def _resolve_country(
+ self,
+ session: aiohttp.ClientSession,
+ ip: str,
+ country_code: str,
+ country: str,
+ retries: int,
+ timeout: int,
+ data: Optional[Dict] = None,
+ ) -> str:
+ country_code = (country_code or "").upper()
+ if country_code != "CN" or not ip:
+ return country
+
+ if ip in self._caches:
+ return self._caches[ip]
+
+ province = self._extract_province(data)
+ if not province:
+ return "中国"
+
+ resolved = f"中国{province}"
+ self._caches[ip] = resolved
+ return resolved
+
+ def _extract_province(self, data: Optional[Dict]) -> str:
+ if not isinstance(data, dict):
+ return ""
+
+ candidates: List[Any] = []
+ for key in REGION_KEYS:
+ candidates.append(data.get(key))
+
+ for parent_key in ("location", "geo"):
+ nested = data.get(parent_key)
+ if isinstance(nested, dict):
+ for key in REGION_KEYS:
+ candidates.append(nested.get(key))
+
+ for value in candidates:
+ province = self._normalize_province(value)
+ if province:
+ return province
+
+ return ""
+
+ @staticmethod
+ def _province_alias_key(province: str) -> str:
+ return re.sub(r"[^a-z0-9]+", "", province.lower())
+
+ @staticmethod
+ def _match_province_alias(key: str) -> str:
+ if not key or key in {"china", "cn", "mainlandchina", "unknown", "na", "null", "none"}:
+ return ""
+
+ candidates = [key]
+ for suffix in CHINA_PROVINCE_EN_SUFFIXES:
+ if key.endswith(suffix):
+ stripped = key[: -len(suffix)]
+ if stripped:
+ candidates.append(stripped)
+
+ for candidate in candidates:
+ province = CHINA_PROVINCE_ALIASES.get(candidate)
+ if province:
+ return province
+
+ for candidate in candidates:
+ for alias, province in CHINA_PROVINCE_ALIAS_PREFIXES:
+ if candidate.startswith(alias):
+ return province
+
+ return ""
+
+ @classmethod
+ def _normalize_province(cls, province: Any) -> str:
+ if isinstance(province, dict):
+ for key in ("name", "name_en", "en", "value"):
+ value = cls._normalize_province(province.get(key))
+ if value:
+ return value
+
+ return ""
+
+ if not isinstance(province, str):
+ return ""
+
+ province = province.strip()
+ if not province:
+ return ""
+
+ if province.lower() in {"-", "n/a", "na", "none", "null", "unknown"}:
+ return ""
+
+ for suffix in CHINA_PROVINCE_SUFFIXES:
+ if province.endswith(suffix):
+ province = province[: -len(suffix)].strip()
+ break
+
+ if re.search(r"[\u4e00-\u9fff]", province):
+ return province if province in CHINA_MAINLAND_PROVINCES else ""
+
+ key = cls._province_alias_key(province)
+ return cls._match_province_alias(key)
+
+
+class IPInfoLibrary(IPLibrary):
+ name = "ipinfo"
+
+ async def build_remark(
+ self,
+ session: aiohttp.ClientSession,
+ ip: str,
+ data: Dict,
+ include_asn_name: bool,
+ retries: int,
+ timeout: int,
+ ) -> str:
+ country_code = (data.get("country") or "").upper()
+ flag = country_flag_emoji(country_code)
+
+ asn_info = data.get("asn", {}) or {}
+ company_info = data.get("company", {}) or {}
+ asn_type = (asn_info.get("type") or "").lower()
+ company_type = (company_info.get("type") or "").lower()
+
+ asn_name = (asn_info.get("domain") or "").strip()
+ if not asn_name or re.match(r"^as\d+\.", asn_name, flags=re.I):
+ asn_name = (asn_info.get("name") or "").strip()
+
+ company_name = short_company_name(asn_name)
+
+ if asn_type == "isp" and company_type == "isp":
+ label = "家宽"
+ elif asn_type == "isp" or company_type == "isp":
+ label = "商宽"
+ elif asn_type == "edu" or company_type == "edu":
+ label = "教育"
+ else:
+ label = ""
+
+ country = await self._resolve_country(
+ session=session,
+ ip=ip,
+ country_code=country_code,
+ country=country_name_zh(country_code) or country_code or "未知",
+ retries=retries,
+ timeout=timeout,
+ data=data,
+ )
+ base = f"{flag} {country}{label}".strip()
+ if include_asn_name and company_name:
+ return f"{base} [{company_name}]".strip()
+
+ return base
+
+ @staticmethod
+ def _is_ipv4(host: str) -> bool:
+ if not host:
+ return False
+ try:
+ return isinstance(ipaddress.ip_address(host), ipaddress.IPv4Address)
+ except ValueError:
+ return False
+
+ async def _resolve_ip(self, session: aiohttp.ClientSession, host: str, retries: int, timeout: int) -> Optional[str]:
+ if self._is_ipv4(host):
+ return host
+
+ url = "https://ipinfo.io/ip"
+ for attempt in range(1, retries + 1):
+ try:
+ async with session.get(
+ url,
+ headers=self._build_headers(url),
+ timeout=aiohttp.ClientTimeout(total=timeout),
+ ) as response:
+ if response.status == 200:
+ text = (await response.text()).strip()
+ try:
+ ipaddress.ip_address(text)
+ return text
+ except ValueError:
+ pass
+ except asyncio.TimeoutError:
+ pass
+ except Exception:
+ pass
+
+ if attempt < retries:
+ await asyncio.sleep(attempt)
+
+ return None
+
+ async def _fetch(
+ self, session: aiohttp.ClientSession, proxy_info: ProxyInfo, retries: int, timeout: int
+ ) -> Tuple[Optional[Dict], Optional[str]]:
+ host = proxy_info.host if proxy_info else ""
+ address = await self._resolve_ip(session, host, retries, timeout)
+ if not address:
+ return None, f"Failed to get IP from ipinfo.io/ip, host: {host}"
+
+ url = f"https://ipinfo.io/widget/demo/{address}"
+ data, error = await self._make_request(session, url, retries, timeout)
+ if not data:
+ return None, error or f"Failed to get IP info from ipinfo.io, ip: {address}"
+
+ return data.get("data", data), None
+
+
+class IPPureLibrary(IPLibrary):
+ name = "ippure"
+
+ async def build_remark(
+ self,
+ session: aiohttp.ClientSession,
+ ip: str,
+ data: Dict,
+ include_asn_name: bool,
+ retries: int,
+ timeout: int,
+ ) -> str:
+ residential = data.get("isResidential")
+ label = "家宽" if residential is True else ""
+
+ country_code = (data.get("countryCode") or "").upper()
+ country = await self._resolve_country(
+ session=session,
+ ip=ip,
+ country_code=country_code,
+ country=country_name_zh(country_code) or (data.get("country") or "未知"),
+ retries=retries,
+ timeout=timeout,
+ data=data,
+ )
+
+ company_name = short_company_name(data.get("asOrganization") or "")
+ score = str(data.get("fraudScore")).zfill(3) if "fraudScore" in data else "NUL"
+
+ # broadcast or native
+ categroy = "NUL" if "isBroadcast" not in data else "B" if data.get("isBroadcast") else "N"
+
+ return self._format_remark(
+ country_code=country_code,
+ country=country,
+ label=label,
+ include_asn_name=include_asn_name,
+ company_name=company_name,
+ detail=f"{score}::{categroy}",
+ )
+
+ async def _fetch(
+ self, session: aiohttp.ClientSession, _: ProxyInfo, retries: int, timeout: int
+ ) -> Tuple[Optional[Dict], Optional[str]]:
+ url = "https://my.ippure.com/v1/info"
+ return await self._make_request(session, url, retries, timeout)
+
+
+class IP2LocationLibrary(IPLibrary):
+ name = "ip2location"
+
+ async def build_remark(
+ self,
+ session: aiohttp.ClientSession,
+ ip: str,
+ data: Dict,
+ include_asn_name: bool,
+ retries: int,
+ timeout: int,
+ ) -> str:
+ as_info = data.get("as_info") or {}
+
+ usage_type = (data.get("usage_type") or "").strip().lower()
+ as_usage_type = ((as_info.get("as_usage_type") if isinstance(as_info, dict) else "") or "").strip().lower()
+
+ check = lambda usage: usage.startswith("isp") or usage == "mob"
+ label = "家宽" if check(usage_type) and check(as_usage_type) else ""
+
+ country_code = (data.get("country_code") or "").upper()
+ country = await self._resolve_country(
+ session=session,
+ ip=ip,
+ country_code=country_code,
+ country=country_name_zh(country_code)
+ or data.get("country_name")
+ or data.get("country", {}).get("name", "")
+ or "未知",
+ retries=retries,
+ timeout=timeout,
+ data=data,
+ )
+
+ provider = (data.get("as", "") or data.get("isp", "") or "").strip()
+ if not provider and as_info and isinstance(as_info, dict):
+ provider = (as_info.get("as_name", "") or as_info.get("as_domain", "")).strip()
+ if not provider:
+ provider = data.get("domain", "").strip() or ""
+
+ company_name = short_company_name(provider)
+ score = str(data.get("fraud_score")).zfill(3) if "fraud_score" in data else "NUL"
+
+ return self._format_remark(
+ country_code=country_code,
+ country=country,
+ label=label,
+ include_asn_name=include_asn_name,
+ company_name=company_name,
+ detail=score,
+ )
+
+ @staticmethod
+ def _extract_data(content: str) -> Dict:
+ if not content or not isinstance(content, str):
+ return {}
+
+ pattern = r'