4bd5c2abab
- ensure-cd-watches：默认并入 DEMAND_WATCHES（Plati/GGSel/FunPay 需求页） - notify-chain：忽略 noise 价差；推送 sold_out/restock 等货架高信号 - record-shop-outcome.sh + 总览三利害卡；post-pay 可接 MANHUANG_SHOP_ID
444 lines
17 KiB
Python
Executable file
444 lines
17 KiB
Python
Executable file
#!/usr/bin/env python3
|
||
"""确保蛮荒供应链相关 URL 已在 ChangeDetection 中监控。
|
||
|
||
读：
|
||
CHANGEDTECTION_URL / CHANGEDTECTION_API_KEY
|
||
或 ~/.config/manhuang/keys.env
|
||
suggested 列表默认内置；可用 --from-cache 读 status.json
|
||
shops.json（本地店 URL）+ 鬼市浅水 feed（--guishi / --shops）
|
||
|
||
用法：
|
||
python3 scripts/manhuang/ensure-cd-watches.py
|
||
python3 scripts/manhuang/ensure-cd-watches.py --shops --guishi
|
||
python3 scripts/manhuang/ensure-cd-watches.py --dry-run
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import os
|
||
import urllib.error
|
||
import urllib.parse
|
||
import urllib.request
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
UA = "joy-manhuang-cd-ensure/1.0"
|
||
DEFAULT_URL = "http://10.211.55.107:5000"
|
||
KEYS = Path.home() / ".config" / "manhuang" / "keys.env"
|
||
CD_CACHE = Path.home() / ".cache" / "manhuang" / "changedetection" / "status.json"
|
||
SHOPS_PATHS = (
|
||
Path.home() / ".config" / "manhuang" / "shops.json",
|
||
Path.home() / "data" / "manhuang" / "shops.json",
|
||
)
|
||
|
||
# 供应链关键页（价/货架/中转指数）
|
||
DEFAULT_WATCHES: list[dict[str, Any]] = [
|
||
{"url": "https://openai.com/chatgpt/pricing/", "title": "msrp · OpenAI ChatGPT pricing", "tag": "msrp", "sku": "openai"},
|
||
{"url": "https://claude.ai/pricing", "title": "msrp · Claude pricing", "tag": "msrp", "sku": "claude"},
|
||
{"url": "https://cursor.com/pricing", "title": "msrp · Cursor pricing", "tag": "msrp", "sku": "cursor"},
|
||
{"url": "https://openrouter.ai/models", "title": "relay · OpenRouter models", "tag": "relay", "sku": "openrouter"},
|
||
{"url": "https://ggsel.net/en/catalog/grok-accounts", "title": "market · GGSel SuperGrok", "tag": "market", "sku": "supergrok"},
|
||
{"url": "https://ggsel.net/en/catalog?query=chatgpt%20plus", "title": "market · GGSel ChatGPT Plus", "tag": "market", "sku": "openai"},
|
||
{"url": "https://ggsel.net/en/catalog?query=claude%20pro", "title": "market · GGSel Claude", "tag": "market", "sku": "claude"},
|
||
{"url": "https://ggsel.net/en/catalog?query=cursor%20pro", "title": "market · GGSel Cursor", "tag": "market", "sku": "cursor"},
|
||
{"url": "https://5sim.net/", "title": "sms · 5sim home", "tag": "sms", "sku": "sms_otp"},
|
||
{"url": "https://tokenx24.com", "title": "relay · tokenx24", "tag": "relay", "sku": "relay_premium"},
|
||
{"url": "https://howardpen9.github.io/awesome-ai-api-proxy/", "title": "relay_index · awesome-ai-api-proxy", "tag": "relay_index", "sku": "newapi_relay"},
|
||
{"url": "https://x.ai/grok", "title": "msrp · xAI Grok", "tag": "msrp", "sku": "supergrok"},
|
||
{"url": "https://funpay.com/en/lots/3173/", "title": "market · FunPay Claude Subscription", "tag": "market", "sku": "claude"},
|
||
{"url": "https://funpay.com/en/lots/3172/", "title": "market · FunPay Claude Accounts", "tag": "market", "sku": "claude"},
|
||
{"url": "https://funpay.com/en/lots/3559/", "title": "market · FunPay ChatGPT Subscription", "tag": "market", "sku": "openai"},
|
||
{"url": "https://funpay.com/en/lots/1355/", "title": "market · FunPay ChatGPT Accounts", "tag": "market", "sku": "openai"},
|
||
{"url": "https://funpay.com/en/lots/3174/", "title": "market · FunPay Grok Accounts", "tag": "market", "sku": "supergrok"},
|
||
{"url": "https://funpay.com/en/lots/3175/", "title": "market · FunPay Grok Subscription", "tag": "market", "sku": "supergrok"},
|
||
{"url": "https://funpay.com/en/lots/4400/", "title": "market · FunPay GLM Accounts", "tag": "market", "sku": "zhipu"},
|
||
{"url": "https://plati.market/?lang=en-US", "title": "market · Plati.Market home", "tag": "market", "sku": "supergrok"},
|
||
{"url": "https://playerok.com/", "title": "market · Playerok home", "tag": "market", "sku": "openai"},
|
||
# GitHub 机读源 / 目录（侧线 A–E）
|
||
{
|
||
"url": "https://raw.githubusercontent.com/howardpen9/awesome-ai-api-proxy/main/data/prices.latest.json",
|
||
"title": "relay_index · awesome prices.latest.json",
|
||
"tag": "github_json",
|
||
"sku": "newapi_relay",
|
||
"hours": 24,
|
||
},
|
||
{
|
||
"url": "https://raw.githubusercontent.com/howardpen9/awesome-ai-api-proxy/main/data/providers.yaml",
|
||
"title": "relay · awesome providers.yaml",
|
||
"tag": "github_yaml",
|
||
"sku": "relay_premium",
|
||
"hours": 24,
|
||
},
|
||
{
|
||
"url": "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json",
|
||
"title": "msrp · LiteLLM model_prices",
|
||
"tag": "github_json",
|
||
"sku": "openai",
|
||
"hours": 24,
|
||
},
|
||
{
|
||
"url": "https://raw.githubusercontent.com/mnfst/awesome-free-llm-apis/main/data.json",
|
||
"title": "free · mnfst free-llm data.json",
|
||
"tag": "github_json",
|
||
"sku": "openrouter",
|
||
"hours": 24,
|
||
},
|
||
{
|
||
"url": "https://raw.githubusercontent.com/cheahjs/free-llm-api-resources/main/README.md",
|
||
"title": "free · cheahjs free-llm-api-resources",
|
||
"tag": "github_readme",
|
||
"sku": "openrouter",
|
||
"hours": 24,
|
||
},
|
||
{
|
||
"url": "https://github.com/cheahjs/free-llm-api-resources",
|
||
"title": "free · cheahjs repo home",
|
||
"tag": "github",
|
||
"sku": "openrouter",
|
||
"hours": 48,
|
||
},
|
||
]
|
||
|
||
# 需求向货架（SKU 通用：钉 catalog/搜索页，不绑单一产品）
|
||
# 覆盖常见 demand / pool 缺口：账号订阅、Codex/额度、OpenCode、接码旁路
|
||
DEMAND_WATCHES: list[dict[str, Any]] = [
|
||
{
|
||
"url": "https://plati.market/search/chatgpt%20plus?lang=en-US",
|
||
"title": "demand · Plati ChatGPT Plus",
|
||
"tag": "demand",
|
||
"sku": "openai",
|
||
"hours": 4,
|
||
},
|
||
{
|
||
"url": "https://plati.market/search/claude%20pro?lang=en-US",
|
||
"title": "demand · Plati Claude Pro",
|
||
"tag": "demand",
|
||
"sku": "claude",
|
||
"hours": 4,
|
||
},
|
||
{
|
||
"url": "https://plati.market/search/cursor%20pro?lang=en-US",
|
||
"title": "demand · Plati Cursor",
|
||
"tag": "demand",
|
||
"sku": "cursor",
|
||
"hours": 4,
|
||
},
|
||
{
|
||
"url": "https://plati.market/search/opencode?lang=en-US",
|
||
"title": "demand · Plati OpenCode / Go",
|
||
"tag": "demand",
|
||
"sku": "opencode_go",
|
||
"hours": 4,
|
||
},
|
||
{
|
||
"url": "https://plati.market/search/codex?lang=en-US",
|
||
"title": "demand · Plati Codex",
|
||
"tag": "demand",
|
||
"sku": "codex_quota",
|
||
"hours": 4,
|
||
},
|
||
{
|
||
"url": "https://ggsel.net/en/catalog?query=opencode",
|
||
"title": "demand · GGSel OpenCode",
|
||
"tag": "demand",
|
||
"sku": "opencode_go",
|
||
"hours": 4,
|
||
},
|
||
{
|
||
"url": "https://ggsel.net/en/catalog?query=codex",
|
||
"title": "demand · GGSel Codex",
|
||
"tag": "demand",
|
||
"sku": "codex_quota",
|
||
"hours": 4,
|
||
},
|
||
{
|
||
"url": "https://ggsel.net/en/catalog?query=chatgpt%20plus",
|
||
"title": "demand · GGSel ChatGPT Plus (query)",
|
||
"tag": "demand",
|
||
"sku": "openai",
|
||
"hours": 6,
|
||
},
|
||
{
|
||
"url": "https://funpay.com/en/lots/3559/",
|
||
"title": "demand · FunPay ChatGPT sub lot",
|
||
"tag": "demand",
|
||
"sku": "openai",
|
||
"hours": 6,
|
||
},
|
||
{
|
||
"url": "https://5sim.net/",
|
||
"title": "demand · 5sim home (sms)",
|
||
"tag": "demand",
|
||
"sku": "sms",
|
||
"hours": 6,
|
||
},
|
||
]
|
||
|
||
# 鬼市浅水：清网入口/Feed（观察改版/宕机，非爬商品）
|
||
GUISHI_WATCHES: list[dict[str, Any]] = [
|
||
{"url": "https://ahmia.fi/", "title": "guishi · Ahmia onion search", "tag": "guishi", "sku": "onion_search", "hours": 12},
|
||
{"url": "https://dark.fail/", "title": "guishi · dark.fail mirrors", "tag": "guishi", "sku": "market_mirror", "hours": 12},
|
||
{"url": "https://onion.live/", "title": "guishi · onion.live catalog", "tag": "guishi", "sku": "onion_search", "hours": 12},
|
||
{"url": "https://www.ransomware.live/", "title": "guishi · ransomware.live", "tag": "guishi", "sku": "leak", "hours": 6},
|
||
{"url": "https://www.ransomlook.io/", "title": "guishi · ransomlook", "tag": "guishi", "sku": "leak", "hours": 6},
|
||
{"url": "https://urlhaus.abuse.ch/", "title": "guishi · URLhaus", "tag": "guishi", "sku": "brand_ioc", "hours": 12},
|
||
{"url": "https://threatfox.abuse.ch/", "title": "guishi · ThreatFox", "tag": "guishi", "sku": "brand_ioc", "hours": 12},
|
||
{"url": "https://www.blackhatworld.com/forums/account-selling-renting-services.310/", "title": "guishi · BHW account selling", "tag": "guishi", "sku": "account", "hours": 12},
|
||
{"url": "https://haveibeenpwned.com/", "title": "guishi · HIBP home", "tag": "guishi", "sku": "leak", "hours": 24},
|
||
]
|
||
|
||
|
||
def _load_keys_env() -> None:
|
||
if not KEYS.is_file():
|
||
return
|
||
try:
|
||
for line in KEYS.read_text(encoding="utf-8").splitlines():
|
||
line = line.strip()
|
||
if not line or line.startswith("#") or "=" not in line:
|
||
continue
|
||
k, _, v = line.partition("=")
|
||
k, v = k.strip(), v.strip().strip('"').strip("'")
|
||
if k and k not in os.environ:
|
||
os.environ[k] = v
|
||
except OSError:
|
||
pass
|
||
|
||
|
||
def _opener_for(url: str) -> urllib.request.OpenerDirector:
|
||
"""内网直连：Python 不认 NO_PROXY CIDR，10.x 会被本机代理劫持。"""
|
||
host = urllib.parse.urlparse(url).hostname or ""
|
||
if host.startswith("10.") or host.startswith("192.168.") or host in ("127.0.0.1", "localhost"):
|
||
return urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
||
return urllib.request.build_opener()
|
||
|
||
|
||
def _req(method: str, url: str, key: str, body: dict | None = None) -> tuple[int, Any]:
|
||
data = None
|
||
headers = {"User-Agent": UA, "Accept": "application/json", "x-api-key": key}
|
||
if body is not None:
|
||
data = json.dumps(body).encode("utf-8")
|
||
headers["Content-Type"] = "application/json"
|
||
req = urllib.request.Request(url, data=data, headers=headers, method=method)
|
||
opener = _opener_for(url)
|
||
try:
|
||
with opener.open(req, timeout=30) as r:
|
||
raw = r.read().decode("utf-8", "replace")
|
||
try:
|
||
return int(getattr(r, "status", 200) or 200), json.loads(raw) if raw else {}
|
||
except json.JSONDecodeError:
|
||
return int(getattr(r, "status", 200) or 200), raw
|
||
except urllib.error.HTTPError as e:
|
||
try:
|
||
raw = e.read().decode("utf-8", "replace")
|
||
except Exception:
|
||
raw = ""
|
||
try:
|
||
return int(e.code), json.loads(raw) if raw else {"error": raw}
|
||
except json.JSONDecodeError:
|
||
return int(e.code), {"error": raw[:200]}
|
||
except Exception as e:
|
||
return 0, {"error": str(e)[:200]}
|
||
|
||
|
||
def _existing_urls(base: str, key: str) -> dict[str, str]:
|
||
code, data = _req("GET", base.rstrip("/") + "/api/v1/watch", key)
|
||
out: dict[str, str] = {}
|
||
if code != 200 or not isinstance(data, dict):
|
||
return out
|
||
for uid, w in data.items():
|
||
if isinstance(w, dict) and w.get("url"):
|
||
out[str(w["url"]).rstrip("/")] = str(uid)
|
||
out[str(w["url"])] = str(uid)
|
||
return out
|
||
|
||
|
||
def _wanted(from_cache: bool) -> list[dict[str, Any]]:
|
||
if from_cache and CD_CACHE.is_file():
|
||
try:
|
||
d = json.loads(CD_CACHE.read_text(encoding="utf-8"))
|
||
sug = d.get("suggested_watches") or []
|
||
if isinstance(sug, list) and sug:
|
||
# cache 建议 + 需求钉（demand 始终并入）
|
||
return [x for x in sug if isinstance(x, dict) and x.get("url")] + list(
|
||
DEMAND_WATCHES
|
||
)
|
||
except (OSError, json.JSONDecodeError):
|
||
pass
|
||
# 默认 = 供应链基线 + 需求货架钉
|
||
return list(DEFAULT_WATCHES) + list(DEMAND_WATCHES)
|
||
|
||
|
||
def _load_shop_watches() -> list[dict[str, Any]]:
|
||
"""本地 shops.json 有 http(s) URL 的店 → CD 盯首页改价/下架。"""
|
||
out: list[dict[str, Any]] = []
|
||
for p in SHOPS_PATHS:
|
||
if not p.is_file():
|
||
continue
|
||
try:
|
||
data = json.loads(p.read_text(encoding="utf-8"))
|
||
except (OSError, json.JSONDecodeError):
|
||
continue
|
||
for s in data.get("shops") or []:
|
||
if not isinstance(s, dict):
|
||
continue
|
||
sid = str(s.get("id") or "")
|
||
url = str(s.get("url") or "").strip()
|
||
if not url.startswith("http"):
|
||
continue
|
||
if "example" in sid.lower() or "demo" in sid.lower() or ".invalid" in url:
|
||
continue
|
||
name = s.get("name") or sid or url
|
||
out.append({
|
||
"url": url,
|
||
"title": f"guishi · shop · {name}"[:120],
|
||
"tag": "shop",
|
||
"sku": ",".join((s.get("sku_for") or [])[:3]) or "shop",
|
||
"hours": 6,
|
||
"source": "shops.json",
|
||
})
|
||
break # first existing shops file wins
|
||
return out
|
||
|
||
|
||
def _merge_wanted(
|
||
base: list[dict[str, Any]],
|
||
*,
|
||
shops: bool,
|
||
guishi: bool,
|
||
demand: bool,
|
||
) -> list[dict[str, Any]]:
|
||
seen: set[str] = set()
|
||
out: list[dict[str, Any]] = []
|
||
extra: list[dict[str, Any]] = []
|
||
if demand:
|
||
extra.extend(DEMAND_WATCHES)
|
||
if guishi:
|
||
extra.extend(GUISHI_WATCHES)
|
||
if shops:
|
||
extra.extend(_load_shop_watches())
|
||
for w in list(base) + extra:
|
||
url = str(w.get("url") or "").strip()
|
||
if not url or url in seen:
|
||
continue
|
||
seen.add(url)
|
||
out.append(w)
|
||
return out
|
||
|
||
|
||
def main() -> int:
|
||
_load_keys_env()
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--dry-run", action="store_true")
|
||
ap.add_argument("--from-cache", action="store_true")
|
||
ap.add_argument("--hours", type=int, default=6, help="default check interval hours")
|
||
ap.add_argument("--shops", action="store_true", help="also watch shops.json URLs")
|
||
ap.add_argument("--guishi", action="store_true", help="also watch 鬼市浅水 feed/入口")
|
||
ap.add_argument("--demand", action="store_true", help="also pin demand SKU catalog/search pages")
|
||
ap.add_argument("--all-guishi", action="store_true", help="--shops + --guishi")
|
||
ap.add_argument(
|
||
"--all",
|
||
action="store_true",
|
||
help="default + demand + shops + guishi（采购雷达全钉）",
|
||
)
|
||
args = ap.parse_args()
|
||
if args.all_guishi:
|
||
args.shops = True
|
||
args.guishi = True
|
||
if args.all:
|
||
args.demand = True
|
||
args.shops = True
|
||
args.guishi = True
|
||
|
||
base = (os.environ.get("CHANGEDTECTION_URL") or DEFAULT_URL).rstrip("/")
|
||
key = (os.environ.get("CHANGEDTECTION_API_KEY") or os.environ.get("CD_API_KEY") or "").strip()
|
||
if not key:
|
||
print("missing CHANGEDTECTION_API_KEY (set in ~/.config/manhuang/keys.env)")
|
||
return 2
|
||
|
||
# health with retries（Flask 忙时会偶发 502）
|
||
import time
|
||
health = 0
|
||
for attempt in range(8):
|
||
try:
|
||
req = urllib.request.Request(base + "/", headers={"User-Agent": UA, "Accept": "text/html"})
|
||
with _opener_for(base + "/").open(req, timeout=12) as r:
|
||
health = int(getattr(r, "status", 200) or 200)
|
||
if 200 <= health < 500:
|
||
break
|
||
except urllib.error.HTTPError as e:
|
||
health = int(e.code)
|
||
except Exception as e:
|
||
print(f"CD health try {attempt+1}: {e}")
|
||
health = 0
|
||
time.sleep(2 + attempt)
|
||
if not (200 <= health < 500):
|
||
print("CD health fail", health, base)
|
||
return 1
|
||
print("CD health", health, base)
|
||
|
||
existing = _existing_urls(base, key)
|
||
print("existing watches with url map", len(existing) // 2)
|
||
|
||
wanted = _merge_wanted(
|
||
_wanted(args.from_cache),
|
||
shops=bool(args.shops),
|
||
guishi=bool(args.guishi),
|
||
demand=bool(args.demand),
|
||
)
|
||
print(
|
||
"wanted",
|
||
len(wanted),
|
||
"demand",
|
||
int(bool(args.demand)),
|
||
"shops",
|
||
sum(1 for w in wanted if w.get("tag") == "shop"),
|
||
"guishi",
|
||
sum(1 for w in wanted if w.get("tag") == "guishi"),
|
||
)
|
||
created, skipped, failed = 0, 0, 0
|
||
for w in wanted:
|
||
url = str(w.get("url") or "").strip()
|
||
if not url:
|
||
continue
|
||
if url in existing or url.rstrip("/") in existing:
|
||
print("skip", url[:70])
|
||
skipped += 1
|
||
continue
|
||
title = w.get("title") or f"manhuang · {w.get('sku') or url}"
|
||
hours = int(w.get("hours") or args.hours or 6)
|
||
payload = {
|
||
"url": url,
|
||
"title": title,
|
||
"paused": False,
|
||
"time_between_check": {"hours": hours},
|
||
# tag as free text in title prefix already; some versions accept tags list of names
|
||
"notification_muted": True,
|
||
}
|
||
if args.dry_run:
|
||
print("dry-create", title, url, f"every {hours}h")
|
||
created += 1
|
||
continue
|
||
code, resp = _req("POST", base + "/api/v1/watch", key, payload)
|
||
if code in (200, 201) and isinstance(resp, dict) and resp.get("uuid"):
|
||
print("created", resp.get("uuid"), title[:50])
|
||
created += 1
|
||
existing[url] = str(resp["uuid"])
|
||
else:
|
||
print("FAIL", code, title[:40], resp)
|
||
failed += 1
|
||
|
||
print(json.dumps({
|
||
"ok": failed == 0,
|
||
"created": created,
|
||
"skipped": skipped,
|
||
"failed": failed,
|
||
"wanted": len(wanted),
|
||
"shops_flag": bool(args.shops),
|
||
"guishi_flag": bool(args.guishi),
|
||
}, ensure_ascii=False))
|
||
return 0 if failed == 0 else 1
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|
