118 lines
4.3 KiB
Python
118 lines
4.3 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import logging
|
|
import sys
|
|
|
|
from . import dnsmasq_gen, fetch, ip_render, normalize
|
|
from .config import Config, ListEntry, load_config
|
|
|
|
log = logging.getLogger("xray-lists")
|
|
|
|
|
|
def gather_ip_lines(cfg: Config, session, entry: ListEntry) -> set[str]:
|
|
text_chunks = []
|
|
for url in entry.urls:
|
|
text = fetch.fetch_url(session, cfg.global_cfg.cache_dir, entry.id, url,
|
|
cfg.global_cfg.http_timeout)
|
|
if text is None:
|
|
log.warning("%s: no data available for %s (network down, no cache) - skipping this url",
|
|
entry.id, url)
|
|
continue
|
|
text_chunks.append(text)
|
|
|
|
all_text = "\n".join(text_chunks + entry.static)
|
|
return normalize.normalize_ip_lines(all_text, entry.id)
|
|
|
|
|
|
def gather_domain_lines(cfg: Config, session, entry: ListEntry) -> set[str]:
|
|
text_chunks = list(entry.static)
|
|
for url in entry.urls:
|
|
text = fetch.fetch_url(session, cfg.global_cfg.cache_dir, entry.id, url,
|
|
cfg.global_cfg.http_timeout)
|
|
if text is None:
|
|
log.warning("%s: no data available for %s (network down, no cache) - skipping this url",
|
|
entry.id, url)
|
|
continue
|
|
text_chunks.append(text)
|
|
|
|
return normalize.normalize_domain_lines("\n".join(text_chunks), entry.id)
|
|
|
|
|
|
def process_ip_entry(cfg: Config, session, entry: ListEntry, dry_run: bool) -> bool:
|
|
cidrs = gather_ip_lines(cfg, session, entry)
|
|
elements = normalize.sort_ips(cidrs)
|
|
content = ip_render.render_elements_file(elements)
|
|
changed = ip_render.write_if_changed(entry.output, content, dry_run=dry_run)
|
|
log.info("%s: %d entries -> %s%s", entry.id, len(elements), entry.output,
|
|
" (changed)" if changed else "")
|
|
return changed
|
|
|
|
|
|
def process_domain_entries(cfg: Config, session, entries: list[ListEntry],
|
|
dry_run: bool) -> bool:
|
|
if not entries:
|
|
return False
|
|
|
|
domain_to_targets: dict[str, list[str]] = {}
|
|
for entry in entries:
|
|
domains = gather_domain_lines(cfg, session, entry)
|
|
for d in domains:
|
|
domain_to_targets.setdefault(d, []).append(entry.dnsmasq_target)
|
|
log.info("%s: %d domains -> %s", entry.id, len(domains), entry.dnsmasq_target)
|
|
|
|
content = dnsmasq_gen.render_nftset_file(domain_to_targets)
|
|
changed = dnsmasq_gen.write_if_changed(cfg.global_cfg.dnsmasq_output, content, dry_run=dry_run)
|
|
log.info("dnsmasq nftset file: %d domains total -> %s%s",
|
|
len(domain_to_targets), cfg.global_cfg.dnsmasq_output,
|
|
" (changed)" if changed else "")
|
|
return changed
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
parser = argparse.ArgumentParser(
|
|
description="fetch ip/cidr and domain lists and generate files for nftables and dnsmasq"
|
|
)
|
|
|
|
parser.add_argument("--config", default="/var/lib/xray-lists/config.yaml", help="path to config file")
|
|
parser.add_argument("--only", help="update only specific list ids (comma-separated)")
|
|
parser.add_argument("--dry-run", action="store_true", help="download and process, but do not write files")
|
|
parser.add_argument("-v", "--verbose", action="store_true", help="enable verbose logging")
|
|
|
|
args = parser.parse_args(argv)
|
|
|
|
logging.basicConfig(
|
|
level=logging.DEBUG if args.verbose else logging.INFO,
|
|
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
|
|
)
|
|
|
|
cfg = load_config(args.config)
|
|
session = fetch.build_session(cfg.global_cfg.proxy, cfg.global_cfg.proxy_user,
|
|
cfg.global_cfg.proxy_pass)
|
|
|
|
only = set(args.only.split(",")) if args.only else None
|
|
|
|
ip_entries = [e for e in cfg.ip_sets if only is None or e.id in only]
|
|
domain_entries = [e for e in cfg.domain_sets if only is None or e.id in only]
|
|
|
|
changed_files = []
|
|
for entry in ip_entries:
|
|
if process_ip_entry(cfg, session, entry, args.dry_run):
|
|
changed_files.append(str(entry.output))
|
|
|
|
if process_domain_entries(cfg, session, domain_entries, args.dry_run):
|
|
changed_files.append(str(cfg.global_cfg.dnsmasq_output))
|
|
|
|
if changed_files:
|
|
print("changed:")
|
|
for f in changed_files:
|
|
print(f" {f}")
|
|
else:
|
|
log.info("nothing changed")
|
|
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|