feat: 更新自动更新脚本，使用 requests 替代 curl 下载文件并添加日志记录

.gitlab-ci.yml

@@ -15,8 +15,8 @@ auto-update:
     - echo "index-url = https://mirrors.aliyun.com/pypi/simple/" >> ~/.pip/pip.conf
     - echo "trusted-host = mirrors.aliyun.com" >> ~/.pip/pip.conf
   script:
-    # 安装 PyYAML 用于解析 list.yaml
-    - pip install pyyaml
+    # 安装 PyYAML 和 Requests
+    - pip show pyyaml requests
     # 运行更新脚本
     - python update_files.py
 

update_files.py

@@ -1,82 +1,130 @@
 #!/usr/bin/env python
 import os
-import subprocess
 import sys
-
+import threading
+from concurrent.futures import ThreadPoolExecutor, as_completed
 import yaml
+import requests
+import subprocess
+import logging
+
+# 初始化日志记录器
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(message)s",
+    handlers=[logging.StreamHandler(sys.stdout)]
+)
+
+LOCK = threading.Lock()
+modified = False
 
 
 def download_file(url):
-    """使用 curl 下载文件内容"""
-    result = subprocess.run(["curl", "-s", url], capture_output=True, text=True)
-    if result.returncode != 0:
-        print(f"❌ 下载 {url} 时出错，跳过。")
+    """使用 requests 下载文件内容，限制超时时间为 3 分钟"""
+    try:
+        response = requests.get(url, timeout=180)
+        response.raise_for_status()
+        return response.text
+    except requests.exceptions.RequestException as e:
+        logging.error(f"❌ 下载 {url} 时出错: {e}")
         return None
-    return result.stdout
+
+
+def process_item(item):
+    global modified
+    file_path = item.get("file")
+    source = item.get("source")
+    if not file_path or not source:
+        logging.warning(f"⚠️ 条目缺少 file 或 source 字段: {item}")
+        return
+
+    logging.info(f"🛠️ 处理 {file_path} 从 {source}")
+    new_content = download_file(source)
+    if new_content is None:
+        return
+
+    # 确保文件路径是相对路径且安全
+    rel_path = os.path.relpath(file_path)
+    os.makedirs(os.path.dirname(rel_path), exist_ok=True)
+
+    updated = False
+    if os.path.exists(rel_path):
+        with open(rel_path, "r", encoding="utf-8") as f:
+            old_content = f.read()
+        if old_content != new_content:
+            updated = True
+    else:
+        updated = True
+
+    if updated:
+        with open(rel_path, "w", encoding="utf-8") as f:
+            f.write(new_content)
+        logging.info(f"🚀 更新了 {rel_path}")
+        with LOCK:
+            modified = True
+
+
+def check_env_vars():
+    """检查必要的环境变量是否存在"""
+    required_vars = ["THE_REPO", "REPO_TOKEN"]
+    missing_vars = [var for var in required_vars if not os.environ.get(var)]
+    if missing_vars:
+        logging.error(f"❌ 缺少必要的环境变量: {', '.join(missing_vars)}")
+        sys.exit(1)
 
 
 def main():
-    # 读取 list.yaml
+    global modified
+
+    # 检查环境变量
+    check_env_vars()
+
+    # 加载 list.yaml 文件
     try:
         with open("list.yaml", "r", encoding="utf-8") as f:
-            items = yaml.safe_load(f)
+            items = yaml.safe_load(f) or []
     except FileNotFoundError:
-        print("❌ list.yaml 文件不存在。")
-        sys.exit(0)
+        logging.error("❌ list.yaml 文件不存在。")
+        sys.exit(1)
+    except yaml.YAMLError as e:
+        logging.error(f"❌ 解析 list.yaml 文件时出错: {e}")
+        sys.exit(1)
 
-    print("🚀 开始更新文件")
-    modified = False
+    logging.info("🚀 开始并发更新文件")
 
-    for item in items:
-        file_path = item.get("file")
-        source = item.get("source")
-        if not file_path or not source:
-            print("条目缺少 file 或 source 字段，跳过。")
-            continue
+    # 使用线程池执行多线程任务
+    max_workers = int(os.getenv("MAX_WORKERS", 5))  # 允许通过环境变量配置并发数
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        futures = [executor.submit(process_item, item) for item in items]
+        for future in as_completed(futures):
+            try:
+                future.result()  # 确保异常被捕获
+            except Exception as e:
+                logging.error(f"❌ 处理任务时出错: {e}")
 
-        print(f"处理 {file_path} 从 {source}")
-        new_content = download_file(source)
-        if new_content is None:
-            # 如果下载失败，继续处理下一个文件
-            continue
-
-        # 去掉前导斜杠（如果希望相对于仓库根目录存放，也可以自行调整目录结构）
-        rel_path = file_path.lstrip("/")
-        # 创建必要目录
-        os.makedirs(os.path.dirname(rel_path), exist_ok=True)
-
-        # 如果文件存在，检查内容是否已更新
-        if os.path.exists(rel_path):
-            with open(rel_path, "r", encoding="utf-8") as f:
-                old_content = f.read()
-            if old_content == new_content:
-                print(f"{rel_path} 内容未改变。")
-                continue
-
-        with open(rel_path, "w", encoding="utf-8") as f:
-            f.write(new_content)
-        print(f"🚀 更新了 {rel_path}")
-        modified = True
-
-    # 如果有修改则提交到仓库
     if modified:
-        # 配置 git 用户信息
-        subprocess.run(["git", "config", "--global", "user.email", "cicd@sugarscat.cn"], check=True)
-        subprocess.run(["git", "config", "--global", "user.name", "CI/CD"], check=True)
+        try:
+            # 配置 Git 用户信息
+            subprocess.run(["git", "config", "--global", "user.email", "cicd@sugarscat.cn"], check=True)
+            subprocess.run(["git", "config", "--global", "user.name", "CI/CD"], check=True)
 
-        repo_url = os.environ.get("THE_REPO")
-        # 将 https:// 替换为 https://oauth2:token@ 形式
-        token = os.environ.get("REPO_TOKEN")
-        new_repo_url = repo_url.replace("https://", f"https://oauth2:{token}@")
-        subprocess.run(["git", "remote", "set-url", "origin", new_repo_url], check=True)
-        subprocess.run(["git", "checkout", "-B", "main"], check=True)
-        subprocess.run(["git", "add", "."], check=True)
-        commit_message = "feat: 自动更新从 list.yaml 下载最新文件"
-        # 如果没有变更，则 git commit 会返回非零状态，可用 || true 处理
-        subprocess.run(["git", "commit", "-m", commit_message], check=False)
-        subprocess.run(["git", "push", "origin", "main"], check=True)
+            # 设置远程仓库 URL（隐藏敏感信息）
+            repo_url = os.environ["THE_REPO"]
+            token = os.environ["REPO_TOKEN"]
+            new_repo_url = repo_url.replace("https://", f"https://oauth2:{token}@")
+            subprocess.run(["git", "remote", "set-url", "origin", new_repo_url], check=True)
+
+            # 提交并推送更改
+            subprocess.run(["git", "checkout", "-B", "main"], check=True)
+            subprocess.run(["git", "add", "."], check=True)
+            commit_message = "feat: 自动更新从 list.yaml 下载最新文件"
+            subprocess.run(["git", "commit", "-m", commit_message], check=False)
+            subprocess.run(["git", "push", "origin", "main"], check=True)
+        except subprocess.CalledProcessError as e:
+            logging.error(f"❌ Git 操作失败: {e}")
+            sys.exit(1)
     else:
-        print("📦 没有需要更新的文件。")
+        logging.info("📦 没有需要更新的文件。")
 
 
 if __name__ == "__main__":