feat(api): 为 Flask 服务添加内存缓存机制

- 添加内存缓存，默认TTL 5分钟（可通过 CACHE_TTL_SECONDS 环境变量配置） - 新增缓存相关端点： - POST /api/v1/cache/clear - 清理缓存 - GET /api/v1/cache/stats - 缓存统计信息 - /api/v1/ohlcv 支持 nocache 参数跳过缓存 - 响应中返回 cached 字段标识是否命中缓存 - 更新 API 文档和版本号到 1.1.0 - 删除不需要的 build-flask-and-push.sh 和 docker-compose.flask.yml
2026-05-07 23:12:32 +08:00
parent ae4cf5d3c8
commit d703974c5b
3 changed files with 113 additions and 201 deletions
--- a/build-flask-and-push.sh
+++ b/build-flask-and-push.sh
@@ -1,118 +0,0 @@
-#!/bin/bash
-# Flask API 服务构建和推送脚本
-# =============================
-
-set -e  # 遇到错误立即退出
-
-# 配置变量
-IMAGE_NAME="${1:-etf-data-fetcher}"
-REGISTRY="192.168.0.115:5000"
-TAG="${2:-latest}"
-FULL_IMAGE_NAME="${REGISTRY}/${IMAGE_NAME}:${TAG}"
-DOCKERFILE="Dockerfile.flask"
-
-echo "========================================="
-echo "Flask API 服务 - 构建和推送脚本"
-echo "========================================="
-echo "镜像名称: ${IMAGE_NAME}"
-echo "标签: ${TAG}"
-echo "Dockerfile: ${DOCKERFILE}"
-echo ""
-
-# 检查 Dockerfile 是否存在
-if [ ! -f "${DOCKERFILE}" ]; then
-    echo "❌ Dockerfile ${DOCKERFILE} 不存在"
-    exit 1
-fi
-
-# 检查并构建基础镜像（如果不存在）
-echo "0. 检查基础镜像..."
-if ! docker images --format "{{.Repository}}:{{.Tag}}" | grep -q "index-base:latest"; then
-    echo "   基础镜像不存在，开始构建..."
-    if [ -f "Dockerfile_base" ]; then
-        docker build --platform linux/arm64 -f Dockerfile_base -t index-base:latest .
-        echo "   ✅ 基础镜像构建成功"
-    else
-        echo "   ⚠️ 未找到 Dockerfile_base，跳过基础镜像构建"
-        exit 1
-    fi
-else
-    echo "   ✅ 基础镜像已存在"
-fi
-
-# 构建镜像
-echo ""
-echo "1. 构建 Flask API 镜像..."
-echo "   镜像名称: ${IMAGE_NAME}"
-echo "   平台: linux/arm64"
-docker build --platform linux/arm64 -f ${DOCKERFILE} -t ${IMAGE_NAME} .
-
-if [ $? -eq 0 ]; then
-    echo "   ✅ 镜像构建成功"
-else
-    echo "   ❌ 镜像构建失败"
-    exit 1
-fi
-
-# 打标签
-echo ""
-echo "2. 为镜像打标签..."
-echo "   标签: ${FULL_IMAGE_NAME}"
-docker tag ${IMAGE_NAME} ${FULL_IMAGE_NAME}
-
-if [ $? -eq 0 ]; then
-    echo "   ✅ 标签添加成功"
-else
-    echo "   ❌ 标签添加失败"
-    exit 1
-fi
-
-# 推送到私有仓库
-echo ""
-echo "3. 推送镜像到私有仓库..."
-echo "   仓库地址: ${REGISTRY}"
-docker push ${FULL_IMAGE_NAME}
-
-if [ $? -eq 0 ]; then
-    echo "   ✅ 镜像推送成功"
-else
-    echo "   ❌ 镜像推送失败"
-    exit 1
-fi
-
-# 清理本地镜像（可选）
-echo ""
-echo "4. 清理临时镜像..."
-docker rmi ${IMAGE_NAME} || true
-
-# 显示最终结果
-echo ""
-echo "========================================="
-echo "🎉 构建和推送完成！"
-echo "========================================="
-echo "镜像地址: ${FULL_IMAGE_NAME}"
-echo ""
-echo "可以使用以下命令运行容器："
-echo ""
-echo "# 基础运行（仅A股数据）"
-echo "docker run -d --name ${IMAGE_NAME}-container \\"
-echo "  -p 5000:5000 \\"
-echo "  -v \$(pwd)/.env:/app/.env:ro \\"
-echo "  ${FULL_IMAGE_NAME}"
-echo ""
-echo "# 启用SSH隧道（支持港美股）"
-echo "docker run -d --name ${IMAGE_NAME}-container \\"
-echo "  -p 5000:5000 \\"
-echo "  -v \$(pwd)/.env:/app/.env:ro \\"
-echo "  -v \$(pwd)/hk_ecs.pem:/app/hk_ecs.pem:ro \\"
-echo "  -e SSH_ENABLED=true \\"
-echo "  -e SSH_HOST=8.218.167.69 \\"
-echo "  -e SSH_USERNAME=root \\"
-echo "  -e SSH_KEY_PATH=hk_ecs.pem \\"
-echo "  ${FULL_IMAGE_NAME}"
-echo ""
-echo "# 测试API"
-echo "curl http://localhost:5000/health"
-echo "curl 'http://localhost:5000/api/v1/ohlcv?code=000300.SH&start=2024-01-01&end=2024-03-31'"
-echo ""
-echo "========================================="
--- a/core/datasource/flask_server.py
+++ b/core/datasource/flask_server.py
@@ -26,9 +26,11 @@ API 文档:
 import os
 import sys
 import json
+import hashlib
 from pathlib import Path
 from datetime import datetime, timedelta
 from typing import Optional, Dict, Any, List
+from functools import wraps

 # 添加项目根目录到路径
 project_root = Path(__file__).parent.parent.parent
@@ -58,6 +60,41 @@ CORS(app)  # 启用跨域支持
 fetcher: Optional[UniversalDataFetcher] = None
 ssh_config: Optional[Dict] = None

+# 内存缓存
+_cache: Dict[str, Any] = {}
+_cache_ttl: int = int(os.getenv('CACHE_TTL_SECONDS', '300'))  # 默认5分钟
+
+
+def _get_cache_key(code: str, start: str, end: str) -> str:
+    """生成缓存键"""
+    key = f"{code}:{start}:{end}"
+    return hashlib.md5(key.encode()).hexdigest()
+
+
+def _get_cached(key: str) -> Optional[Any]:
+    """获取缓存数据"""
+    if key not in _cache:
+        return None
+    
+    data, timestamp = _cache[key]
+    if datetime.now().timestamp() - timestamp > _cache_ttl:
+        # 缓存过期
+        del _cache[key]
+        return None
+    
+    return data
+
+
+def _set_cache(key: str, data: Any):
+    """设置缓存数据"""
+    _cache[key] = (data, datetime.now().timestamp())
+
+
+def clear_cache():
+    """清理所有缓存"""
+    _cache.clear()
+    print("✓ 缓存已清理")
+

 def get_ssh_config() -> Optional[Dict]:
    """
@@ -151,13 +188,22 @@ def index():
    """首页 - API 信息"""
    return jsonify({
        "name": "Universal Data Fetcher API",
-        "version": "1.0.0",
+        "version": "1.1.0",
        "description": "统一数据获取服务，支持A股、港股、美股、期货、加密货币",
+        "features": [
+            "自动资产类型识别",
+            "内存缓存（默认5分钟TTL）",
+            "SSH隧道支持（港美股）",
+            "批量数据获取"
+        ],
        "endpoints": {
            "health": "/health",
            "asset_type": "/api/v1/asset-type?code={code}",
            "ohlcv": "/api/v1/ohlcv?code={code}&start={YYYY-MM-DD}&end={YYYY-MM-DD}",
+            "ohlcv_nocache": "/api/v1/ohlcv?code={code}&start={YYYY-MM-DD}&end={YYYY-MM-DD}&nocache=true",
            "batch": "POST /api/v1/ohlcv/batch",
+            "cache_clear": "POST /api/v1/cache/clear",
+            "cache_stats": "/api/v1/cache/stats",
        },
        "supported_assets": [
            "A股指数 (000300.SH)",
@@ -169,6 +215,10 @@ def index():
            "期货 (AU.SHF)",
            "加密货币 (BTC)",
        ],
+        "cache_config": {
+            "ttl_seconds": _cache_ttl,
+            "current_size": len(_cache)
+        },
        "ssh_status": "enabled" if ssh_config and ssh_config.get('enabled') else "disabled",
    })

@@ -231,13 +281,14 @@ def detect_asset_type():
@app.route('/api/v1/ohlcv')
 def get_ohlcv():
    """
-    获取单只标的的 OHLCV 数据
+    获取单只标的的 OHLCV 数据（支持缓存）
    
    Query Parameters:
        code: 标的代码 (required)
        start: 开始日期，格式 YYYY-MM-DD (optional, 默认90天前)
        end: 结束日期，格式 YYYY-MM-DD (optional, 默认今天)
        retry: 重试次数 (optional, 默认3)
+        nocache: 是否跳过缓存 (optional, 默认false)
        
    Returns:
        {
@@ -248,13 +299,15 @@ def get_ohlcv():
                ...
            ],
            "count": 58,
-            "date_range": {"start": "2024-01-02", "end": "2024-03-29"}
+            "date_range": {"start": "2024-01-02", "end": "2024-03-29"},
+            "cached": false
        }
    """
    code = request.args.get('code', '').strip()
    start = request.args.get('start', '').strip()
    end = request.args.get('end', '').strip()
    retry = request.args.get('retry', '3')
+    nocache = request.args.get('nocache', 'false').lower() == 'true'
    
    # 参数验证
    if not code:
@@ -285,6 +338,14 @@ def get_ohlcv():
    # 检测资产类型
    asset_type = AssetTypeDetector.detect(code)
    
+    # 检查缓存
+    cache_key = _get_cache_key(code, start, end)
+    if not nocache:
+        cached_data = _get_cached(cache_key)
+        if cached_data:
+            cached_data['cached'] = True
+            return jsonify(cached_data)
+    
    # 获取数据
    fetcher = get_fetcher()
    
@@ -304,6 +365,11 @@ def get_ohlcv():
        result = dataframe_to_json(df)
        result['code'] = code
        result['asset_type'] = asset_type
+        result['cached'] = False
+        
+        # 存入缓存
+        if not nocache:
+            _set_cache(cache_key, result.copy())
        
        return jsonify(result)
        
@@ -431,6 +497,48 @@ def batch_ohlcv():
    })


+@app.route('/api/v1/cache/clear', methods=['POST'])
+def clear_cache_endpoint():
+    """
+    清理所有缓存数据
+    
+    Returns:
+        {"message": "Cache cleared", "cache_size": 0}
+    """
+    cache_size = len(_cache)
+    clear_cache()
+    return jsonify({
+        "message": "Cache cleared successfully",
+        "previous_size": cache_size,
+        "current_size": 0
+    })
+
+
+@app.route('/api/v1/cache/stats')
+def cache_stats():
+    """
+    获取缓存统计信息
+    
+    Returns:
+        {
+            "cache_size": 10,
+            "cache_ttl_seconds": 300,
+            "memory_estimate_mb": 5.2
+        }
+    """
+    # 估算内存使用（粗略估计）
+    import sys
+    total_size = 0
+    for key, (data, _) in _cache.items():
+        total_size += sys.getsizeof(key) + sys.getsizeof(data)
+    
+    return jsonify({
+        "cache_size": len(_cache),
+        "cache_ttl_seconds": _cache_ttl,
+        "memory_estimate_mb": round(total_size / 1024 / 1024, 2)
+    })
+
+
@app.route('/api/v1/supported-codes')
 def get_supported_codes():
    """
@@ -493,6 +601,8 @@ def not_found(error):
            "/api/v1/asset-type?code={code}",
            "/api/v1/ohlcv?code={code}&start={YYYY-MM-DD}&end={YYYY-MM-DD}",
            "/api/v1/ohlcv/batch",
+            "/api/v1/cache/clear",
+            "/api/v1/cache/stats",
            "/api/v1/supported-codes",
        ]
    }), 404
--- a/docker-compose.flask.yml
+++ b/docker-compose.flask.yml
@@ -1,80 +0,0 @@
-# Flask API 服务 Docker Compose 配置
-# ===================================
-
-version: '3.8'
-
-services:
-  etf-data-fetcher:
-    build:
-      context: .
-      dockerfile: Dockerfile.flask
-    image: 192.168.0.115:5000/etf-data-fetcher:latest
-    container_name: etf-data-fetcher
-    
-    ports:
-      - "5000:5000"
-    
-    environment:
-      # Flask 配置
-      - FLASK_ENV=production
-      - PYTHONUNBUFFERED=1
-      
-      # SSH 隧道配置（可选，启用后可获取港美股数据）
-      - SSH_ENABLED=${SSH_ENABLED:-false}
-      - SSH_HOST=${SSH_HOST:-8.218.167.69}
-      - SSH_PORT=${SSH_PORT:-22}
-      - SSH_USERNAME=${SSH_USERNAME:-root}
-      - SSH_KEY_PATH=${SSH_KEY_PATH:-hk_ecs.pem}
-      - SSH_LOCAL_PORT=${SSH_LOCAL_PORT:-1080}
-    
-    volumes:
-      # 环境变量配置（必需：TUSHARE_TOKEN）
-      - ./.env:/app/.env:ro
-      
-      # SSH 私钥（可选，获取港美股数据需要）
-      - ./hk_ecs.pem:/app/hk_ecs.pem:ro
-      
-      # 数据目录（可选，用于持久化缓存）
-      - ./data:/app/data
-      
-      # 日志目录
-      - ./logs:/app/logs
-    
-    # 健康检查
-    healthcheck:
-      test: ["CMD", "python", "-c", "import requests; requests.get('http://localhost:5000/health')"]
-      interval: 30s
-      timeout: 10s
-      retries: 3
-      start_period: 10s
-    
-    # 重启策略
-    restart: unless-stopped
-    
-    # 资源限制
-    deploy:
-      resources:
-        limits:
-          cpus: '2.0'
-          memory: 2G
-        reservations:
-          cpus: '0.5'
-          memory: 512M
-
-  # 可选：Nginx 反向代理
-  nginx:
-    image: nginx:alpine
-    container_name: etf-data-fetcher-nginx
-    ports:
-      - "80:80"
-    volumes:
-      - ./nginx.conf:/etc/nginx/nginx.conf:ro
-    depends_on:
-      - etf-data-fetcher
-    restart: unless-stopped
-    profiles:
-      - with-nginx
-
-networks:
-  default:
-    name: etf-network