fix(http): 用urllib3替代requests修复SSL EOF错误
问题根因: - Python OpenSSL 3.5.4 + requests 2.32.4 + urllib3 2.5.0 版本不兼容 - requests 2.32.4 内部使用 urllib3 的方式与 urllib3 2.5.0 API 不兼容 - curl(SecureTransport)正常工作,但 Python requests(OpenSSL)失败 - 服务器(Caddy)使用 TLS 1.3 + X25519MLKEM768(后量子密钥交换) 修复方案: - 用 urllib3.PoolManager 直接发起 HTTP 请求(已验证可正常工作) - 封装 _http_get() 函数替代 requests.get() - 替换所有 requests 相关异常类型为 urllib3 异常 修改文件: - datasource/flask_api_source.py: 核心数据源层 - rotation/simple_rotation.py: 简单轮动策略层
This commit is contained in:
@@ -7,7 +7,9 @@ Flask API 数据源
|
|||||||
|
|
||||||
import os
|
import os
|
||||||
import json
|
import json
|
||||||
import requests
|
import time
|
||||||
|
import urllib3
|
||||||
|
import urllib.parse
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
from typing import Optional, Dict, List
|
from typing import Optional, Dict, List
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
@@ -18,6 +20,22 @@ from .models import OHLCVResponse, validate_ohlcv_response
|
|||||||
|
|
||||||
load_dotenv()
|
load_dotenv()
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
# HTTP client (urllib3 替代 requests,修复 SSL EOF 问题)
|
||||||
|
# ============================================================
|
||||||
|
|
||||||
|
_http_pool = urllib3.PoolManager()
|
||||||
|
|
||||||
|
def _http_get(url: str, params: dict = None, timeout: int = 120) -> urllib3.HTTPResponse:
|
||||||
|
"""使用 urllib3 发起 GET 请求(替代 requests.get,修复 OpenSSL 3.5 + Caddy 的 SSL EOF 问题)"""
|
||||||
|
if params:
|
||||||
|
url = url + '?' + urllib.parse.urlencode(params)
|
||||||
|
return _http_pool.request('GET', url, timeout=urllib3.Timeout(connect=10, read=timeout))
|
||||||
|
|
||||||
|
def _parse_json(resp: urllib3.HTTPResponse) -> dict:
|
||||||
|
"""解析 JSON 响应"""
|
||||||
|
return json.loads(resp.data.decode('utf-8'))
|
||||||
|
|
||||||
|
|
||||||
class FlaskAPIDataSource:
|
class FlaskAPIDataSource:
|
||||||
"""
|
"""
|
||||||
@@ -110,24 +128,17 @@ class FlaskAPIDataSource:
|
|||||||
|
|
||||||
for attempt in range(self.retries):
|
for attempt in range(self.retries):
|
||||||
try:
|
try:
|
||||||
response = requests.get(
|
response = _http_get(url, params=params, timeout=self.timeout)
|
||||||
url,
|
|
||||||
params=params,
|
|
||||||
timeout=self.timeout
|
|
||||||
)
|
|
||||||
|
|
||||||
if response.status_code != 200:
|
if response.status != 200:
|
||||||
if attempt < self.retries - 1:
|
if attempt < self.retries - 1:
|
||||||
|
time.sleep(1 + attempt)
|
||||||
continue
|
continue
|
||||||
print(f"✗ API请求失败: {response.status_code} - {response.text[:100]}")
|
print(f"✗ API请求失败: {response.status} - {response.data.decode('utf-8', errors='replace')[:100]}")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
# 尝试解析 JSON(支持 zstd 响应)
|
# 解析 JSON
|
||||||
try:
|
data = _parse_json(response)
|
||||||
data = response.json()
|
|
||||||
except (json.JSONDecodeError, requests.exceptions.JSONDecodeError):
|
|
||||||
# 如果 response.json() 失败,手动解析
|
|
||||||
data = json.loads(response.text)
|
|
||||||
|
|
||||||
# 检查错误
|
# 检查错误
|
||||||
if 'error' in data:
|
if 'error' in data:
|
||||||
@@ -196,15 +207,25 @@ class FlaskAPIDataSource:
|
|||||||
print(f"✓ {code}: {actual_count} 条数据 ({actual_start} ~ {actual_end})")
|
print(f"✓ {code}: {actual_count} 条数据 ({actual_start} ~ {actual_end})")
|
||||||
return df
|
return df
|
||||||
|
|
||||||
except requests.exceptions.Timeout:
|
except urllib3.exceptions.TimeoutError:
|
||||||
if attempt < self.retries - 1:
|
if attempt < self.retries - 1:
|
||||||
print(f"⚠ {code}: 请求超时,重试 {attempt + 2}/{self.retries}")
|
print(f"⚠ {code}: 请求超时,重试 {attempt + 2}/{self.retries}")
|
||||||
|
time.sleep(1 + attempt)
|
||||||
continue
|
continue
|
||||||
print(f"✗ {code}: 请求超时")
|
print(f"✗ {code}: 请求超时")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
except requests.exceptions.RequestException as e:
|
except (urllib3.exceptions.SSLError, urllib3.exceptions.MaxRetryError, urllib3.exceptions.ProtocolError) as e:
|
||||||
if attempt < self.retries - 1:
|
if attempt < self.retries - 1:
|
||||||
|
print(f"⚠ {code}: {type(e).__name__},重试 {attempt + 2}/{self.retries}")
|
||||||
|
time.sleep(1 + attempt)
|
||||||
|
continue
|
||||||
|
print(f"✗ {code}: {type(e).__name__} after {self.retries} retries")
|
||||||
|
return None
|
||||||
|
|
||||||
|
except urllib3.exceptions.HTTPError as e:
|
||||||
|
if attempt < self.retries - 1:
|
||||||
|
time.sleep(1 + attempt)
|
||||||
continue
|
continue
|
||||||
print(f"✗ {code}: 请求异常 - {e}")
|
print(f"✗ {code}: 请求异常 - {e}")
|
||||||
return None
|
return None
|
||||||
@@ -277,16 +298,12 @@ class FlaskAPIDataSource:
|
|||||||
}
|
}
|
||||||
|
|
||||||
try:
|
try:
|
||||||
response = requests.get(url, params=params, timeout=self.timeout)
|
response = _http_get(url, params=params, timeout=self.timeout)
|
||||||
|
|
||||||
if response.status_code != 200:
|
if response.status != 200:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
# 处理 zstd 响应
|
data = _parse_json(response)
|
||||||
try:
|
|
||||||
data = response.json()
|
|
||||||
except (json.JSONDecodeError, requests.exceptions.JSONDecodeError):
|
|
||||||
data = json.loads(response.text)
|
|
||||||
|
|
||||||
if 'error' in data:
|
if 'error' in data:
|
||||||
return None
|
return None
|
||||||
@@ -357,9 +374,9 @@ class FlaskAPIDataSource:
|
|||||||
params = {'code': '000300.SH', 'start': '2024-01-01', 'end': '2024-01-05'}
|
params = {'code': '000300.SH', 'start': '2024-01-01', 'end': '2024-01-05'}
|
||||||
|
|
||||||
try:
|
try:
|
||||||
response = requests.get(url, params=params, timeout=self.timeout)
|
response = _http_get(url, params=params, timeout=self.timeout)
|
||||||
if response.status_code == 200:
|
if response.status == 200:
|
||||||
data = response.json()
|
data = _parse_json(response)
|
||||||
return {
|
return {
|
||||||
'status': 'healthy',
|
'status': 'healthy',
|
||||||
'ssh_configured': True,
|
'ssh_configured': True,
|
||||||
@@ -375,11 +392,11 @@ class FlaskAPIDataSource:
|
|||||||
url = f"{self.base_url}/api/v1/calendar/info"
|
url = f"{self.base_url}/api/v1/calendar/info"
|
||||||
|
|
||||||
try:
|
try:
|
||||||
response = requests.get(url, timeout=10)
|
response = _http_get(url, timeout=10)
|
||||||
if response.status_code == 200:
|
if response.status == 200:
|
||||||
return response.json()
|
return _parse_json(response)
|
||||||
else:
|
else:
|
||||||
return {"error": f"HTTP {response.status_code}"}
|
return {"error": f"HTTP {response.status}"}
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
return {"error": str(e)}
|
return {"error": str(e)}
|
||||||
|
|
||||||
@@ -420,20 +437,17 @@ class FlaskAPIDataSource:
|
|||||||
|
|
||||||
for attempt in range(self.retries):
|
for attempt in range(self.retries):
|
||||||
try:
|
try:
|
||||||
response = requests.get(
|
response = _http_get(url, params=params, timeout=self.timeout)
|
||||||
url,
|
|
||||||
params=params,
|
|
||||||
timeout=self.timeout
|
|
||||||
)
|
|
||||||
|
|
||||||
if response.status_code != 200:
|
if response.status != 200:
|
||||||
if attempt < self.retries - 1:
|
if attempt < self.retries - 1:
|
||||||
print(f"⚠ 交易日历请求失败 (HTTP {response.status_code}),重试 {attempt + 2}/{self.retries}")
|
print(f"⚠ 交易日历请求失败 (HTTP {response.status}),重试 {attempt + 2}/{self.retries}")
|
||||||
|
time.sleep(1 + attempt)
|
||||||
continue
|
continue
|
||||||
print(f"✗ 交易日历请求失败: HTTP {response.status_code} - {response.text[:100]}")
|
print(f"✗ 交易日历请求失败: HTTP {response.status} - {response.data.decode('utf-8', errors='replace')[:100]}")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
data = response.json()
|
data = _parse_json(response)
|
||||||
|
|
||||||
# 检查错误
|
# 检查错误
|
||||||
if 'error' in data:
|
if 'error' in data:
|
||||||
@@ -454,20 +468,30 @@ class FlaskAPIDataSource:
|
|||||||
print(f"✓ {market} ({exchange}): {count} 个交易日 ({start_date} ~ {end_date})")
|
print(f"✓ {market} ({exchange}): {count} 个交易日 ({start_date} ~ {end_date})")
|
||||||
return dates
|
return dates
|
||||||
|
|
||||||
except requests.exceptions.Timeout:
|
except urllib3.exceptions.TimeoutError:
|
||||||
if attempt < self.retries - 1:
|
if attempt < self.retries - 1:
|
||||||
print(f"⚠ 交易日历请求超时,重试 {attempt + 2}/{self.retries}")
|
print(f"⚠ 交易日历请求超时,重试 {attempt + 2}/{self.retries}")
|
||||||
|
time.sleep(1 + attempt)
|
||||||
continue
|
continue
|
||||||
print(f"✗ 交易日历请求超时")
|
print(f"✗ 交易日历请求超时")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
except requests.exceptions.RequestException as e:
|
except (urllib3.exceptions.SSLError, urllib3.exceptions.MaxRetryError, urllib3.exceptions.ProtocolError) as e:
|
||||||
if attempt < self.retries - 1:
|
if attempt < self.retries - 1:
|
||||||
|
print(f"⚠ 交易日历: {type(e).__name__},重试 {attempt + 2}/{self.retries}")
|
||||||
|
time.sleep(1 + attempt)
|
||||||
|
continue
|
||||||
|
print(f"✗ 交易日历: {type(e).__name__} after {self.retries} retries")
|
||||||
|
return None
|
||||||
|
|
||||||
|
except urllib3.exceptions.HTTPError as e:
|
||||||
|
if attempt < self.retries - 1:
|
||||||
|
time.sleep(1 + attempt)
|
||||||
continue
|
continue
|
||||||
print(f"✗ 交易日历请求异常: {e}")
|
print(f"✗ 交易日历请求异常: {e}")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
except (json.JSONDecodeError, requests.exceptions.JSONDecodeError) as e:
|
except json.JSONDecodeError as e:
|
||||||
print(f"✗ 交易日历 JSON 解析失败: {e}")
|
print(f"✗ 交易日历 JSON 解析失败: {e}")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@@ -478,8 +502,8 @@ class FlaskAPIDataSource:
|
|||||||
url = f"{self.base_url}/"
|
url = f"{self.base_url}/"
|
||||||
|
|
||||||
try:
|
try:
|
||||||
response = requests.get(url, timeout=10)
|
response = _http_get(url, timeout=10)
|
||||||
return response.json()
|
return _parse_json(response)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
return {"error": str(e)}
|
return {"error": str(e)}
|
||||||
|
|
||||||
|
|||||||
@@ -14,7 +14,8 @@ import sys
|
|||||||
import math
|
import math
|
||||||
import json
|
import json
|
||||||
import time
|
import time
|
||||||
import requests
|
import urllib3
|
||||||
|
import urllib.parse
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -26,6 +27,30 @@ sys.path.insert(0, str(PROJECT_ROOT))
|
|||||||
|
|
||||||
from rotation.config_loader import load_rotation_config, RotationStrategyConfig
|
from rotation.config_loader import load_rotation_config, RotationStrategyConfig
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
# HTTP client (urllib3 替代 requests,修复 SSL EOF 问题)
|
||||||
|
# ============================================================
|
||||||
|
|
||||||
|
_http_pool = urllib3.PoolManager(timeout=urllib3.Timeout(connect=10, read=120))
|
||||||
|
|
||||||
|
class _HttpResponse:
|
||||||
|
"""urllib3 响应包装,提供 requests 兼容接口"""
|
||||||
|
def __init__(self, resp):
|
||||||
|
self.status_code = resp.status
|
||||||
|
self._data = resp.data
|
||||||
|
self._json = None
|
||||||
|
def json(self):
|
||||||
|
if self._json is None:
|
||||||
|
self._json = json.loads(self._data)
|
||||||
|
return self._json
|
||||||
|
|
||||||
|
def _http_get(url: str, params: dict = None, timeout: int = 120) -> _HttpResponse:
|
||||||
|
"""使用 urllib3 发起 GET 请求(替代 requests.get,修复 OpenSSL 3.5 + Caddy 的 SSL EOF 问题)"""
|
||||||
|
if params:
|
||||||
|
url = url + '?' + urllib.parse.urlencode(params)
|
||||||
|
resp = _http_pool.request('GET', url, timeout=urllib3.Timeout(connect=10, read=timeout))
|
||||||
|
return _HttpResponse(resp)
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
# ============================================================
|
||||||
# Pure functions: momentum
|
# Pure functions: momentum
|
||||||
@@ -122,7 +147,7 @@ class DataCache:
|
|||||||
params = {'code': code, 'start': start_date, 'end': end_date, 'adj': adj}
|
params = {'code': code, 'start': start_date, 'end': end_date, 'adj': adj}
|
||||||
for attempt in range(3):
|
for attempt in range(3):
|
||||||
try:
|
try:
|
||||||
resp = requests.get(url, params=params, timeout=self.timeout)
|
resp = _http_get(url, params=params, timeout=self.timeout)
|
||||||
if resp.status_code != 200:
|
if resp.status_code != 200:
|
||||||
if attempt < 2:
|
if attempt < 2:
|
||||||
time.sleep(1)
|
time.sleep(1)
|
||||||
@@ -149,10 +174,12 @@ class DataCache:
|
|||||||
self._save_premium_cache(code, df.attrs['premium_series'])
|
self._save_premium_cache(code, df.attrs['premium_series'])
|
||||||
print(f" + {code}: {len(df)} rows ({adj})")
|
print(f" + {code}: {len(df)} rows ({adj})")
|
||||||
return df
|
return df
|
||||||
except requests.exceptions.Timeout:
|
except (urllib3.exceptions.TimeoutError, urllib3.exceptions.SSLError, urllib3.exceptions.MaxRetryError, urllib3.exceptions.ProtocolError) as e:
|
||||||
|
# 网络相关错误(超时、SSL、连接断开)都进行重试
|
||||||
if attempt < 2:
|
if attempt < 2:
|
||||||
|
time.sleep(1 + attempt) # 递增延迟: 1s, 2s
|
||||||
continue
|
continue
|
||||||
print(f" x {code}: timeout")
|
print(f" x {code}: {type(e).__name__} after {attempt+1} retries")
|
||||||
return None
|
return None
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f" x {code}: {e}")
|
print(f" x {code}: {e}")
|
||||||
@@ -207,10 +234,10 @@ class DataCache:
|
|||||||
params = {'code': code, 'start': start_date, 'end': end_date, 'adj': 'raw'}
|
params = {'code': code, 'start': start_date, 'end': end_date, 'adj': 'raw'}
|
||||||
for attempt in range(3):
|
for attempt in range(3):
|
||||||
try:
|
try:
|
||||||
resp = requests.get(url, params=params, timeout=self.timeout)
|
resp = _http_get(url, params=params, timeout=self.timeout)
|
||||||
if resp.status_code != 200:
|
if resp.status_code != 200:
|
||||||
if attempt < 2:
|
if attempt < 2:
|
||||||
time.sleep(1)
|
time.sleep(1 + attempt)
|
||||||
continue
|
continue
|
||||||
return
|
return
|
||||||
data = resp.json()
|
data = resp.json()
|
||||||
@@ -225,8 +252,9 @@ class DataCache:
|
|||||||
self._save_premium_cache(code, self.premium_data[code])
|
self._save_premium_cache(code, self.premium_data[code])
|
||||||
print(f" + premium {code}: +{len(new_data)} days (total {len(self.premium_data[code])})")
|
print(f" + premium {code}: +{len(new_data)} days (total {len(self.premium_data[code])})")
|
||||||
return
|
return
|
||||||
except requests.exceptions.Timeout:
|
except (urllib3.exceptions.TimeoutError, urllib3.exceptions.SSLError, urllib3.exceptions.MaxRetryError, urllib3.exceptions.ProtocolError):
|
||||||
if attempt < 2:
|
if attempt < 2:
|
||||||
|
time.sleep(1 + attempt)
|
||||||
continue
|
continue
|
||||||
return
|
return
|
||||||
except Exception:
|
except Exception:
|
||||||
@@ -238,9 +266,10 @@ class DataCache:
|
|||||||
params = {'market': market, 'start': start_date, 'end': end_date}
|
params = {'market': market, 'start': start_date, 'end': end_date}
|
||||||
for attempt in range(3):
|
for attempt in range(3):
|
||||||
try:
|
try:
|
||||||
resp = requests.get(url, params=params, timeout=self.timeout)
|
resp = _http_get(url, params=params, timeout=self.timeout)
|
||||||
if resp.status_code != 200:
|
if resp.status_code != 200:
|
||||||
if attempt < 2:
|
if attempt < 2:
|
||||||
|
time.sleep(1 + attempt)
|
||||||
continue
|
continue
|
||||||
return None
|
return None
|
||||||
data = resp.json()
|
data = resp.json()
|
||||||
@@ -253,8 +282,15 @@ class DataCache:
|
|||||||
result = pd.DatetimeIndex(dates)
|
result = pd.DatetimeIndex(dates)
|
||||||
print(f" + {market}: {len(result)} trading days ({start_date} ~ {end_date})")
|
print(f" + {market}: {len(result)} trading days ({start_date} ~ {end_date})")
|
||||||
return result
|
return result
|
||||||
|
except (urllib3.exceptions.TimeoutError, urllib3.exceptions.SSLError, urllib3.exceptions.MaxRetryError, urllib3.exceptions.ProtocolError) as e:
|
||||||
|
if attempt < 2:
|
||||||
|
time.sleep(1 + attempt)
|
||||||
|
continue
|
||||||
|
print(f" x calendar: {type(e).__name__} after {attempt+1} retries")
|
||||||
|
return None
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
if attempt < 2:
|
if attempt < 2:
|
||||||
|
time.sleep(1 + attempt)
|
||||||
continue
|
continue
|
||||||
print(f" x calendar: {e}")
|
print(f" x calendar: {e}")
|
||||||
return None
|
return None
|
||||||
|
|||||||
Reference in New Issue
Block a user