Python爬虫头部伪装详解

一、HTTP头部基础概念

1. 常见HTTP头部字段

headers = {
    'User-Agent': '',      # 用户代理，标识浏览器/设备信息
    'Referer': '',         # 来源页面URL
    'Cookie': '',          # 网站身份凭证
    'Accept': '',          # 可接受的响应类型
    'Accept-Language': '', # 语言偏好
    'Accept-Encoding': '', # 可接受的压缩编码
    'Connection': '',      # 连接控制
    'Host': ''             # 目标主机
}

2. 头部伪装的核心目的

绕过反爬虫机制
模拟真实用户行为
解决403/404等访问限制
突破地域/设备限制

二、基础伪装技巧

1. 随机User-Agent生成

from fake_useragent import UserAgent

# 方法1：使用fake-useragent库
ua = UserAgent()
random_ua = ua.random

# 方法2：自定义UA列表
user_agents = [
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36...',
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15...',
    'Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15...'
]

2. 完整头部模板

def get_random_headers():
    return {
        'User-Agent': UserAgent().random,
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
        'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',
        'Accept-Encoding': 'gzip, deflate, br',
        'Connection': 'keep-alive',
        'Upgrade-Insecure-Requests': '1',
        'Referer': 'https://www.google.com/',
        'DNT': '1'  # 禁止追踪
    }

三、高级伪装策略

1. 动态Cookie处理

import requests
from bs4 import BeautifulSoup

def get_dynamic_cookies(url):
    session = requests.Session()
    # 首次访问获取初始Cookie
    response = session.get(url, headers=get_random_headers())
    # 解析页面获取动态参数
    soup = BeautifulSoup(response.text, 'html.parser')
    csrf_token = soup.find('input', {'name': 'csrf_token'})['value']
    # 更新Cookie
    session.cookies.update({'csrftoken': csrf_token})
    return session.cookies.get_dict()

2. Referer链模拟

referer_chain = [
    'https://www.google.com/search?q=目标网站',
    'https://www.baidu.com/s?wd=目标关键词',
    'https://www.bing.com/search?q=相关内容'
]

def get_next_referer():
    """轮换使用不同的来源页面"""
    current = getattr(get_next_referer, 'current', 0)
    referer = referer_chain[current % len(referer_chain)]
    get_next_referer.current = current + 1
    return referer

3. 请求间隔随机化

import time
import random

def random_delay():
    """随机延迟1-5秒"""
    time.sleep(random.uniform(1, 5))

def random_quick_delay():
    """快速模式随机延迟0.1-1秒""" 
    time.sleep(random.uniform(0.1, 1))

四、反反爬虫技巧

1. 头部指纹检测绕过

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36...',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
    'Accept-Encoding': 'gzip, deflate, br',  # 注意：requests不支持br解码
    'Accept-Language': 'zh-CN,zh;q=0.9',
    'Cache-Control': 'max-age=0',
    'Connection': 'keep-alive',
    'Upgrade-Insecure-Requests': '1',
    'Sec-Fetch-Dest': 'document',
    'Sec-Fetch-Mode': 'navigate',
    'Sec-Fetch-Site': 'none',
    'Sec-Fetch-User': '?1',
    'Pragma': 'no-cache'
}

2. TLS指纹伪装

# 使用curl_cffi库模拟浏览器TLS指纹
from curl_cffi import requests

response = requests.get(
    "https://目标网站.com",
    headers=get_random_headers(),
    impersonate="chrome110"  # 模拟Chrome 110的TLS指纹
)

3. WebSocket握手头伪装

ws_headers = {
    'Connection': 'Upgrade',
    'Upgrade': 'websocket',
    'Sec-WebSocket-Version': '13',
    'Sec-WebSocket-Key': 'x3JJHMbDL1EzLkh9GBhXDw==',
    'Origin': 'https://目标网站.com'
}

五、实战案例

案例1：电商网站商品爬取

import requests
from random import choice

def scrape_ecommerce(product_id):
    url = f"https://api.ecommerce.com/products/{product_id}"

    proxies = [
        'http://user:pass@proxy1.com:8000',
        'http://user:pass@proxy2.com:8000'
    ]

    headers = {
        'User-Agent': choice(user_agents),
        'Referer': 'https://www.ecommerce.com/',
        'X-Requested-With': 'XMLHttpRequest',
        'Sec-Ch-Ua': '"Chromium";v="110", "Not A(Brand";v="24"',
        'Sec-Ch-Ua-Mobile': '?0',
        'Sec-Ch-Ua-Platform': '"Windows"'
    }

    response = requests.get(
        url,
        headers=headers,
        proxies={'http': choice(proxies)},
        timeout=10
    )

    return response.json()

案例2：动态内容加载

from selenium.webdriver import Chrome
from selenium.webdriver.chrome.options import Options

def scrape_dynamic_content(url):
    options = Options()
    options.add_argument(f'user-agent={UserAgent().random}')
    options.add_argument('--disable-blink-features=AutomationControlled')
    options.add_experimental_option("excludeSwitches", ["enable-automation"])

    driver = Chrome(options=options)
    driver.execute_cdp_cmd(
        'Network.setUserAgentOverride',
        {'userAgent': UserAgent().random}
    )

    driver.get(url)
    content = driver.page_source
    driver.quit()

    return content

六、检测与调试

1. 头部检测工具

def check_headers(url):
    test_url = 'https://httpbin.org/headers'
    response = requests.get(test_url, headers=get_random_headers())
    print("当前请求头部：")
    print(json.dumps(response.json(), indent=2))