import requests
import json

def fetch_ajax_data(url, params, headers):
response = requests.get(url, params=params, headers=headers)
return response.json()

# 示例使用
url = 'https://api.example.com/data'
params = {
'page': 1,
'limit': 10
}
headers = {
'user-agent': 'mozilla/5.0 (windows nt 10.0; win64; x64) applewebkit/537.36 (khtml, like gecko) chrome/91.0.4472.124 safari/537.36',
'referer': 'https://www.example.com',
'x-requested-with': 'xmlhttprequest'
}

data = fetch_ajax_data(url, params, headers)
print(json.dumps(data, indent=2))

from selenium import webdriver
from selenium.webdriver.chrome.options import options
from selenium.webdriver.common.by import by
from selenium.webdriver.support.ui import webdriverwait
from selenium.webdriver.support import expected_conditions as ec
import json

def fetch_ajax_data_with_selenium(url):
chrome_options = options()
chrome_options.add_argument("--headless")

driver = webdriver.chrome(options=chrome_options)
driver.get(url)

# 等待特定元素加载完成
element = webdriverwait(driver, 10).until(
ec.presence_of_element_located((by.class_name, "data-container"))
)

# 提取数据
data = driver.ｅxecute_script("return window.ajaxdata;") # 假设数据存储在全局变量中

driver.quit()
return data

# 使用示例
url = 'https://www.example.com/ajax-page'
data = fetch_ajax_data_with_selenium(url)
print(json.dumps(data, indent=2))

import requests

def fetch_https_data(url):
response = requests.get(url)
response.raise_for_status() # 如果请求不成功则抛出异常
return response.text

# 使用示例
url = 'https://api.github.com'
data = fetch_https_data(url)
print(data)

import requests
import certifi

def fetch_https_data_with_cert(url):
response = requests.get(url, verify=certifi.where())
response.raise_for_status()
return response.text

# 使用示例
url = 'https://self-signed.badssl.com/'
try:
data = fetch_https_data_with_cert(url)
print(data)
except requests.exceptions.sslerror as e:
print(f"ssl证书验证失败: {e}")

import requests

def fetch_https_data_with_client_cert(url, cert_path):
response = requests.get(url, cert=cert_path)
response.raise_for_status()
return response.text

# 使用示例
url = 'https://client.badssl.com/'
cert_path = ('path/to/client.crt', 'path/to/client.key')
try:
data = fetch_https_data_with_client_cert(url, cert_path)
print(data)
except requests.exceptions.requestexception as e:
print(f"请求失败: {e}")

import requests
import json
from selenium import webdriver
from selenium.webdriver.chrome.options import options
from selenium.webdriver.common.by import by
from selenium.webdriver.support.ui import webdriverwait
from selenium.webdriver.support import expected_conditions as ec

class advancedscraper:
def __init__(self):
self.session = requests.session()
self.session.headers.update({
'user-agent': 'mozilla/5.0 (windows nt 10.0; win64; x64) applewebkit/537.36 (khtml, like gecko) chrome/91.0.4472.124 safari/537.36'
})

chrome_options = options()
chrome_options.add_argument("--headless")
self.driver = webdriver.chrome(options=chrome_options)

def fetch_api_data(self, url, params=none):
response = self.session.get(url, params=params)
response.raise_for_status()
return response.json()

def fetch_ajax_data(self, url):
self.driver.get(url)

# 等待ajax内容加载
webdriverwait(self.driver, 10).until(
ec.presence_of_element_located((by.class_name, "ajax-content"))
)

# 提取数据
data = self.driver.ｅxecute_script("return window.ajaxdata;")
return data

def scrape_website(self, base_url, api_endpoint):
# 爬取主页面
main_page = self.session.get(base_url)
main_page.raise_for_status()

# 爬取api数据
api_data = self.fetch_api_data(api_endpoint)

# 爬取ajax加载的数据
ajax_data = self.fetch_ajax_data(base_url)

return {
'main_page': main_page.text,
'api_data': api_data,
'ajax_data': ajax_data
}

def close(self):
self.driver.quit()

# 使用示例
def main():
scraper = advancedscraper()
try:
data = scraper.scrape_website(
'https://example.com',
'https://api.example.com/data'
)
print(json.dumps(data, indent=2))
finally:
scraper.close()

if __name__ == '__main__':
main()