page-fetcher.ts•1.7 kB
import { HttpsProxyAgent } from 'https-proxy-agent';
import fetch from 'node-fetch';
import type { Browser } from 'puppeteer';
export class PageFetcher {
constructor(private proxy: string) {}
async fetchWithProxy(url: string): Promise<string> {
const options: any = {
headers: {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
}
};
if (this.proxy) {
options.agent = new HttpsProxyAgent(this.proxy);
}
const response = await fetch(url, options);
if (!response.ok) {
throw new Error(`获取网页失败: ${response.status} ${response.statusText}`);
}
return response.text();
}
async fetchWithPuppeteer(url: string): Promise<string> {
let browser: Browser | undefined;
try {
const { default: puppeteer } = await import('puppeteer');
const args = ['--no-sandbox', '--disable-setuid-sandbox'];
if (this.proxy) {
args.push(`--proxy-server=${this.proxy}`);
}
browser = await puppeteer.launch({
headless: true,
args
});
const page = await browser.newPage();
await page.goto(url, { waitUntil: 'networkidle0', timeout: 30000 });
const content = await page.content();
await browser.close();
return content;
} catch (error) {
if (browser) {
await browser.close();
}
const errorMessage = error instanceof Error ? error.message : String(error);
throw new Error(`使用 Puppeteer 获取页面失败。如果需要 JavaScript 渲染，请先安装 Puppeteer: npm install puppeteer。错误: ${errorMessage}`);
}
}
}