(cache)sveltekit-web-scraper/src/routes/api/web-scraper/+server.js at master · b-dulaney/sveltekit-web-scraper · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
import { dev } from '$app/environment';
import {
	LOCAL_CHROMIUM_PATH,
	CHROMIUM_DOWNLOAD_URL,
	BLOB_READ_WRITE_TOKEN
} from '$env/static/private';
import { put } from '@vercel/blob';
import chromium from '@sparticuz/chromium-min';
import puppeteer from 'puppeteer-core';
import { json } from '@sveltejs/kit';

/** @type {import('@sveltejs/kit').RequestHandler} */
export async function POST({ request }) {
	const { url } = await request.json();

	try {
		const screenshot = await takeScreenshot(url);
		const imageURL = await uploadImage(screenshot, url);
		return json({ success: true, imageURL });
	} catch (/** @type {any}*/ e) {
		return json({ success: false, error: e?.message });
	}
}

/**
 * Takes a screenshot of the page at the given URL with Puppeteer and returns the image Buffer
 * @param {string} pageURL
 * @returns {Promise<Buffer>}
 */
async function takeScreenshot(pageURL) {
	const browser = await puppeteer.launch({
		args: chromium.args,
		defaultViewport: chromium.defaultViewport,
		executablePath: dev ? LOCAL_CHROMIUM_PATH : await chromium.executablePath(CHROMIUM_DOWNLOAD_URL)
	});

	const page = await browser.newPage();
	page.setDefaultNavigationTimeout(60 * 1000);

	// This try/catch block handles the cases where the page fails to load but returns a valid HTTP status as well
	// as the case where the goto() function throws an error (e.g. due to a timeout)
	try {
		const response = await page.goto(pageURL);
		if (response && !response.ok) {
			await browser.close();
			console.warn(`Failed to load page: ${response.status()} ${response.statusText()}`);
			throw new Error(`Failed to load page: ${response.status()} ${response.statusText()}`);
		}
	} catch (/** @type {any}*/ e) {
		await browser.close();
		console.warn(`Failed to load page: ${e?.message}`);
		throw new Error(`Failed to load page: ${e?.message}`);
	}

	const screenshot = await page.screenshot({
		type: 'webp',
		captureBeyondViewport: false
	});
	await browser.close();
	return screenshot;
}

/**
 * Uploads the image to vercel blob storage and returns the image url
 * @param {Buffer} buffer
 * @param {string} pageURL
 * @returns {Promise<string>}
 */
async function uploadImage(buffer, pageURL) {
	const pathName = trimUrl(pageURL);
	const { url } = await put(`screenshots/${pathName}`, buffer, {
		access: 'public',
		token: BLOB_READ_WRITE_TOKEN,
		contentType: 'image/webp'
	});

	return url;
}

/**
 * Trims http protocol and any trailing slashes from the url to prevent extra folder creation in vercel blob storage
 * @param {string} pageURL
 * @returns {string}
 */
function trimUrl(pageURL) {
	// Remove leading protocol
	let trimmedUrl = pageURL.replace(/(^\w+:|^)\/\//, '');
	// Remove trailing slashes
	trimmedUrl = trimmedUrl.replace(/\/+$/, '');
	return trimmedUrl;
}