流式传输大型 JSON

使用 Streams API 处理若完全缓冲会超出 Worker 128 MB 内存限制的 JSON 载荷。流式传输允许你在数据到达时增量解析和转换 JSON。这比将整个载荷缓冲到内存中更快，Worker 可以立即开始处理数据，并能在内存限制内处理 GB 级载荷或文件。

@streamparser/json-whatwg ↗ 库提供与 Web Streams API 兼容的流式 JSON 解析器。

安装依赖：

npm install @streamparser/json-whatwg

流式传输 JSON 请求体

此示例解析大型 JSON 请求体，并在不将整个载荷加载到内存的情况下提取特定字段。

import { JSONParser } from "@streamparser/json-whatwg";

export default {
	async fetch(request): Promise<Response> {
		const parser = new JSONParser({ paths: ["$.users.*"] });

		const users: string[] = [];

		// Pipe the request body through the JSON parser
		const reader = request.body
			.pipeThrough(parser)
			.getReader();

		// Process matching JSON values as they stream in
		while (true) {
			const { done, value } = await reader.read();
			if (done) break;
			// Extract only the name field from each user object
			if (value.value?.name) {
				users.push(value.value.name);
			}
		}

		return Response.json({ userNames: users });
	},
} satisfies ExportedHandler;

import { JSONParser } from "@streamparser/json-whatwg";

export default {
	async fetch(request) {
		const parser = new JSONParser({ paths: ["$.users.*"] });

		const users = [];

		// Pipe the request body through the JSON parser
		const reader = request.body
			.pipeThrough(parser)
			.getReader();

		// Process matching JSON values as they stream in
		while (true) {
			const { done, value } = await reader.read();
			if (done) break;
			// Extract only the name field from each user object
			if (value.value?.name) {
				users.push(value.value.name);
			}
		}

		return Response.json({ userNames: users });
	},
};

流式传输并转换 JSON 响应

此示例从上游 API 获取大型 JSON 响应，转换特定字段，并将修改后的响应流式传输给客户端。

import { JSONParser } from "@streamparser/json-whatwg";

export default {
	async fetch(request): Promise<Response> {
		const response = await fetch("https://api.example.com/large-dataset.json");

		const parser = new JSONParser({ paths: ["$.items.*"] });

		const { readable, writable } = new TransformStream();
		const writer = writable.getWriter();
		const encoder = new TextEncoder();

		// Process the upstream response in the background
		(async () => {
			const reader = response.body
				.pipeThrough(parser)
				.getReader();

			await writer.write(encoder.encode('{"processedItems":['));
			let first = true;

			while (true) {
				const { done, value } = await reader.read();
				if (done) break;

				// Transform each item as it streams through
				const item = value.value;
				const transformed = {
					id: item.id,
					title: item.title.toUpperCase(),
					processed: true,
				};

				if (!first) await writer.write(encoder.encode(","));
				first = false;
				await writer.write(encoder.encode(JSON.stringify(transformed)));
			}

			await writer.write(encoder.encode("]}"));
			await writer.close();
		})();

		return new Response(readable, {
			headers: { "Content-Type": "application/json" },
		});
	},
} satisfies ExportedHandler;

import { JSONParser } from "@streamparser/json-whatwg";

export default {
	async fetch(request) {
		const response = await fetch("https://api.example.com/large-dataset.json");

		const parser = new JSONParser({ paths: ["$.items.*"] });

		const { readable, writable } = new TransformStream();
		const writer = writable.getWriter();
		const encoder = new TextEncoder();

		// Process the upstream response in the background
		(async () => {
			const reader = response.body
				.pipeThrough(parser)
				.getReader();

			await writer.write(encoder.encode('{"processedItems":['));
			let first = true;

			while (true) {
				const { done, value } = await reader.read();
				if (done) break;

				// Transform each item as it streams through
				const item = value.value;
				const transformed = {
					id: item.id,
					title: item.title.toUpperCase(),
					processed: true,
				};

				if (!first) await writer.write(encoder.encode(","));
				first = false;
				await writer.write(encoder.encode(JSON.stringify(transformed)));
			}

			await writer.write(encoder.encode("]}"));
			await writer.close();
		})();

		return new Response(readable, {
			headers: { "Content-Type": "application/json" },
		});
	},
};

流式传输大型 JSON

流式传输 JSON 请求体

流式传输并转换 JSON 响应

相关资源