crawler.test.ts•2.83 kB
import { assert, beforeEach, describe, expect, inject, it } from "vitest";
import { createKarakeepClient } from "@karakeep/sdk";
import { createTestUser } from "../../utils/api";
import { waitUntil } from "../../utils/general";
describe("Crawler Tests", () => {
const port = inject("karakeepPort");
if (!port) {
throw new Error("Missing required environment variables");
}
let client: ReturnType<typeof createKarakeepClient>;
let apiKey: string;
async function getBookmark(bookmarkId: string) {
const { data } = await client.GET(`/bookmarks/{bookmarkId}`, {
params: {
path: {
bookmarkId,
},
query: {
includeContent: true,
},
},
});
return data;
}
beforeEach(async () => {
apiKey = await createTestUser();
client = createKarakeepClient({
baseUrl: `http://localhost:${port}/api/v1/`,
headers: {
"Content-Type": "application/json",
authorization: `Bearer ${apiKey}`,
},
});
});
it("should crawl a website", async () => {
let { data: bookmark } = await client.POST("/bookmarks", {
body: {
type: "link",
url: "http://nginx:80/hello.html",
},
});
assert(bookmark);
await waitUntil(
async () => {
const data = await getBookmark(bookmark!.id);
assert(data);
assert(data.content.type === "link");
return data.content.crawledAt !== null;
},
"Bookmark is crawled",
20000,
);
bookmark = await getBookmark(bookmark.id);
assert(bookmark && bookmark.content.type === "link");
expect(bookmark.content.crawledAt).toBeDefined();
expect(bookmark.content.htmlContent).toContain("Hello World");
expect(bookmark.content.title).toContain("My test title");
expect(bookmark.content.url).toBe("http://nginx:80/hello.html");
expect(
bookmark.assets.find((a) => a.assetType === "screenshot"),
).toBeDefined();
});
it("image lings jobs be converted into images", async () => {
let { data: bookmark } = await client.POST("/bookmarks", {
body: {
type: "link",
url: "http://nginx:80/image.png",
},
});
assert(bookmark);
await waitUntil(
async () => {
const data = await getBookmark(bookmark!.id);
assert(data);
return data.content.type === "asset";
},
"Bookmark is crawled and converted to an image",
20000,
);
bookmark = await getBookmark(bookmark.id);
assert(bookmark && bookmark.content.type === "asset");
expect(bookmark.content.assetType).toBe("image");
expect(bookmark.content.assetId).toBeDefined();
expect(bookmark.content.fileName).toBe("image.png");
expect(bookmark.content.sourceUrl).toBe("http://nginx:80/image.png");
});
});