diff --git a/src/config/sources/an.ts b/src/config/sources/access_now.ts similarity index 95% rename from src/config/sources/an.ts rename to src/config/sources/access_now.ts index f03a055..1b8fca5 100644 --- a/src/config/sources/an.ts +++ b/src/config/sources/access_now.ts @@ -1,7 +1,7 @@ import type { SourceConfig } from "@/core/types"; -export const anSource: SourceConfig = { - id: "an", +export const AccessNowSource: SourceConfig = { + id: "access_now", name: "Access Now", type: "listing", listing: { diff --git a/src/config/sources/duk.ts b/src/config/sources/declassified_uk.ts similarity index 93% rename from src/config/sources/duk.ts rename to src/config/sources/declassified_uk.ts index 798b962..f11f34d 100644 --- a/src/config/sources/duk.ts +++ b/src/config/sources/declassified_uk.ts @@ -1,7 +1,7 @@ import type { SourceConfig } from "@/core/types"; -export const dukSource: SourceConfig = { - id: "duk", +export const DeclassifiedUkSource: SourceConfig = { + id: "declassified_uk", name: "Declassified UK", type: "listing", disableJavascript: true, diff --git a/src/config/sources/eff.ts b/src/config/sources/electronic_frontier_foundation.ts similarity index 91% rename from src/config/sources/eff.ts rename to src/config/sources/electronic_frontier_foundation.ts index b554ebe..303943c 100644 --- a/src/config/sources/eff.ts +++ b/src/config/sources/electronic_frontier_foundation.ts @@ -1,7 +1,7 @@ import type { SourceConfig } from "@/core/types"; -export const effSource: SourceConfig = { - id: "eff", +export const ElectronicFrontierFoundationSource: SourceConfig = { + id: "electronic_frontier_foundation", name: "Electronic Frontier Foundation", type: "listing", listing: { diff --git a/src/config/sources/fpf.ts b/src/config/sources/freedom_press_foundation.ts similarity index 91% rename from src/config/sources/fpf.ts rename to src/config/sources/freedom_press_foundation.ts index 7759f56..8817962 100644 --- a/src/config/sources/fpf.ts +++ b/src/config/sources/freedom_press_foundation.ts @@ -1,7 +1,7 @@ import type { SourceConfig } from "@/core/types"; -export const fpfSource: SourceConfig = { - id: "fpf", +export const FreedomPressFoundationSource: SourceConfig = { + id: "freedom_press_foundation", name: "Freedom of the Press Foundation", type: "listing", listing: { diff --git a/src/config/sources/index.ts b/src/config/sources/index.ts index 1b0a81f..29caffe 100644 --- a/src/config/sources/index.ts +++ b/src/config/sources/index.ts @@ -1,18 +1,18 @@ import type { SourceConfig } from "@/core/types"; -import { anSource } from "./an.js"; -import { dukSource } from "./duk.js"; -import { effSource } from "./eff.js"; -import { fpfSource } from "./fpf.js"; -import { lpeSource } from "./lpe.js"; -import { p2pSource } from "./p2p.js"; -import { tfSource } from "./tf.js"; +import { AccessNowSource } from "./access_now.js"; +import { DeclassifiedUkSource } from "./declassified_uk.js"; +import { ElectronicFrontierFoundationSource } from "./electronic_frontier_foundation.js"; +import { FreedomPressFoundationSource } from "./freedom_press_foundation.js"; +import { LogosPressEngineSource } from "./logos_press_engine.js"; +import { P2pFoundationSource } from "./p2p_foundation.js"; +import { TorrentFreakSource } from "./torrent_freak.js"; export const sources: SourceConfig[] = [ - effSource, - fpfSource, - lpeSource, - p2pSource, - dukSource, - tfSource, - anSource, + ElectronicFrontierFoundationSource, + FreedomPressFoundationSource, + LogosPressEngineSource, + P2pFoundationSource, + DeclassifiedUkSource, + TorrentFreakSource, + AccessNowSource, ]; diff --git a/src/config/sources/lpe.ts b/src/config/sources/logos_press_engine.ts similarity index 92% rename from src/config/sources/lpe.ts rename to src/config/sources/logos_press_engine.ts index c15025b..8918c55 100644 --- a/src/config/sources/lpe.ts +++ b/src/config/sources/logos_press_engine.ts @@ -1,7 +1,7 @@ import type { SourceConfig } from "@/core/types"; -export const lpeSource: SourceConfig = { - id: "lpe", +export const LogosPressEngineSource: SourceConfig = { + id: "logos_press_engine", name: "Logos Press Engine", type: "listing", listing: { diff --git a/src/config/sources/p2p.ts b/src/config/sources/p2p_foundation.ts similarity index 92% rename from src/config/sources/p2p.ts rename to src/config/sources/p2p_foundation.ts index 9c8c6a6..fb7523a 100644 --- a/src/config/sources/p2p.ts +++ b/src/config/sources/p2p_foundation.ts @@ -1,7 +1,7 @@ import type { SourceConfig } from "@/core/types"; -export const p2pSource: SourceConfig = { - id: "p2p", +export const P2pFoundationSource: SourceConfig = { + id: "p2p_foundation", name: "P2P Foundation", type: "listing", disableJavascript: true, diff --git a/src/config/sources/tf.ts b/src/config/sources/torrent_freak.ts similarity index 92% rename from src/config/sources/tf.ts rename to src/config/sources/torrent_freak.ts index f6c67d3..cad20aa 100644 --- a/src/config/sources/tf.ts +++ b/src/config/sources/torrent_freak.ts @@ -1,7 +1,7 @@ import type { SourceConfig } from "@/core/types"; -export const tfSource: SourceConfig = { - id: "tf", +export const TorrentFreakSource: SourceConfig = { + id: "torrent_freak", name: "TorrentFreak", type: "listing", listing: { diff --git a/src/tests/__fixtures__/an/biden-digital-rights.ts b/src/tests/__fixtures__/access_now/biden-digital-rights.ts similarity index 100% rename from src/tests/__fixtures__/an/biden-digital-rights.ts rename to src/tests/__fixtures__/access_now/biden-digital-rights.ts diff --git a/src/tests/__fixtures__/an/kenya-sim-card-biometrics.ts b/src/tests/__fixtures__/access_now/kenya-sim-card-biometrics.ts similarity index 100% rename from src/tests/__fixtures__/an/kenya-sim-card-biometrics.ts rename to src/tests/__fixtures__/access_now/kenya-sim-card-biometrics.ts diff --git a/src/tests/__fixtures__/an/russias-record-war-on-connectivity.ts b/src/tests/__fixtures__/access_now/russias-record-war-on-connectivity.ts similarity index 100% rename from src/tests/__fixtures__/an/russias-record-war-on-connectivity.ts rename to src/tests/__fixtures__/access_now/russias-record-war-on-connectivity.ts diff --git a/src/tests/__fixtures__/an/vodafone-challenged-release-transparency-report.ts b/src/tests/__fixtures__/access_now/vodafone-challenged-release-transparency-report.ts similarity index 100% rename from src/tests/__fixtures__/an/vodafone-challenged-release-transparency-report.ts rename to src/tests/__fixtures__/access_now/vodafone-challenged-release-transparency-report.ts diff --git a/src/tests/__fixtures__/duk/genocide-questions-avoided-as-starmer-meets-israeli-president.ts b/src/tests/__fixtures__/declassified_uk/genocide-questions-avoided-as-starmer-meets-israeli-president.ts similarity index 100% rename from src/tests/__fixtures__/duk/genocide-questions-avoided-as-starmer-meets-israeli-president.ts rename to src/tests/__fixtures__/declassified_uk/genocide-questions-avoided-as-starmer-meets-israeli-president.ts diff --git a/src/tests/__fixtures__/duk/how-the-uk-security-services-neutralised-the-countrys-leading-liberal-newspaper.ts b/src/tests/__fixtures__/declassified_uk/how-the-uk-security-services-neutralised-the-countrys-leading-liberal-newspaper.ts similarity index 100% rename from src/tests/__fixtures__/duk/how-the-uk-security-services-neutralised-the-countrys-leading-liberal-newspaper.ts rename to src/tests/__fixtures__/declassified_uk/how-the-uk-security-services-neutralised-the-countrys-leading-liberal-newspaper.ts diff --git a/src/tests/__fixtures__/duk/maersk-the-shipping-company-transporting-arms-to-israel.ts b/src/tests/__fixtures__/declassified_uk/maersk-the-shipping-company-transporting-arms-to-israel.ts similarity index 100% rename from src/tests/__fixtures__/duk/maersk-the-shipping-company-transporting-arms-to-israel.ts rename to src/tests/__fixtures__/declassified_uk/maersk-the-shipping-company-transporting-arms-to-israel.ts diff --git a/src/tests/__fixtures__/duk/rishi-sunaks-mission-creep-in-yemen.ts b/src/tests/__fixtures__/declassified_uk/rishi-sunaks-mission-creep-in-yemen.ts similarity index 100% rename from src/tests/__fixtures__/duk/rishi-sunaks-mission-creep-in-yemen.ts rename to src/tests/__fixtures__/declassified_uk/rishi-sunaks-mission-creep-in-yemen.ts diff --git a/src/tests/__fixtures__/eff/21-44.ts b/src/tests/__fixtures__/electronics_frontier_foundation/21-44.ts similarity index 100% rename from src/tests/__fixtures__/eff/21-44.ts rename to src/tests/__fixtures__/electronics_frontier_foundation/21-44.ts diff --git a/src/tests/__fixtures__/eff/eff-awards-spotlight-software-freedom-law-center-india.ts b/src/tests/__fixtures__/electronics_frontier_foundation/eff-awards-spotlight-software-freedom-law-center-india.ts similarity index 100% rename from src/tests/__fixtures__/eff/eff-awards-spotlight-software-freedom-law-center-india.ts rename to src/tests/__fixtures__/electronics_frontier_foundation/eff-awards-spotlight-software-freedom-law-center-india.ts diff --git a/src/tests/__fixtures__/eff/eff-commerce-department-we-must-revise-overbroad-export-control-proposal.ts b/src/tests/__fixtures__/electronics_frontier_foundation/eff-commerce-department-we-must-revise-overbroad-export-control-proposal.ts similarity index 100% rename from src/tests/__fixtures__/eff/eff-commerce-department-we-must-revise-overbroad-export-control-proposal.ts rename to src/tests/__fixtures__/electronics_frontier_foundation/eff-commerce-department-we-must-revise-overbroad-export-control-proposal.ts diff --git a/src/tests/__fixtures__/eff/trailblazing-tech-scholar-danah-boyd-groundbreaking-cyberpunk-author-william-gibson.ts b/src/tests/__fixtures__/electronics_frontier_foundation/trailblazing-tech-scholar-danah-boyd-groundbreaking-cyberpunk-author-william-gibson.ts similarity index 100% rename from src/tests/__fixtures__/eff/trailblazing-tech-scholar-danah-boyd-groundbreaking-cyberpunk-author-william-gibson.ts rename to src/tests/__fixtures__/electronics_frontier_foundation/trailblazing-tech-scholar-danah-boyd-groundbreaking-cyberpunk-author-william-gibson.ts diff --git a/src/tests/__fixtures__/eff/wiring-big-brother-machine.ts b/src/tests/__fixtures__/electronics_frontier_foundation/wiring-big-brother-machine.ts similarity index 100% rename from src/tests/__fixtures__/eff/wiring-big-brother-machine.ts rename to src/tests/__fixtures__/electronics_frontier_foundation/wiring-big-brother-machine.ts diff --git a/src/tests/__fixtures__/fpf/a-massive-failure-in-kansas-two-years-since-the-marion-county-record-raid.ts b/src/tests/__fixtures__/freedom_press_foundation/a-massive-failure-in-kansas-two-years-since-the-marion-county-record-raid.ts similarity index 100% rename from src/tests/__fixtures__/fpf/a-massive-failure-in-kansas-two-years-since-the-marion-county-record-raid.ts rename to src/tests/__fixtures__/freedom_press_foundation/a-massive-failure-in-kansas-two-years-since-the-marion-county-record-raid.ts diff --git a/src/tests/__fixtures__/fpf/how-aaron-swartz-fought-for-government-transparency.ts b/src/tests/__fixtures__/freedom_press_foundation/how-aaron-swartz-fought-for-government-transparency.ts similarity index 100% rename from src/tests/__fixtures__/fpf/how-aaron-swartz-fought-for-government-transparency.ts rename to src/tests/__fixtures__/freedom_press_foundation/how-aaron-swartz-fought-for-government-transparency.ts diff --git a/src/tests/__fixtures__/fpf/new-election-blog-catalogs-media-suppression-by-candidates-campaigns.ts b/src/tests/__fixtures__/freedom_press_foundation/new-election-blog-catalogs-media-suppression-by-candidates-campaigns.ts similarity index 100% rename from src/tests/__fixtures__/fpf/new-election-blog-catalogs-media-suppression-by-candidates-campaigns.ts rename to src/tests/__fixtures__/freedom_press_foundation/new-election-blog-catalogs-media-suppression-by-candidates-campaigns.ts diff --git a/src/tests/__fixtures__/fpf/prosecutor-puts-doge-ahead-of-first-amendment.ts b/src/tests/__fixtures__/freedom_press_foundation/prosecutor-puts-doge-ahead-of-first-amendment.ts similarity index 100% rename from src/tests/__fixtures__/fpf/prosecutor-puts-doge-ahead-of-first-amendment.ts rename to src/tests/__fixtures__/freedom_press_foundation/prosecutor-puts-doge-ahead-of-first-amendment.ts diff --git a/src/tests/__fixtures__/lpe/august-2025.ts b/src/tests/__fixtures__/logos_press_engine/august-2025.ts similarity index 100% rename from src/tests/__fixtures__/lpe/august-2025.ts rename to src/tests/__fixtures__/logos_press_engine/august-2025.ts diff --git a/src/tests/__fixtures__/lpe/keycard-manifesto.ts b/src/tests/__fixtures__/logos_press_engine/keycard-manifesto.ts similarity index 100% rename from src/tests/__fixtures__/lpe/keycard-manifesto.ts rename to src/tests/__fixtures__/logos_press_engine/keycard-manifesto.ts diff --git a/src/tests/__fixtures__/lpe/logos-a-declaration-of-independence-in-cyberspace.ts b/src/tests/__fixtures__/logos_press_engine/logos-a-declaration-of-independence-in-cyberspace.ts similarity index 100% rename from src/tests/__fixtures__/lpe/logos-a-declaration-of-independence-in-cyberspace.ts rename to src/tests/__fixtures__/logos_press_engine/logos-a-declaration-of-independence-in-cyberspace.ts diff --git a/src/tests/__fixtures__/p2p/book-of-the-day-abundance-the-future-is-better-than-you-think.ts b/src/tests/__fixtures__/p2p_foundation/book-of-the-day-abundance-the-future-is-better-than-you-think.ts similarity index 100% rename from src/tests/__fixtures__/p2p/book-of-the-day-abundance-the-future-is-better-than-you-think.ts rename to src/tests/__fixtures__/p2p_foundation/book-of-the-day-abundance-the-future-is-better-than-you-think.ts diff --git a/src/tests/__fixtures__/p2p/great-transition-alternative-paths-better-climate-just-future.ts b/src/tests/__fixtures__/p2p_foundation/great-transition-alternative-paths-better-climate-just-future.ts similarity index 100% rename from src/tests/__fixtures__/p2p/great-transition-alternative-paths-better-climate-just-future.ts rename to src/tests/__fixtures__/p2p_foundation/great-transition-alternative-paths-better-climate-just-future.ts diff --git a/src/tests/__fixtures__/p2p/take-back-the-app-a-dialogue-on-platform-cooperativism-free-software-and-discos.ts b/src/tests/__fixtures__/p2p_foundation/take-back-the-app-a-dialogue-on-platform-cooperativism-free-software-and-discos.ts similarity index 100% rename from src/tests/__fixtures__/p2p/take-back-the-app-a-dialogue-on-platform-cooperativism-free-software-and-discos.ts rename to src/tests/__fixtures__/p2p_foundation/take-back-the-app-a-dialogue-on-platform-cooperativism-free-software-and-discos.ts diff --git a/src/tests/__fixtures__/p2p/trusting-google-or-not.ts b/src/tests/__fixtures__/p2p_foundation/trusting-google-or-not.ts similarity index 100% rename from src/tests/__fixtures__/p2p/trusting-google-or-not.ts rename to src/tests/__fixtures__/p2p_foundation/trusting-google-or-not.ts diff --git a/src/tests/__fixtures__/tf/employee-who-leaked-spider-man-blu-ray-sentenced-to-nearly-5-years-in-prison-on-gun-charge.ts b/src/tests/__fixtures__/torrent_freak/employee-who-leaked-spider-man-blu-ray-sentenced-to-nearly-5-years-in-prison-on-gun-charge.ts similarity index 100% rename from src/tests/__fixtures__/tf/employee-who-leaked-spider-man-blu-ray-sentenced-to-nearly-5-years-in-prison-on-gun-charge.ts rename to src/tests/__fixtures__/torrent_freak/employee-who-leaked-spider-man-blu-ray-sentenced-to-nearly-5-years-in-prison-on-gun-charge.ts diff --git a/src/tests/__fixtures__/tf/github-takes-down-pirate-streaming-app-king-club-following-mpa-complaint-200819.ts b/src/tests/__fixtures__/torrent_freak/github-takes-down-pirate-streaming-app-king-club-following-mpa-complaint-200819.ts similarity index 100% rename from src/tests/__fixtures__/tf/github-takes-down-pirate-streaming-app-king-club-following-mpa-complaint-200819.ts rename to src/tests/__fixtures__/torrent_freak/github-takes-down-pirate-streaming-app-king-club-following-mpa-complaint-200819.ts diff --git a/src/tests/__fixtures__/tf/swedish-isp-backs-down-allofmp3com-no-longer-blocked.ts b/src/tests/__fixtures__/torrent_freak/swedish-isp-backs-down-allofmp3com-no-longer-blocked.ts similarity index 100% rename from src/tests/__fixtures__/tf/swedish-isp-backs-down-allofmp3com-no-longer-blocked.ts rename to src/tests/__fixtures__/torrent_freak/swedish-isp-backs-down-allofmp3com-no-longer-blocked.ts diff --git a/src/tests/integration/an-integration.test.ts b/src/tests/integration/access_now-integration.test.ts similarity index 82% rename from src/tests/integration/an-integration.test.ts rename to src/tests/integration/access_now-integration.test.ts index dab7388..1dcf922 100644 --- a/src/tests/integration/an-integration.test.ts +++ b/src/tests/integration/access_now-integration.test.ts @@ -1,14 +1,14 @@ import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; -import { anSource as config } from "@/config/sources/an.js"; +import { AccessNowSource as config } from "@/config/sources/access_now.js"; import { createContentPageExtractor } from "@/crawlers/extractors/ContentPageExtractor"; import { createListingPageExtractor } from "@/crawlers/extractors/ListingPageExtractor"; import type { BrowserHandler } from "@/crawlers/handlers/BrowserHandler"; import { createBrowserHandler } from "@/crawlers/handlers/BrowserHandler"; import { navigateToNextPage } from "@/crawlers/handlers/PaginationHandler"; -import fixture4 from "@/tests/__fixtures__/an/biden-digital-rights"; -import fixture2 from "@/tests/__fixtures__/an/kenya-sim-card-biometrics"; -import fixture1 from "@/tests/__fixtures__/an/russias-record-war-on-connectivity"; -import fixture3 from "@/tests/__fixtures__/an/vodafone-challenged-release-transparency-report"; +import fixture4 from "@/tests/__fixtures__/access_now/biden-digital-rights"; +import fixture2 from "@/tests/__fixtures__/access_now/kenya-sim-card-biometrics"; +import fixture1 from "@/tests/__fixtures__/access_now/russias-record-war-on-connectivity"; +import fixture3 from "@/tests/__fixtures__/access_now/vodafone-challenged-release-transparency-report"; const ifDescribe = process.env.INT_TEST === "true" ? describe : describe.skip; @@ -24,7 +24,7 @@ ifDescribe("Access Now integration tests", () => { await browser.close(); }); - it("should crawl AN listing page", async () => { + it("should crawl Access Now listing page", async () => { const page = await browser.setupNewPage(config.listing.url); const extractor = createListingPageExtractor(); const result = await extractor.extractItemsFromPage(page, config, [], 0); @@ -34,12 +34,12 @@ ifDescribe("Access Now integration tests", () => { expect(result.items.every((item) => !!item.publishedDate)).toBeTruthy(); }); - it("should crawl to next AN listing page", async () => { + it("should crawl to next Access Now listing page", async () => { const page = await browser.setupNewPage(config.listing.url); expect(await navigateToNextPage(page, config)).toBeTruthy(); }); - it("should crawl multiple AN content pages", async () => { + it("should crawl multiple Access Now content pages", async () => { const testCases = [ { url: "https://www.accessnow.org/russias-record-war-on-connectivity/", diff --git a/src/tests/integration/duk-integration.test.ts b/src/tests/integration/declassified_uk-integration.test.ts similarity index 80% rename from src/tests/integration/duk-integration.test.ts rename to src/tests/integration/declassified_uk-integration.test.ts index 8b186b9..b6d2498 100644 --- a/src/tests/integration/duk-integration.test.ts +++ b/src/tests/integration/declassified_uk-integration.test.ts @@ -1,14 +1,14 @@ import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; -import { dukSource as config } from "@/config/sources/duk.js"; +import { DeclassifiedUkSource as config } from "@/config/sources/declassified_uk.js"; import { createContentPageExtractor } from "@/crawlers/extractors/ContentPageExtractor"; import { createListingPageExtractor } from "@/crawlers/extractors/ListingPageExtractor"; import type { BrowserHandler } from "@/crawlers/handlers/BrowserHandler"; import { createBrowserHandler } from "@/crawlers/handlers/BrowserHandler"; import { navigateToNextPage } from "@/crawlers/handlers/PaginationHandler"; -import fixture4 from "@/tests/__fixtures__/duk/genocide-questions-avoided-as-starmer-meets-israeli-president"; -import fixture1 from "@/tests/__fixtures__/duk/how-the-uk-security-services-neutralised-the-countrys-leading-liberal-newspaper"; -import fixture3 from "@/tests/__fixtures__/duk/maersk-the-shipping-company-transporting-arms-to-israel"; -import fixture2 from "@/tests/__fixtures__/duk/rishi-sunaks-mission-creep-in-yemen"; +import fixture4 from "@/tests/__fixtures__/declassified_uk/genocide-questions-avoided-as-starmer-meets-israeli-president"; +import fixture1 from "@/tests/__fixtures__/declassified_uk/how-the-uk-security-services-neutralised-the-countrys-leading-liberal-newspaper"; +import fixture3 from "@/tests/__fixtures__/declassified_uk/maersk-the-shipping-company-transporting-arms-to-israel"; +import fixture2 from "@/tests/__fixtures__/declassified_uk/rishi-sunaks-mission-creep-in-yemen"; const ifDescribe = process.env.INT_TEST === "true" ? describe : describe.skip; @@ -24,7 +24,7 @@ ifDescribe("Declassified UK integration tests", () => { await browser.close(); }); - it("should crawl DUK listing page", async () => { + it("should crawl Declassified UK listing page", async () => { const page = await browser.setupNewPage(config.listing.url); const extractor = createListingPageExtractor(); const result = await extractor.extractItemsFromPage(page, config, [], 0); @@ -34,12 +34,12 @@ ifDescribe("Declassified UK integration tests", () => { expect(result.items.every((item) => !!item.publishedDate)).toBeTruthy(); }); - it("should crawl to next DUK listing page", async () => { + it("should crawl to next Declassified UK listing page", async () => { const page = await browser.setupNewPage(config.listing.url); expect(await navigateToNextPage(page, config)).toBeTruthy(); }); - it("should crawl multiple DUK content pages", async () => { + it("should crawl multiple Declassified UK content pages", async () => { const testCases = [ { url: "https://www.declassifieduk.org/how-the-uk-security-services-neutralised-the-countrys-leading-liberal-newspaper/", diff --git a/src/tests/integration/eff-integration.test.ts b/src/tests/integration/electronics_frontier_foundation-integration.test.ts similarity index 74% rename from src/tests/integration/eff-integration.test.ts rename to src/tests/integration/electronics_frontier_foundation-integration.test.ts index 3bd2801..52e447d 100644 --- a/src/tests/integration/eff-integration.test.ts +++ b/src/tests/integration/electronics_frontier_foundation-integration.test.ts @@ -1,19 +1,19 @@ import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; -import { effSource as config } from "@/config/sources/eff.js"; +import { ElectronicFrontierFoundationSource as config } from "@/config/sources/electronic_frontier_foundation.js"; import { createContentPageExtractor } from "@/crawlers/extractors/ContentPageExtractor"; import { createListingPageExtractor } from "@/crawlers/extractors/ListingPageExtractor"; import type { BrowserHandler } from "@/crawlers/handlers/BrowserHandler"; import { createBrowserHandler } from "@/crawlers/handlers/BrowserHandler"; import { navigateToNextPage } from "@/crawlers/handlers/PaginationHandler"; -import fixture3 from "@/tests/__fixtures__/eff/21-44"; -import fixture1 from "@/tests/__fixtures__/eff/eff-awards-spotlight-software-freedom-law-center-india"; -import fixture2 from "@/tests/__fixtures__/eff/eff-commerce-department-we-must-revise-overbroad-export-control-proposal"; -import fixture4 from "@/tests/__fixtures__/eff/trailblazing-tech-scholar-danah-boyd-groundbreaking-cyberpunk-author-william-gibson"; -import fixture5 from "@/tests/__fixtures__/eff/wiring-big-brother-machine"; +import fixture3 from "@/tests/__fixtures__/electronics_frontier_foundation/21-44"; +import fixture1 from "@/tests/__fixtures__/electronics_frontier_foundation/eff-awards-spotlight-software-freedom-law-center-india"; +import fixture2 from "@/tests/__fixtures__/electronics_frontier_foundation/eff-commerce-department-we-must-revise-overbroad-export-control-proposal"; +import fixture4 from "@/tests/__fixtures__/electronics_frontier_foundation/trailblazing-tech-scholar-danah-boyd-groundbreaking-cyberpunk-author-william-gibson"; +import fixture5 from "@/tests/__fixtures__/electronics_frontier_foundation/wiring-big-brother-machine"; const ifDescribe = process.env.INT_TEST === "true" ? describe : describe.skip; -ifDescribe("Electronics Foundation integration tests", () => { +ifDescribe("Electronics Frontier Foundation integration tests", () => { let browser: BrowserHandler; vi.setConfig({ testTimeout: 60000 }); @@ -25,7 +25,7 @@ ifDescribe("Electronics Foundation integration tests", () => { await browser.close(); }); - it("should crawl EFF listing page", async () => { + it("should crawl Electronics Frontier Foundation listing page", async () => { const page = await browser.setupNewPage(config.listing.url); const extractor = createListingPageExtractor(); const result = await extractor.extractItemsFromPage(page, config, [], 0); @@ -35,12 +35,12 @@ ifDescribe("Electronics Foundation integration tests", () => { expect(result.items.every((item) => !!item.publishedDate)).toBeTruthy(); }); - it("should crawl to next EFF listing page", async () => { + it("should crawl to next Electronics Frontier Foundation listing page", async () => { const page = await browser.setupNewPage(config.listing.url); expect(await navigateToNextPage(page, config)).toBeTruthy(); }); - it("should crawl multiple EFF content pages", async () => { + it("should crawl multiple Electronics Frontier Foundation content pages", async () => { const testCases = [ { url: "https://www.eff.org/deeplinks/2025/08/eff-awards-spotlight-software-freedom-law-center-india", diff --git a/src/tests/integration/fpf-integration.test.ts b/src/tests/integration/freedom_press_foundation-integration.test.ts similarity index 75% rename from src/tests/integration/fpf-integration.test.ts rename to src/tests/integration/freedom_press_foundation-integration.test.ts index 686a61f..19f61c4 100644 --- a/src/tests/integration/fpf-integration.test.ts +++ b/src/tests/integration/freedom_press_foundation-integration.test.ts @@ -1,18 +1,18 @@ import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; -import { fpfSource as config } from "@/config/sources/fpf.js"; +import { FreedomPressFoundationSource as config } from "@/config/sources/freedom_press_foundation.js"; import { createContentPageExtractor } from "@/crawlers/extractors/ContentPageExtractor"; import { createListingPageExtractor } from "@/crawlers/extractors/ListingPageExtractor"; import type { BrowserHandler } from "@/crawlers/handlers/BrowserHandler"; import { createBrowserHandler } from "@/crawlers/handlers/BrowserHandler"; import { navigateToNextPage } from "@/crawlers/handlers/PaginationHandler"; -import fixture1 from "@/tests/__fixtures__/fpf/a-massive-failure-in-kansas-two-years-since-the-marion-county-record-raid"; -import fixture2 from "@/tests/__fixtures__/fpf/how-aaron-swartz-fought-for-government-transparency"; -import fixture3 from "@/tests/__fixtures__/fpf/new-election-blog-catalogs-media-suppression-by-candidates-campaigns"; -import fixture4 from "@/tests/__fixtures__/fpf/prosecutor-puts-doge-ahead-of-first-amendment"; +import fixture1 from "@/tests/__fixtures__/freedom_press_foundation/a-massive-failure-in-kansas-two-years-since-the-marion-county-record-raid"; +import fixture2 from "@/tests/__fixtures__/freedom_press_foundation/how-aaron-swartz-fought-for-government-transparency"; +import fixture3 from "@/tests/__fixtures__/freedom_press_foundation/new-election-blog-catalogs-media-suppression-by-candidates-campaigns"; +import fixture4 from "@/tests/__fixtures__/freedom_press_foundation/prosecutor-puts-doge-ahead-of-first-amendment"; const ifDescribe = process.env.INT_TEST === "true" ? describe : describe.skip; -ifDescribe("Freedom Press integration tests", () => { +ifDescribe("Freedom Press Foundation integration tests", () => { let browser: BrowserHandler; vi.setConfig({ testTimeout: 60000 }); @@ -24,7 +24,7 @@ ifDescribe("Freedom Press integration tests", () => { await browser.close(); }); - it("should crawl FPF listing page", async () => { + it("should crawl Freedom Press Foundation listing page", async () => { const page = await browser.setupNewPage(config.listing.url); const extractor = createListingPageExtractor(); const result = await extractor.extractItemsFromPage(page, config, [], 0); @@ -34,12 +34,12 @@ ifDescribe("Freedom Press integration tests", () => { expect(result.items.every((item) => !!item.publishedDate)).toBeTruthy(); }); - it("should crawl to next FPF listing page", async () => { + it("should crawl to next Freedom Press Foundation listing page", async () => { const page = await browser.setupNewPage(config.listing.url); expect(await navigateToNextPage(page, config)).toBeTruthy(); }); - it("should crawl multiple FPF content pages", async () => { + it("should crawl multiple Freedom Press Foundation content pages", async () => { const testCases = [ { url: "https://freedom.press/issues/a-massive-failure-in-kansas-two-years-since-the-marion-county-record-raid/", diff --git a/src/tests/integration/lpe-integration.test.ts b/src/tests/integration/logos_press_engine-integration.test.ts similarity index 80% rename from src/tests/integration/lpe-integration.test.ts rename to src/tests/integration/logos_press_engine-integration.test.ts index 459fa91..2d5dc81 100644 --- a/src/tests/integration/lpe-integration.test.ts +++ b/src/tests/integration/logos_press_engine-integration.test.ts @@ -1,16 +1,16 @@ import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; -import { lpeSource as config } from "@/config/sources/lpe.js"; +import { LogosPressEngineSource as config } from "@/config/sources/logos_press_engine.js"; import { createContentPageExtractor } from "@/crawlers/extractors/ContentPageExtractor"; import { createListingPageExtractor } from "@/crawlers/extractors/ListingPageExtractor"; import type { BrowserHandler } from "@/crawlers/handlers/BrowserHandler"; import { createBrowserHandler } from "@/crawlers/handlers/BrowserHandler"; -import fixture1 from "@/tests/__fixtures__/lpe/august-2025"; -import fixture3 from "@/tests/__fixtures__/lpe/keycard-manifesto"; -import fixture2 from "@/tests/__fixtures__/lpe/logos-a-declaration-of-independence-in-cyberspace"; +import fixture1 from "@/tests/__fixtures__/logos_press_engine/august-2025"; +import fixture3 from "@/tests/__fixtures__/logos_press_engine/keycard-manifesto"; +import fixture2 from "@/tests/__fixtures__/logos_press_engine/logos-a-declaration-of-independence-in-cyberspace"; const ifDescribe = process.env.INT_TEST === "true" ? describe : describe.skip; -ifDescribe("Logos integration tests", () => { +ifDescribe("Logos Press Engine integration tests", () => { let browser: BrowserHandler; vi.setConfig({ testTimeout: 60000 }); @@ -22,7 +22,7 @@ ifDescribe("Logos integration tests", () => { await browser.close(); }); - it("should crawl LPE listing page", async () => { + it("should crawl Logos Press Engine listing page", async () => { const page = await browser.setupNewPage(config.listing.url); const extractor = createListingPageExtractor(); const result = await extractor.extractItemsFromPage(page, config, [], 0); @@ -34,12 +34,12 @@ ifDescribe("Logos integration tests", () => { }); // Logos Press Engine has only 1 page for now - /* it("should crawl to next LPE listing page", async () => { + /* it("should crawl to next Logos Press Engine listing page", async () => { const page = await setupPage(browser, config.listing.url); expect(await navigateToNextPage(page, config)).toBeTruthy(); }); */ - it("should crawl multiple LPE content pages", async () => { + it("should crawl multiple Logos Press Engine content pages", async () => { const testCases = [ { url: "https://press.logos.co/article/august-2025", diff --git a/src/tests/integration/p2p-integration.test.ts b/src/tests/integration/p2p_foundation-integration.test.ts similarity index 80% rename from src/tests/integration/p2p-integration.test.ts rename to src/tests/integration/p2p_foundation-integration.test.ts index 766755c..e5b1ed9 100644 --- a/src/tests/integration/p2p-integration.test.ts +++ b/src/tests/integration/p2p_foundation-integration.test.ts @@ -1,14 +1,14 @@ import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; -import { p2pSource as config } from "@/config/sources/p2p.js"; +import { P2pFoundationSource as config } from "@/config/sources/p2p_foundation.js"; import { createContentPageExtractor } from "@/crawlers/extractors/ContentPageExtractor"; import { createListingPageExtractor } from "@/crawlers/extractors/ListingPageExtractor"; import type { BrowserHandler } from "@/crawlers/handlers/BrowserHandler"; import { createBrowserHandler } from "@/crawlers/handlers/BrowserHandler"; import { navigateToNextPage } from "@/crawlers/handlers/PaginationHandler"; -import fixture2 from "@/tests/__fixtures__/p2p/book-of-the-day-abundance-the-future-is-better-than-you-think"; -import fixture3 from "@/tests/__fixtures__/p2p/great-transition-alternative-paths-better-climate-just-future"; -import fixture4 from "@/tests/__fixtures__/p2p/take-back-the-app-a-dialogue-on-platform-cooperativism-free-software-and-discos"; -import fixture1 from "@/tests/__fixtures__/p2p/trusting-google-or-not"; +import fixture2 from "@/tests/__fixtures__/p2p_foundation/book-of-the-day-abundance-the-future-is-better-than-you-think"; +import fixture3 from "@/tests/__fixtures__/p2p_foundation/great-transition-alternative-paths-better-climate-just-future"; +import fixture4 from "@/tests/__fixtures__/p2p_foundation/take-back-the-app-a-dialogue-on-platform-cooperativism-free-software-and-discos"; +import fixture1 from "@/tests/__fixtures__/p2p_foundation/trusting-google-or-not"; const ifDescribe = process.env.INT_TEST === "true" ? describe : describe.skip; @@ -24,7 +24,7 @@ ifDescribe("P2P Foundation integration tests", () => { await browser.close(); }); - it("should crawl P2P listing page", async () => { + it("should crawl P2P Foundation listing page", async () => { const page = await browser.setupNewPage(config.listing.url); const extractor = createListingPageExtractor(); const result = await extractor.extractItemsFromPage(page, config, [], 0); @@ -34,12 +34,12 @@ ifDescribe("P2P Foundation integration tests", () => { expect(result.items.every((item) => !!item.publishedDate)).toBeTruthy(); }); - it("should crawl to next P2P listing page", async () => { + it("should crawl to next P2P Foundation listing page", async () => { const page = await browser.setupNewPage(config.listing.url); expect(await navigateToNextPage(page, config)).toBeTruthy(); }); - it("should crawl multiple P2P content pages", async () => { + it("should crawl multiple P2P Foundation content pages", async () => { const testCases = [ { url: "https://blog.p2pfoundation.net/trusting-google-or-not/", diff --git a/src/tests/integration/tf-integration.test.ts b/src/tests/integration/torrent_freak-integration.test.ts similarity index 79% rename from src/tests/integration/tf-integration.test.ts rename to src/tests/integration/torrent_freak-integration.test.ts index 314d5df..17b0681 100644 --- a/src/tests/integration/tf-integration.test.ts +++ b/src/tests/integration/torrent_freak-integration.test.ts @@ -1,17 +1,17 @@ import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; -import { tfSource as config } from "@/config/sources/tf.js"; +import { TorrentFreakSource as config } from "@/config/sources/torrent_freak.js"; import { createContentPageExtractor } from "@/crawlers/extractors/ContentPageExtractor"; import { createListingPageExtractor } from "@/crawlers/extractors/ListingPageExtractor"; import type { BrowserHandler } from "@/crawlers/handlers/BrowserHandler"; import { createBrowserHandler } from "@/crawlers/handlers/BrowserHandler"; import { navigateToNextPage } from "@/crawlers/handlers/PaginationHandler"; -import fixture3 from "@/tests/__fixtures__/tf/employee-who-leaked-spider-man-blu-ray-sentenced-to-nearly-5-years-in-prison-on-gun-charge"; -import fixture2 from "@/tests/__fixtures__/tf/github-takes-down-pirate-streaming-app-king-club-following-mpa-complaint-200819"; -import fixture1 from "@/tests/__fixtures__/tf/swedish-isp-backs-down-allofmp3com-no-longer-blocked"; +import fixture3 from "@/tests/__fixtures__/torrent_freak/employee-who-leaked-spider-man-blu-ray-sentenced-to-nearly-5-years-in-prison-on-gun-charge"; +import fixture2 from "@/tests/__fixtures__/torrent_freak/github-takes-down-pirate-streaming-app-king-club-following-mpa-complaint-200819"; +import fixture1 from "@/tests/__fixtures__/torrent_freak/swedish-isp-backs-down-allofmp3com-no-longer-blocked"; const ifDescribe = process.env.INT_TEST === "true" ? describe : describe.skip; -ifDescribe("TorrentFreak integration tests", () => { +ifDescribe("Torrent Freak integration tests", () => { let browser: BrowserHandler; vi.setConfig({ testTimeout: 60000 }); @@ -23,7 +23,7 @@ ifDescribe("TorrentFreak integration tests", () => { await browser.close(); }); - it("should crawl TF listing page", async () => { + it("should crawl Torrent Freak listing page", async () => { const page = await browser.setupNewPage(config.listing.url); const extractor = createListingPageExtractor(); const result = await extractor.extractItemsFromPage(page, config, [], 0); @@ -33,12 +33,12 @@ ifDescribe("TorrentFreak integration tests", () => { expect(result.items.every((item) => !!item.publishedDate)).toBeTruthy(); }); - it("should crawl to next TF listing page", async () => { + it("should crawl to next Torrent Freak listing page", async () => { const page = await browser.setupNewPage(config.listing.url); expect(await navigateToNextPage(page, config)).toBeTruthy(); }); - it("should crawl multiple TF content pages", async () => { + it("should crawl multiple Torrent Freak content pages", async () => { const testCases = [ { url: "https://torrentfreak.com/swedish-isp-backs-down-allofmp3com-no-longer-blocked/",