feat: implement access now crawling configuration

README.md

@@ -71,6 +71,10 @@ Options:
   -h, --help                display help for command
 ```
 
+##### Notes
+
+- Avoid running multiple crawl operations at the same time, as currently encountering database lock triggers an exception which ends the crawling (could be improved)
+
 #### Serve - Starting the API server
 
 ```

src/config/sources/an.ts

@@ -0,0 +1,59 @@
+import type { SourceConfig } from "@/core/types";
+
+export const anSource: SourceConfig = {
+	id: "an",
+	name: "Access Now",
+	type: "listing",
+	listing: {
+		url: "https://www.accessnow.org/news-updates/?_language=english",
+		pagination: {
+			next_button_selector: ".post-grid-pagination .facetwp-page.next",
+			delaySec: 10, // Access Now blocks IP address when it recognizes a crawling operation
+		},
+		container_selector: ".post-grid.facetwp-template .post-grid-item",
+		shouldExcludeItem: (containerHtml, values) => {
+			const excludedPaths = [
+				"accessnow.org/press-release",
+				"accessnow.org/guide",
+			];
+			return (
+				containerHtml.includes("post-grid-item--external-icon") ||
+				!!excludedPaths.filter((path) => values?.url?.includes(path)).length
+			);
+		},
+		fields: {
+			title: {
+				selector: ".post-grid-item--title",
+				attribute: "text",
+			},
+			url: {
+				selector: ".post-grid-item--link",
+				attribute: "href",
+			},
+			date: {
+				selector: ".post-grid-item--date",
+				attribute: "text",
+			},
+		},
+	},
+	content: {
+		container_selector: "#post-container",
+		fields: {
+			title: {
+				selector: "header h1",
+				attribute: "text",
+				optional: true,
+			},
+			content: {
+				selector: ".entry-content",
+				attribute: "node",
+			},
+			author: {
+				selector: "#authors",
+				exclude_selectors: [".profilePic", ".authorInfo > a"],
+				attribute: "text",
+				optional: true,
+			},
+		},
+	},
+};

src/config/sources/duk.ts

@@ -10,21 +10,19 @@ export const dukSource: SourceConfig = {
 		pagination: {
 			next_button_selector: ".wp-pagenavi .nextpostslink",
 		},
-		items: {
-			container_selector: ".blog-with-tags.ls-archive-blog .et_pb_post",
-			fields: {
-				title: {
-					selector: ".entry-title",
-					attribute: "text",
-				},
-				url: {
-					selector: ".entry-title a",
-					attribute: "href",
-				},
-				date: {
-					selector: ".post-meta .published",
-					attribute: "text",
-				},
+		container_selector: ".blog-with-tags.ls-archive-blog .et_pb_post",
+		fields: {
+			title: {
+				selector: ".entry-title",
+				attribute: "text",
+			},
+			url: {
+				selector: ".entry-title a",
+				attribute: "href",
+			},
+			date: {
+				selector: ".post-meta .published",
+				attribute: "text",
 			},
 		},
 	},

src/config/sources/eff.ts

@@ -4,42 +4,39 @@ export const effSource: SourceConfig = {
 	id: "eff",
 	name: "Electronic Frontier Foundation",
 	type: "listing",
-	content_url_excludes: [
-		"eff.org/event/",
-		"eff.org/wp/",
-		"eff.org/cases/",
-		"eff.org/calendar/",
-	],
 	listing: {
 		url: "https://eff.org/updates",
 		pagination: {
 			next_button_selector: ".pager__item.pager__item--next a",
 		},
-		items: {
-			container_selector: ".views-row article.node",
-			fields: {
-				title: {
-					selector: ".node__title",
-					attribute: "text",
-				},
-				url: {
-					selector: ".node__title a",
-					attribute: "href",
-				},
-				date: {
-					selector: ".node-date",
-					attribute: "text",
-				},
-				excerpt: {
-					selector: ".node__content",
-					attribute: "text",
-					optional: true,
-				},
-				author: {
-					selector: ".node-author",
-					attribute: "text",
-					optional: true,
-				},
+		container_selector: ".views-row article.node",
+		shouldExcludeItem: (_, values) => {
+			const excludedPaths = [
+				"eff.org/event/",
+				"eff.org/wp/",
+				"eff.org/cases/",
+				"eff.org/calendar/",
+			];
+			return !!excludedPaths.filter((path) => values?.url?.includes(path))
+				.length;
+		},
+		fields: {
+			title: {
+				selector: ".node__title",
+				attribute: "text",
+			},
+			url: {
+				selector: ".node__title a",
+				attribute: "href",
+			},
+			date: {
+				selector: ".node-date",
+				attribute: "text",
+			},
+			author: {
+				selector: ".node-author",
+				attribute: "text",
+				optional: true,
 			},
 		},
 	},

src/config/sources/fpf.ts

@@ -9,31 +9,24 @@ export const fpfSource: SourceConfig = {
 		pagination: {
 			next_button_selector: ".pagination .pagination-link:nth-of-type(2)",
 		},
-		items: {
-			container_selector: ".article-list .card-listing",
-			fields: {
-				title: {
-					selector: ".heading .card-link",
-					attribute: "text",
-				},
-				url: {
-					selector: ".heading .card-link",
-					attribute: "href",
-				},
-				date: {
-					selector: ".meta-info time",
-					attribute: "datetime",
-				},
-				excerpt: {
-					selector: ".inner-content p",
-					attribute: "text",
-					optional: true,
-				},
-				author: {
-					selector: ".meta-info .card-meta-link:not(:nth-child(1))",
-					attribute: "text",
-					optional: true,
-				},
+		container_selector: ".article-list .card-listing",
+		fields: {
+			title: {
+				selector: ".heading .card-link",
+				attribute: "text",
+			},
+			url: {
+				selector: ".heading .card-link",
+				attribute: "href",
+			},
+			date: {
+				selector: ".meta-info time",
+				attribute: "datetime",
+			},
+			author: {
+				selector: ".meta-info .card-meta-link:not(:nth-child(1))",
+				attribute: "text",
+				optional: true,
 			},
 		},
 	},

src/config/sources/index.ts

@@ -1,4 +1,5 @@
 import type { SourceConfig } from "@/core/types";
+import { anSource } from "./an.js";
 import { dukSource } from "./duk.js";
 import { effSource } from "./eff.js";
 import { fpfSource } from "./fpf.js";
@@ -13,4 +14,5 @@ export const sources: SourceConfig[] = [
 	p2pSource,
 	dukSource,
 	tfSource,
+	anSource,
 ];

src/config/sources/lpe.ts

@@ -9,26 +9,19 @@ export const lpeSource: SourceConfig = {
 		pagination: {
 			next_button_selector: "",
 		},
-		items: {
-			container_selector: ".section .post-card",
-			fields: {
-				title: {
-					selector: ".post-card__title",
-					attribute: "text",
-				},
-				url: {
-					selector: ".post-card__title",
-					attribute: "href",
-				},
-				date: {
-					selector: ".post-card__label span:nth-of-type(2)",
-					attribute: "text",
-				},
-				excerpt: {
-					selector: ".post-card__content > span",
-					attribute: "text",
-					optional: true,
-				},
+		container_selector: ".section .post-card",
+		fields: {
+			title: {
+				selector: ".post-card__title",
+				attribute: "text",
+			},
+			url: {
+				selector: ".post-card__title",
+				attribute: "href",
+			},
+			date: {
+				selector: ".post-card__label span:nth-of-type(2)",
+				attribute: "text",
 			},
 		},
 	},

src/config/sources/p2p.ts

@@ -10,26 +10,19 @@ export const p2pSource: SourceConfig = {
 		pagination: {
 			next_button_selector: ".nav-previous a",
 		},
-		items: {
-			container_selector: ".blog-masonry article",
-			fields: {
-				title: {
-					selector: ".entry-title",
-					attribute: "text",
-				},
-				url: {
-					selector: ".entry-title a",
-					attribute: "href",
-				},
-				date: {
-					selector: ".entry-date",
-					attribute: "text",
-				},
-				excerpt: {
-					selector: ".entry-content",
-					attribute: "text",
-					optional: true,
-				},
+		container_selector: ".blog-masonry article",
+		fields: {
+			title: {
+				selector: ".entry-title",
+				attribute: "text",
+			},
+			url: {
+				selector: ".entry-title a",
+				attribute: "href",
+			},
+			date: {
+				selector: ".entry-date",
+				attribute: "text",
 			},
 		},
 	},

src/config/sources/tf.ts

@@ -9,21 +9,19 @@ export const tfSource: SourceConfig = {
 		pagination: {
 			next_button_selector: ".page__navigation .navigation__link.next",
 		},
-		items: {
-			container_selector: ".page__content .preview-article",
-			fields: {
-				title: {
-					selector: ".preview-article__title",
-					attribute: "text",
-				},
-				url: {
-					selector: "& > a",
-					attribute: "href",
-				},
-				date: {
-					selector: ".preview-article__published time",
-					attribute: "text",
-				},
+		container_selector: ".page__content .preview-article",
+		fields: {
+			title: {
+				selector: ".preview-article__title",
+				attribute: "text",
+			},
+			url: {
+				selector: "& > a",
+				attribute: "href",
+			},
+			date: {
+				selector: ".preview-article__published time",
+				attribute: "text",
 			},
 		},
 	},