Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 25 additions & 10 deletions AwfulCore/Sources/AwfulCore/Scraping/Helpers.swift
Original file line number Diff line number Diff line change
Expand Up @@ -108,20 +108,35 @@ func scrapeCustomTitle(_ html: HTMLNode) -> RawHTML? {
}


func scrapePageDropdown(_ node: HTMLNode) -> (pageNumber: Int?, pageCount: Int?) {
struct PageNavigationData {
let currentPage: Int
let totalPages: Int
let baseURL: String?
let perPage: Int?
}

func scrapePageNavigationData(_ node: HTMLNode) -> PageNavigationData? {
guard let pageDiv = node.firstNode(matchingSelector: "div.pages") else {
return (nil, nil)
return nil
}
let pageSelect = pageDiv.firstNode(matchingSelector: "select")
let pageCount = pageSelect?.childElementNodes.count ?? 1

let pageNumber = pageSelect
.flatMap { $0.firstNode(matchingSelector: "option[selected]") }
.flatMap { $0["value"] }
.flatMap { Int($0) }
?? 1
if let currentPageStr = pageDiv["data-current-page"],
let totalPagesStr = pageDiv["data-total-pages"],
let currentPage = Int(currentPageStr),
let totalPages = Int(totalPagesStr) {

let baseURL = pageDiv["data-base-url"]
let perPage = pageDiv["data-per-page"].flatMap { Int($0) }

return PageNavigationData(
currentPage: currentPage,
totalPages: totalPages,
baseURL: baseURL,
perPage: perPage
)
}

return (pageNumber: pageNumber, pageCount: pageCount)
return nil
}

enum ForumGroupID: String {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,9 @@ public struct PostsPageScrapeResult: ScrapeResult {

isSingleUserFilterEnabled = body.firstNode(matchingSelector: "table.post a.user_jump[title *= 'Remove']") != nil

(pageNumber: pageNumber, pageCount: pageCount) = scrapePageDropdown(body)
let pageNavData = scrapePageNavigationData(body)
pageNumber = pageNavData?.currentPage
pageCount = pageNavData?.totalPages

let posts = try body
.nodes(matchingSelector: "table.post")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,9 @@ public struct ThreadListScrapeResult: ScrapeResult {

isBookmarkedThreadsPage = body.firstNode(matchingSelector: "form[name='bookmarks']") != nil

(pageNumber: pageNumber, pageCount: pageCount) = scrapePageDropdown(body)
let pageNavData = scrapePageNavigationData(body)
pageNumber = pageNavData?.currentPage
pageCount = pageNavData?.totalPages
}
}

Expand Down
Loading
Loading