139 lines
3.6 KiB
Go
139 lines
3.6 KiB
Go
package feed
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"github.com/mmcdole/gofeed"
|
|
"log/slog"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
type lobstersPostResponseJson struct {
|
|
ShortID string `json:"short_id"`
|
|
ShortIDURL string `json:"short_id_url"`
|
|
CreatedAt string `json:"created_at"`
|
|
Title string `json:"title"`
|
|
URL string `json:"url"`
|
|
Score int `json:"score"`
|
|
Flags int `json:"flags"`
|
|
CommentCount int `json:"comment_count"`
|
|
Description string `json:"description"`
|
|
DescriptionPlain string `json:"description_plain"`
|
|
CommentsURL string `json:"comments_url"`
|
|
SubmitterUser string `json:"submitter_user"`
|
|
UserIsAuthor bool `json:"user_is_author"`
|
|
Tags []string `json:"tags"`
|
|
Comments []struct {
|
|
ShortID string `json:"short_id"`
|
|
ShortIDURL string `json:"short_id_url"`
|
|
CreatedAt string `json:"created_at"`
|
|
UpdatedAt string `json:"updated_at"`
|
|
IsDeleted bool `json:"is_deleted"`
|
|
IsModerated bool `json:"is_moderated"`
|
|
Score int `json:"score"`
|
|
Flags int `json:"flags"`
|
|
ParentComment any `json:"parent_comment"`
|
|
Comment string `json:"comment"`
|
|
CommentPlain string `json:"comment_plain"`
|
|
URL string `json:"url"`
|
|
Depth int `json:"depth"`
|
|
CommentingUser string `json:"commenting_user"`
|
|
} `json:"comments"`
|
|
}
|
|
|
|
var lobstersParser = gofeed.NewParser()
|
|
|
|
func getLobstersTopPostIds(feed string) ([]string, error) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
|
defer cancel()
|
|
|
|
request := &RSSFeedRequest{
|
|
Url: feed,
|
|
Title: "Lobsters",
|
|
}
|
|
|
|
f, err := lobstersParser.ParseURLWithContext(request.Url, ctx)
|
|
|
|
if err != nil {
|
|
return nil, fmt.Errorf("%w: could not fetch posts from %s", ErrNoContent, feed)
|
|
}
|
|
|
|
postIds := make([]string, 0, len(f.Items))
|
|
|
|
for i := range f.Items {
|
|
postIds = append(postIds, f.Items[i].GUID)
|
|
}
|
|
|
|
return postIds, nil
|
|
}
|
|
|
|
func getLobstersPostsFromIds(postIds []string) (ForumPosts, error) {
|
|
requests := make([]*http.Request, len(postIds))
|
|
|
|
for i, id := range postIds {
|
|
request, _ := http.NewRequest("GET", id+".json", nil)
|
|
requests[i] = request
|
|
}
|
|
|
|
task := decodeJsonFromRequestTask[lobstersPostResponseJson](defaultClient)
|
|
job := newJob(task, requests).withWorkers(30)
|
|
results, errs, err := workerPoolDo(job)
|
|
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
posts := make(ForumPosts, 0, len(postIds))
|
|
|
|
for i := range results {
|
|
if errs[i] != nil {
|
|
slog.Error("Failed to fetch or parse lobsters post", "error", errs[i], "url", requests[i].URL)
|
|
continue
|
|
}
|
|
|
|
tags := strings.Join(results[i].Tags, ",")
|
|
|
|
if tags != "" {
|
|
tags = " [" + tags + "]"
|
|
}
|
|
|
|
createdAt, _ := time.Parse(time.RFC3339, results[i].CreatedAt)
|
|
|
|
posts = append(posts, ForumPost{
|
|
Title: results[i].Title + tags,
|
|
DiscussionUrl: results[i].CommentsURL,
|
|
TargetUrl: results[i].URL,
|
|
TargetUrlDomain: extractDomainFromUrl(results[i].URL),
|
|
CommentCount: results[i].CommentCount,
|
|
Score: results[i].Score,
|
|
TimePosted: createdAt,
|
|
})
|
|
}
|
|
|
|
if len(posts) == 0 {
|
|
return nil, ErrNoContent
|
|
}
|
|
|
|
if len(posts) != len(postIds) {
|
|
return posts, fmt.Errorf("%w could not fetch some lobsters posts", ErrPartialContent)
|
|
}
|
|
|
|
return posts, nil
|
|
}
|
|
|
|
func FetchLobstersTopPosts(feed string, limit int) (ForumPosts, error) {
|
|
postIds, err := getLobstersTopPostIds(feed)
|
|
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if len(postIds) > limit {
|
|
postIds = postIds[:limit]
|
|
}
|
|
|
|
return getLobstersPostsFromIds(postIds)
|
|
}
|