Merge branch 'release/v0.5.0' into lobsters-widget

internal/feed/changedetection.go

@@ -0,0 +1,139 @@
+package feed
+
+import (
+	"fmt"
+	"log/slog"
+	"net/http"
+	"sort"
+	"strings"
+	"time"
+)
+
+type ChangeDetectionWatch struct {
+	Title        string
+	URL          string
+	LastChanged  time.Time
+	DiffURL      string
+	PreviousHash string
+}
+
+type ChangeDetectionWatches []ChangeDetectionWatch
+
+func (r ChangeDetectionWatches) SortByNewest() ChangeDetectionWatches {
+	sort.Slice(r, func(i, j int) bool {
+		return r[i].LastChanged.After(r[j].LastChanged)
+	})
+
+	return r
+}
+
+type changeDetectionResponseJson struct {
+	Title        string `json:"title"`
+	URL          string `json:"url"`
+	LastChanged  int64  `json:"last_changed"`
+	DateCreated  int64  `json:"date_created"`
+	PreviousHash string `json:"previous_md5"`
+}
+
+func FetchWatchUUIDsFromChangeDetection(instanceURL string, token string) ([]string, error) {
+	request, _ := http.NewRequest("GET", fmt.Sprintf("%s/api/v1/watch", instanceURL), nil)
+
+	if token != "" {
+		request.Header.Add("x-api-key", token)
+	}
+
+	uuidsMap, err := decodeJsonFromRequest[map[string]struct{}](defaultClient, request)
+
+	if err != nil {
+		return nil, fmt.Errorf("could not fetch list of watch UUIDs: %v", err)
+	}
+
+	uuids := make([]string, 0, len(uuidsMap))
+
+	for uuid := range uuidsMap {
+		uuids = append(uuids, uuid)
+	}
+
+	return uuids, nil
+}
+
+func FetchWatchesFromChangeDetection(instanceURL string, requestedWatchIDs []string, token string) (ChangeDetectionWatches, error) {
+	watches := make(ChangeDetectionWatches, 0, len(requestedWatchIDs))
+
+	if len(requestedWatchIDs) == 0 {
+		return watches, nil
+	}
+
+	requests := make([]*http.Request, len(requestedWatchIDs))
+
+	for i, repository := range requestedWatchIDs {
+		request, _ := http.NewRequest("GET", fmt.Sprintf("%s/api/v1/watch/%s", instanceURL, repository), nil)
+
+		if token != "" {
+			request.Header.Add("x-api-key", token)
+		}
+
+		requests[i] = request
+	}
+
+	task := decodeJsonFromRequestTask[changeDetectionResponseJson](defaultClient)
+	job := newJob(task, requests).withWorkers(15)
+	responses, errs, err := workerPoolDo(job)
+
+	if err != nil {
+		return nil, err
+	}
+
+	var failed int
+
+	for i := range responses {
+		if errs[i] != nil {
+			failed++
+			slog.Error("Failed to fetch or parse change detection watch", "error", errs[i], "url", requests[i].URL)
+			continue
+		}
+
+		watchJson := responses[i]
+
+		watch := ChangeDetectionWatch{
+			URL:     watchJson.URL,
+			DiffURL: fmt.Sprintf("%s/diff/%s?from_version=%d", instanceURL, requestedWatchIDs[i], watchJson.LastChanged-1),
+		}
+
+		if watchJson.LastChanged == 0 {
+			watch.LastChanged = time.Unix(watchJson.DateCreated, 0)
+		} else {
+			watch.LastChanged = time.Unix(watchJson.LastChanged, 0)
+		}
+
+		if watchJson.Title != "" {
+			watch.Title = watchJson.Title
+		} else {
+			watch.Title = strings.TrimPrefix(strings.Trim(stripURLScheme(watchJson.URL), "/"), "www.")
+		}
+
+		if watchJson.PreviousHash != "" {
+			var hashLength = 8
+
+			if len(watchJson.PreviousHash) < hashLength {
+				hashLength = len(watchJson.PreviousHash)
+			}
+
+			watch.PreviousHash = watchJson.PreviousHash[0:hashLength]
+		}
+
+		watches = append(watches, watch)
+	}
+
+	if len(watches) == 0 {
+		return nil, ErrNoContent
+	}
+
+	watches.SortByNewest()
+
+	if failed > 0 {
+		return watches, fmt.Errorf("%w: could not get %d watches", ErrPartialContent, failed)
+	}
+
+	return watches, nil
+}

internal/feed/rss.go

@@ -3,8 +3,11 @@ package feed
 import (
 	"context"
 	"fmt"
+	"html"
 	"log/slog"
+	"regexp"
 	"sort"
+	"strings"
 	"time"
 
 	"github.com/mmcdole/gofeed"
@@ -16,12 +19,34 @@ type RSSFeedItem struct {
 	Title       string
 	Link        string
 	ImageURL    string
+	Categories  []string
+	Description string
 	PublishedAt time.Time
 }
 
+// doesn't cover all cases but works the vast majority of the time
+var htmlTagsWithAttributesPattern = regexp.MustCompile(`<\/?[a-zA-Z0-9-]+ *(?:[a-zA-Z-]+=(?:"|').*?(?:"|') ?)* *\/?>`)
+var sequentialWhitespacePattern = regexp.MustCompile(`\s+`)
+
+func sanitizeFeedDescription(description string) string {
+	if description == "" {
+		return ""
+	}
+
+	description = strings.ReplaceAll(description, "\n", " ")
+	description = htmlTagsWithAttributesPattern.ReplaceAllString(description, "")
+	description = sequentialWhitespacePattern.ReplaceAllString(description, " ")
+	description = strings.TrimSpace(description)
+	description = html.UnescapeString(description)
+
+	return description
+}
+
 type RSSFeedRequest struct {
-	Url   string `yaml:"url"`
-	Title string `yaml:"title"`
+	Url             string `yaml:"url"`
+	Title           string `yaml:"title"`
+	HideCategories  bool   `yaml:"hide-categories"`
+	HideDescription bool   `yaml:"hide-description"`
 }
 
 type RSSFeedItems []RSSFeedItem
@@ -57,6 +82,36 @@ func getItemsFromRSSFeedTask(request RSSFeedRequest) ([]RSSFeedItem, error) {
 			Link:       item.Link,
 		}
 
+		if !request.HideDescription && item.Description != "" {
+			description, _ := limitStringLength(item.Description, 1000)
+			description = sanitizeFeedDescription(description)
+			description, limited := limitStringLength(description, 200)
+
+			if limited {
+				description += "…"
+			}
+
+			rssItem.Description = description
+		}
+
+		if !request.HideCategories {
+			var categories = make([]string, 0, 6)
+
+			for _, category := range item.Categories {
+				if len(categories) == 6 {
+					break
+				}
+
+				if len(category) == 0 || len(category) > 30 {
+					continue
+				}
+
+				categories = append(categories, category)
+			}
+
+			rssItem.Categories = categories
+		}
+
 		if request.Title != "" {
 			rssItem.ChannelName = request.Title
 		} else {

internal/feed/twitch.go

@@ -44,6 +44,12 @@ func (channels TwitchChannels) SortByViewers() {
 	})
 }
 
+func (channels TwitchChannels) SortByLive() {
+	sort.SliceStable(channels, func(i, j int) bool {
+		return channels[i].IsLive && !channels[j].IsLive
+	})
+}
+
 type twitchOperationResponse struct {
 	Data       json.RawMessage
 	Extensions struct {

internal/feed/utils.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"net/url"
+	"regexp"
 	"slices"
 	"strings"
 )
@@ -77,3 +78,20 @@ func maybeCopySliceWithoutZeroValues[T int | float64](values []T) []T {
 
 	return values
 }
+
+
+var urlSchemePattern = regexp.MustCompile(`^[a-z]+:\/\/`)
+
+func stripURLScheme(url string) string {
+	return urlSchemePattern.ReplaceAllString(url, "")
+}
+
+func limitStringLength(s string, max int) (string, bool) {
+	asRunes := []rune(s)
+
+	if len(asRunes) > max {
+		return string(asRunes[:max]), true
+	}
+
+	return s, false
+}