RSSCollector/process_rss.go

package main

import (
	"crypto/tls"
	"encoding/xml"
	"html/template"
	"io"
	"log"
	"net/http"
	"sort"
	"time"

	"github.com/boltdb/bolt"
)

type Rss2 struct {
	XMLName xml.Name `xml:"rss"`
	Version string   `xml:"version,attr"`
	// Required
	Title       string `xml:"channel>title"`
	Link        string `xml:"channel>link"`
	Description string `xml:"channel>description"`
	// Optional
	PubDate  string `xml:"channel>pubDate"`
	ItemList []Item `xml:"channel>item"`
}

type Item struct {
	// Required
	Title       string        `xml:"title"`
	Link        string        `xml:"link"`
	Description template.HTML `xml:"description"`
	// Optional
	Content  template.HTML `xml:"encoded"`
	PubDate  string        `xml:"pubDate"`
	Comments string        `xml:"comments"`
}

type SendItems struct {
	ItemList []Item
}

type ByPubDate []Item

func (a ByPubDate) Len() int { return len(a) }
func (a ByPubDate) Less(i, j int) bool {
	timeone, _ := time.Parse("Mon, 02 Jan 2006 15:04:05 -0700", a[i].PubDate)
	timetwo, _ := time.Parse("Mon, 02 Jan 2006 15:04:05 -0700", a[j].PubDate)
	return timeone.Unix() > timetwo.Unix()
}
func (a ByPubDate) Swap(i, j int) { a[i], a[j] = a[j], a[i] }

func GetRSS(name string, url string) (*Rss2, error) {
	rss := &Rss2{}

	var netClient = &http.Client{}

	customTransport := &(*http.DefaultTransport.(*http.Transport)) // make shallow copy
	timeout := time.Duration(240 * time.Second)
	customTransport = &http.Transport{
		IdleConnTimeout:       timeout,
		ResponseHeaderTimeout: timeout,
		DisableKeepAlives:     false,
		DisableCompression:    false,
		ForceAttemptHTTP2:     true,
		TLSClientConfig:       &tls.Config{InsecureSkipVerify: true},
		TLSHandshakeTimeout:   timeout,
		MaxIdleConns:          20,
		MaxIdleConnsPerHost:   100,
		MaxConnsPerHost:       100,
	}
	netClient = &http.Client{Transport: customTransport, Timeout: timeout}
	request, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	request.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:97.0) Gecko/20100101 Firefox/97.0")

	resp, err := netClient.Do(request)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()

	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	// Start RSS decoding from file
	if err := xml.Unmarshal(body, rss); err != nil {
		return nil, err
	}

	return rss, nil
}

func ProcessRss(rss Rss2, dbpath string, rssname string) (*SendItems, error) {
	var si SendItems
	db, err := bolt.Open(dbpath, 0600, nil)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()
	db.Update(func(tx *bolt.Tx) error {
		_, err := tx.CreateBucketIfNotExists([]byte(rssname))
		if err != nil {
			return err
		}
		return nil
	})

	for _, v := range rss.ItemList {
		db.View(func(tx *bolt.Tx) error {
			// Assume bucket exists and has keys
			b := tx.Bucket([]byte(rssname))
			c := b.Cursor()
			flag := false
			for key, _ := c.First(); key != nil; key, _ = c.Next() {
				if v.Link == string(key) {
					flag = true
					break
				}
			}
			if !flag {
				si.ItemList = append(si.ItemList, v)
			}
			return nil
		})
	}

	sort.Sort(ByPubDate(si.ItemList))
	if len(si.ItemList) > 0 {
		return &si, nil
	} else {
		return nil, nil
	}
}