package main import ( "crypto/tls" "encoding/xml" "html/template" "io" "log" "net/http" "sort" "time" "github.com/boltdb/bolt" ) type Rss2 struct { XMLName xml.Name `xml:"rss"` Version string `xml:"version,attr"` // Required Title string `xml:"channel>title"` Link string `xml:"channel>link"` Description string `xml:"channel>description"` // Optional PubDate string `xml:"channel>pubDate"` ItemList []Item `xml:"channel>item"` } type Item struct { // Required Title string `xml:"title"` Link string `xml:"link"` Description template.HTML `xml:"description"` // Optional Content template.HTML `xml:"encoded"` PubDate string `xml:"pubDate"` Comments string `xml:"comments"` } type SendItems struct { ItemList []Item } type ByPubDate []Item func (a ByPubDate) Len() int { return len(a) } func (a ByPubDate) Less(i, j int) bool { timeone, _ := time.Parse("Mon, 02 Jan 2006 15:04:05 -0700", a[i].PubDate) timetwo, _ := time.Parse("Mon, 02 Jan 2006 15:04:05 -0700", a[j].PubDate) return timeone.Unix() > timetwo.Unix() } func (a ByPubDate) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func GetRSS(name string, url string) (*Rss2, error) { rss := &Rss2{} var netClient = &http.Client{} customTransport := &(*http.DefaultTransport.(*http.Transport)) // make shallow copy timeout := time.Duration(240 * time.Second) customTransport = &http.Transport{ IdleConnTimeout: timeout, ResponseHeaderTimeout: timeout, DisableKeepAlives: false, DisableCompression: false, ForceAttemptHTTP2: true, TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, TLSHandshakeTimeout: timeout, MaxIdleConns: 20, MaxIdleConnsPerHost: 100, MaxConnsPerHost: 100, } netClient = &http.Client{Transport: customTransport, Timeout: timeout} request, err := http.NewRequest("GET", url, nil) if err != nil { return nil, err } request.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:97.0) Gecko/20100101 Firefox/97.0") resp, err := netClient.Do(request) if err != nil { return nil, err } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { return nil, err } // Start RSS decoding from file if err := xml.Unmarshal(body, rss); err != nil { return nil, err } return rss, nil } func ProcessRss(rss Rss2, dbpath string, rssname string) (*SendItems, error) { var si SendItems db, err := bolt.Open(dbpath, 0600, nil) if err != nil { log.Fatal(err) } defer db.Close() db.Update(func(tx *bolt.Tx) error { _, err := tx.CreateBucketIfNotExists([]byte(rssname)) if err != nil { return err } return nil }) for _, v := range rss.ItemList { db.View(func(tx *bolt.Tx) error { // Assume bucket exists and has keys b := tx.Bucket([]byte(rssname)) c := b.Cursor() flag := false for key, _ := c.First(); key != nil; key, _ = c.Next() { if v.Link == string(key) { flag = true break } } if !flag { si.ItemList = append(si.ItemList, v) } return nil }) } sort.Sort(ByPubDate(si.ItemList)) if len(si.ItemList) > 0 { return &si, nil } else { return nil, nil } }