|
|
|
@ -1,14 +1,13 @@ |
|
|
|
package main |
|
|
|
package main |
|
|
|
|
|
|
|
|
|
|
|
import ( |
|
|
|
import ( |
|
|
|
"encoding/json" |
|
|
|
|
|
|
|
"encoding/xml" |
|
|
|
|
|
|
|
"log" |
|
|
|
"log" |
|
|
|
"os" |
|
|
|
"os" |
|
|
|
"ssender/internal/config" |
|
|
|
"ssender/internal/config" |
|
|
|
"ssender/internal/models" |
|
|
|
"ssender/internal/db" |
|
|
|
|
|
|
|
"ssender/internal/rss" |
|
|
|
|
|
|
|
"ssender/internal/send" |
|
|
|
|
|
|
|
|
|
|
|
"github.com/boltdb/bolt" |
|
|
|
|
|
|
|
"github.com/jessevdk/go-flags" |
|
|
|
"github.com/jessevdk/go-flags" |
|
|
|
) |
|
|
|
) |
|
|
|
|
|
|
|
|
|
|
|
@ -18,108 +17,9 @@ type Options struct { |
|
|
|
InitDB bool `short:"i" long:"initdb" description:"Run initialize from current file"` |
|
|
|
InitDB bool `short:"i" long:"initdb" description:"Run initialize from current file"` |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
var ConfigPath = "/etc/ssender/config.yml" |
|
|
|
var ( |
|
|
|
|
|
|
|
ConfigPath = "/etc/ssender/config.yml" |
|
|
|
func NewRSS(rssPath string) (*models.Rss2, error) { |
|
|
|
) |
|
|
|
rss := &models.Rss2{} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Open rss2 file
|
|
|
|
|
|
|
|
file, err := os.Open(rssPath) |
|
|
|
|
|
|
|
if err != nil { |
|
|
|
|
|
|
|
return nil, err |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
defer file.Close() |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
d := xml.NewDecoder(file) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Start RSS decoding from file
|
|
|
|
|
|
|
|
if err := d.Decode(&rss); err != nil { |
|
|
|
|
|
|
|
return nil, err |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
return rss, nil |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
var senditems models.SendItems |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
func FindItems(rss models.Rss2, dbpath string) { |
|
|
|
|
|
|
|
db, err := bolt.Open(dbpath, 0600, nil) |
|
|
|
|
|
|
|
if err != nil { |
|
|
|
|
|
|
|
log.Fatal(err) |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
defer db.Close() |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
for _, v := range rss.ItemList { |
|
|
|
|
|
|
|
db.View(func(tx *bolt.Tx) error { |
|
|
|
|
|
|
|
// Assume bucket exists and has keys
|
|
|
|
|
|
|
|
b := tx.Bucket([]byte("rss")) |
|
|
|
|
|
|
|
c := b.Cursor() |
|
|
|
|
|
|
|
flag := false |
|
|
|
|
|
|
|
for key, _ := c.First(); key != nil; key, _ = c.Next() { |
|
|
|
|
|
|
|
if v.Link == string(key) { |
|
|
|
|
|
|
|
flag = true |
|
|
|
|
|
|
|
break |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
if flag { |
|
|
|
|
|
|
|
senditems.ItemList = append(senditems.ItemList, v) |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
return nil |
|
|
|
|
|
|
|
}) |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
func InitDb(rss models.Rss2, dbpath string) { |
|
|
|
|
|
|
|
log.Println("Initialize DB") |
|
|
|
|
|
|
|
db, err := bolt.Open(dbpath, 0600, nil) |
|
|
|
|
|
|
|
if err != nil { |
|
|
|
|
|
|
|
log.Fatal(err) |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
defer db.Close() |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
db.Update(func(tx *bolt.Tx) error { |
|
|
|
|
|
|
|
b, err := tx.CreateBucketIfNotExists([]byte("rss")) |
|
|
|
|
|
|
|
if err != nil { |
|
|
|
|
|
|
|
return err |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
for _, v := range rss.ItemList { |
|
|
|
|
|
|
|
encoded, err := json.Marshal(v) |
|
|
|
|
|
|
|
if err != nil { |
|
|
|
|
|
|
|
return err |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
err = b.Put([]byte(v.Link), encoded) |
|
|
|
|
|
|
|
if err != nil { |
|
|
|
|
|
|
|
return err |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
return nil |
|
|
|
|
|
|
|
}) |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
func UpdateDb(dbpath string) { |
|
|
|
|
|
|
|
log.Println("Update DB") |
|
|
|
|
|
|
|
db, err := bolt.Open(dbpath, 0600, nil) |
|
|
|
|
|
|
|
if err != nil { |
|
|
|
|
|
|
|
log.Fatal(err) |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
defer db.Close() |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
db.Update(func(tx *bolt.Tx) error { |
|
|
|
|
|
|
|
b := tx.Bucket([]byte("rss")) |
|
|
|
|
|
|
|
for _, v := range senditems.ItemList { |
|
|
|
|
|
|
|
encoded, err := json.Marshal(v) |
|
|
|
|
|
|
|
if err != nil { |
|
|
|
|
|
|
|
return err |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
err = b.Put([]byte(v.Link), encoded) |
|
|
|
|
|
|
|
if err != nil { |
|
|
|
|
|
|
|
return err |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
return nil |
|
|
|
|
|
|
|
}) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
func main() { |
|
|
|
func main() { |
|
|
|
log.Println("Run processing") |
|
|
|
log.Println("Run processing") |
|
|
|
@ -139,10 +39,11 @@ func main() { |
|
|
|
} |
|
|
|
} |
|
|
|
log.Println("Flags processed") |
|
|
|
log.Println("Flags processed") |
|
|
|
|
|
|
|
|
|
|
|
if options.ConfigPath != "" { |
|
|
|
if options.ConfigPath == "" { |
|
|
|
|
|
|
|
log.Fatalf("Config path not found in options!") |
|
|
|
|
|
|
|
} |
|
|
|
log.Printf("Config from: %s\n", options.ConfigPath) |
|
|
|
log.Printf("Config from: %s\n", options.ConfigPath) |
|
|
|
ConfigPath = options.ConfigPath |
|
|
|
ConfigPath = options.ConfigPath |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Get config
|
|
|
|
// Get config
|
|
|
|
cfg, err := config.NewConfig(ConfigPath) |
|
|
|
cfg, err := config.NewConfig(ConfigPath) |
|
|
|
@ -153,22 +54,37 @@ func main() { |
|
|
|
|
|
|
|
|
|
|
|
// Parse rss file
|
|
|
|
// Parse rss file
|
|
|
|
log.Printf("Parse file %s \n", options.FileParse) |
|
|
|
log.Printf("Parse file %s \n", options.FileParse) |
|
|
|
rss, err := NewRSS(options.FileParse) |
|
|
|
rss, err := rss.NewRSS(options.FileParse) |
|
|
|
if err != nil { |
|
|
|
if err != nil { |
|
|
|
log.Fatal(err) |
|
|
|
log.Fatal(err) |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
if options.InitDB { |
|
|
|
if options.InitDB { |
|
|
|
InitDb(*rss, cfg.Dbpath) |
|
|
|
err := db.InitDb(*rss, cfg.Dbpath) |
|
|
|
} else { |
|
|
|
if err != nil { |
|
|
|
|
|
|
|
log.Fatal(err) |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
//Find new items
|
|
|
|
//Find new items
|
|
|
|
FindItems(*rss, cfg.Dbpath) |
|
|
|
sendItems, err := db.FindItems(*rss, cfg.Dbpath) |
|
|
|
|
|
|
|
if err != nil { |
|
|
|
|
|
|
|
log.Fatal(err) |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
if len(senditems.ItemList) > 0 { |
|
|
|
if len(sendItems.ItemList) > 0 { |
|
|
|
// Run send data depended on configuration options
|
|
|
|
// Run send data depended on configuration options
|
|
|
|
log.Println("Run send process") |
|
|
|
log.Println("Run send process") |
|
|
|
cfg.RunSend(senditems) |
|
|
|
err := send.RunSend(sendItems, *cfg) |
|
|
|
UpdateDb(cfg.Dbpath) |
|
|
|
if err != nil { |
|
|
|
|
|
|
|
log.Fatal(err) |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
// Run update db
|
|
|
|
|
|
|
|
err = db.UpdateDb(cfg.Dbpath, sendItems) |
|
|
|
|
|
|
|
if err != nil { |
|
|
|
|
|
|
|
log.Fatal(err) |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
log.Println("End processing") |
|
|
|
log.Println("End processing") |
|
|
|
} |
|
|
|
} |
|
|
|
|