From 9541e278909696e2a21ef6e0e3ef448bfc4a6707 Mon Sep 17 00:00:00 2001 From: "Andrey Ivanov (NetMoose)" Date: Thu, 24 Feb 2022 21:26:07 +0500 Subject: [PATCH] Add initdb. Add find items for sending. Update db after send. --- .gitignore | 3 +- go.mod | 1 + go.sum | 2 + main.go | 123 +++++++++++++++++++++++++++++++++++++++++++++++------ 4 files changed, 116 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 77c271d..6637737 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.yml -main \ No newline at end of file +main +*.xml \ No newline at end of file diff --git a/go.mod b/go.mod index 2d420a8..e9dc5c0 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module mod go 1.17 require ( + github.com/boltdb/bolt v1.3.1 // indirect github.com/jessevdk/go-flags v1.5.0 // indirect golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 07c6e45..e438d95 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= +github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4 h1:EZ2mChiOa8udjfp6rRmswTbtZN/QzUQp4ptM4rnjHvc= diff --git a/main.go b/main.go index e0ac06e..b63ff9c 100644 --- a/main.go +++ b/main.go @@ -1,16 +1,19 @@ package main import ( + "encoding/json" "encoding/xml" "html/template" "log" "os" + "github.com/boltdb/bolt" "github.com/jessevdk/go-flags" "gopkg.in/yaml.v2" ) type Config struct { + Dbpath string `yaml:"dbpath"` Telegram struct { Send bool `yaml:"send"` Token string `yaml:"token"` @@ -48,8 +51,9 @@ func NewConfig(configPath string) (*Config, error) { } type Options struct { - FileParse string `short:"f" long:"fileparse" description:"File for parce (rss xml)"` + FileParse string `short:"f" long:"fileparse" description:"File for parce (rss xml)" required:"true"` ConfigPath string `short:"c" long:"configpath" description:"Config file path"` + InitDB bool `short:"i" long:"initdb" description:"Run initialize from current file"` } var ConfigPath = "/etc/ssender/config.yml" @@ -97,19 +101,105 @@ func NewRSS(rssPath string) (*Rss2, error) { return rss, nil } +type SendItems struct { + ItemList []Item +} + +var senditems SendItems + +func FindItems(rss Rss2, dbpath string) { + db, err := bolt.Open(dbpath, 0600, nil) + if err != nil { + log.Fatal(err) + } + defer db.Close() + + for _, v := range rss.ItemList { + db.View(func(tx *bolt.Tx) error { + // Assume bucket exists and has keys + b := tx.Bucket([]byte("rss")) + c := b.Cursor() + flag := false + for key, _ := c.First(); key != nil; key, _ = c.Next() { + if v.Link == string(key) { + flag = true + break + } + } + if flag != true { + senditems.ItemList = append(senditems.ItemList, v) + } + return nil + }) + } +} + +func InitDb(rss Rss2, dbpath string) { + log.Println("Initialize DB") + db, err := bolt.Open(dbpath, 0600, nil) + if err != nil { + log.Fatal(err) + } + defer db.Close() + + db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucketIfNotExists([]byte("rss")) + if err != nil { + return err + } + for _, v := range rss.ItemList { + encoded, err := json.Marshal(v) + if err != nil { + return err + } + err = b.Put([]byte(v.Link), encoded) + if err != nil { + return err + } + } + return nil + }) +} + func (config Config) RunSend() { if config.Telegram.Send { - log.Println("Send to telegram.") + log.Println("Send to telegram") } if config.VK.Send { - log.Println("Send to VK.") + log.Println("Send to VK") } if config.Facebook.Send { - log.Println("Send to Facebook.") + log.Println("Send to Facebook") + } +} + +func UpdateDb(dbpath string) { + log.Println("Update DB") + db, err := bolt.Open(dbpath, 0600, nil) + if err != nil { + log.Fatal(err) } + defer db.Close() + + db.Update(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("rss")) + for _, v := range senditems.ItemList { + encoded, err := json.Marshal(v) + if err != nil { + return err + } + err = b.Put([]byte(v.Link), encoded) + if err != nil { + return err + } + } + return nil + }) + } func main() { + log.Println("Run processing") // Parse flags var options Options var parser = flags.NewParser(&options, flags.Default) @@ -124,7 +214,7 @@ func main() { os.Exit(1) } } - log.Println("Flags processed.") + log.Println("Flags processed") if options.ConfigPath != "" { log.Printf("Config from: %s\n", options.ConfigPath) @@ -136,17 +226,26 @@ func main() { if err != nil { log.Fatal(err) } - log.Println("Config processed.") + log.Println("Config processed") - // Parse file + // Parse rss file log.Printf("Parse file %s \n", options.FileParse) rss, err := NewRSS(options.FileParse) if err != nil { log.Fatal(err) } - log.Printf("Rss: %v\n", rss) - - // Run send data depended on configuration options - log.Println("Run send process.") - cfg.RunSend() + if options.InitDB { + InitDb(*rss, cfg.Dbpath) + } else { + //Find new items + FindItems(*rss, cfg.Dbpath) + + if len(senditems.ItemList) > 0 { + // Run send data depended on configuration options + log.Println("Run send process") + cfg.RunSend() + UpdateDb(cfg.Dbpath) + } + } + log.Println("End processing") }