package main import ( "flag" "path/filepath" "strings" "time" "git.wntrmute.dev/kyle/nlink/nomad" "git.sr.ht/~thrrgilag/woodstock" "git.wntrmute.dev/kyle/goutils/config" "git.wntrmute.dev/kyle/goutils/die" "git.wntrmute.dev/kyle/goutils/log" ) var defaultRSS = "https://nomad.wntrmute.net/u/kyle.rss" func defaultPath(file string) string { return filepath.Join("/etc", "nlink", file) } func fetchItems(db *nomad.DB, feed string, markOnly bool) error { rssFeed := nomad.NewURLSource(feed) items, err := nomad.FetchRSS(rssFeed) if err != nil { return nil } items, err = db.Filter(nil, items) if err != nil { return err } if items == nil { return nil } if markOnly { tx, err := db.Begin() if err != nil { return err } for _, item := range items { log.Debugf("marking item: %s", item.Title) if err = db.Mark(tx, item); err != nil { return err } } tx.Commit() if err := saveDatabase(config.Get("local_database")); err != nil { return err } return nil } client := woodstock.NewClient(config.Get("pnut_id"), config.Get("pnut_secret")) client.SetAccessToken(config.Get("pnut_auth_token")) for _, item := range items { p := nomad.NewPost(item) if err = p.Fetch(); err != nil { log.Err(err) continue } if err = p.Post(client); err != nil { log.Err(err) continue } if err = db.Mark(nil, item); err != nil { log.Err(err) continue } } if err := saveDatabase(config.Get("local_database")); err != nil { log.Err(err) } return nil } const defaultInterval = 5 * time.Minute func splitFeeds() []string { feeds := strings.Split(config.Get("local_rss_feed_url"), " ") for i := range feeds { feeds[i] = strings.TrimSpace(feeds[i]) log.Debugf("feed: %s", feeds[i]) } return feeds } func main() { var ( configFile string interval time.Duration initDB bool level string markOnly bool ) flag.StringVar(&configFile, "f", defaultPath("nlink.conf"), "`path` to config file") flag.BoolVar(&initDB, "i", false, "initialize a new DB") flag.StringVar(&level, "l", "DEBUG", "log level") flag.BoolVar(&markOnly, "m", false, "only mark posts as having been posted") flag.DurationVar(&interval, "t", defaultInterval, "interval between fetching posts") flag.Parse() logOpts := log.DefaultOptions("nlink", false) logOpts.Level = level if err := log.Setup(logOpts); err != nil { die.If(err) } log.Infoln("nlink is starting") log.Debugf("loading config file %s\n", configFile) if err := config.LoadFile(configFile); err != nil { log.Fatal(err) } log.Debugln("config keys") for _, k := range config.ListKeys() { log.Debugf("key: %s", k) } log.Infof("minio_bucket: %s", config.Get("minio_bucket")) if err := restoreDatabase(config.Get("local_database")); err != nil { log.Fatal(err) } if initDB { db := nomad.NewDB(config.Get("local_database")) if err := db.Create(); err != nil { log.Fatal(err) } db.Close() } log.Debugf("connecting to db %s", config.Get("local_database")) db := nomad.NewDB(config.Get("local_database")) if err := db.Connect(); err != nil { log.Emerg(err) } defer db.Close() feeds := splitFeeds() if markOnly { log.Info("marking database") for _, feed := range feeds { log.Debugf("marking feed %s", feed) if err := fetchItems(db, feed, markOnly); err != nil { log.Fatal(err) } } return } log.Debugf("will look for new items every %s\n", interval) for { log.Infoln("running fetch loop") for _, feed := range feeds { log.Infof("fetching items for feed %s", feed) if err := fetchItems(db, feed, markOnly); err != nil { log.Errln(err) } } time.Sleep(interval) } log.Infoln("exiting") }