nlink/main.go

184 lines
3.6 KiB
Go
Raw Normal View History

2023-05-06 07:14:37 +00:00
package main
import (
"flag"
"path/filepath"
2023-08-29 19:03:43 +00:00
"strings"
2023-05-06 07:14:37 +00:00
"time"
"git.wntrmute.dev/kyle/nlink/nomad"
2023-05-06 09:12:15 +00:00
"git.sr.ht/~thrrgilag/woodstock"
2023-05-06 07:14:37 +00:00
"git.wntrmute.dev/kyle/goutils/config"
"git.wntrmute.dev/kyle/goutils/die"
2023-08-29 19:03:43 +00:00
"git.wntrmute.dev/kyle/goutils/log"
2023-05-06 07:14:37 +00:00
)
var defaultRSS = "https://nomad.wntrmute.net/u/kyle.rss"
func defaultPath(file string) string {
2023-05-06 07:32:00 +00:00
return filepath.Join("/etc", "nlink", file)
2023-05-06 07:14:37 +00:00
}
2023-08-29 19:03:43 +00:00
func fetchItems(db *nomad.DB, feed string, markOnly bool) error {
rssFeed := nomad.NewURLSource(feed)
2023-05-06 07:14:37 +00:00
items, err := nomad.FetchRSS(rssFeed)
if err != nil {
return nil
}
items, err = db.Filter(nil, items)
if err != nil {
return err
}
if items == nil {
return nil
}
if markOnly {
tx, err := db.Begin()
if err != nil {
return err
}
for _, item := range items {
2023-08-29 19:03:43 +00:00
log.Debugf("marking item: %s", item.Title)
2023-05-06 07:14:37 +00:00
if err = db.Mark(tx, item); err != nil {
return err
}
}
tx.Commit()
2023-05-06 07:32:00 +00:00
2023-08-29 19:03:43 +00:00
if err := saveDatabase(config.Get("local_database")); err != nil {
2023-05-06 07:32:00 +00:00
return err
}
return nil
2023-05-06 07:14:37 +00:00
}
client := woodstock.NewClient(config.Get("pnut_id"), config.Get("pnut_secret"))
2023-05-06 20:42:58 +00:00
client.SetAccessToken(config.Get("pnut_auth_token"))
2023-05-06 07:14:37 +00:00
for _, item := range items {
p := nomad.NewPost(item)
2023-05-06 08:52:30 +00:00
if err = p.Fetch(); err != nil {
log.Err(err)
continue
}
2023-05-06 07:14:37 +00:00
2023-05-06 08:52:30 +00:00
if err = p.Post(client); err != nil {
log.Err(err)
continue
}
2023-05-06 07:14:37 +00:00
2023-05-06 08:52:30 +00:00
if err = db.Mark(nil, item); err != nil {
log.Err(err)
continue
}
2023-05-06 07:14:37 +00:00
}
2023-05-06 08:52:30 +00:00
if err := saveDatabase(config.Get("local_database")); err != nil {
2023-05-06 07:32:00 +00:00
log.Err(err)
}
2023-05-06 07:14:37 +00:00
return nil
}
2023-05-06 08:52:30 +00:00
const defaultInterval = 5 * time.Minute
2023-05-06 07:14:37 +00:00
2023-08-29 19:03:43 +00:00
func splitFeeds() []string {
feeds := strings.Split(config.Get("local_rss_feed_url"), " ")
for i := range feeds {
feeds[i] = strings.TrimSpace(feeds[i])
log.Debugf("feed: %s", feeds[i])
}
return feeds
}
2023-05-06 07:14:37 +00:00
func main() {
var (
configFile string
interval time.Duration
initDB bool
level string
markOnly bool
)
flag.StringVar(&configFile, "f", defaultPath("nlink.conf"), "`path` to config file")
flag.BoolVar(&initDB, "i", false, "initialize a new DB")
2023-05-06 08:27:50 +00:00
flag.StringVar(&level, "l", "DEBUG", "log level")
2023-05-06 07:14:37 +00:00
flag.BoolVar(&markOnly, "m", false, "only mark posts as having been posted")
2023-05-06 08:52:30 +00:00
flag.DurationVar(&interval, "t", defaultInterval, "interval between fetching posts")
2023-05-06 07:14:37 +00:00
flag.Parse()
2023-08-29 19:03:43 +00:00
logOpts := log.DefaultOptions("nlink", false)
logOpts.Level = level
2023-05-06 07:14:37 +00:00
if err := log.Setup(logOpts); err != nil {
die.If(err)
}
2023-08-29 20:15:50 +00:00
log.Infoln("nlink is starting")
2023-05-06 08:52:30 +00:00
log.Debugf("loading config file %s\n", configFile)
if err := config.LoadFile(configFile); err != nil {
2023-05-06 07:32:00 +00:00
log.Fatal(err)
}
2023-08-29 20:00:05 +00:00
log.Debugln("config keys")
for _, k := range config.ListKeys() {
log.Debugf("key: %s", k)
}
2023-05-06 08:52:30 +00:00
log.Infof("minio_bucket: %s", config.Get("minio_bucket"))
if err := restoreDatabase(config.Get("local_database")); err != nil {
2023-05-06 07:14:37 +00:00
log.Fatal(err)
}
if initDB {
db := nomad.NewDB(config.Get("local_database"))
2023-05-06 08:52:30 +00:00
if err := db.Create(); err != nil {
log.Fatal(err)
}
2023-05-06 07:14:37 +00:00
db.Close()
}
2023-05-06 08:52:30 +00:00
2023-08-29 19:03:43 +00:00
log.Debugf("connecting to db %s", config.Get("local_database"))
2023-05-06 07:14:37 +00:00
db := nomad.NewDB(config.Get("local_database"))
2023-05-06 08:52:30 +00:00
if err := db.Connect(); err != nil {
log.Emerg(err)
}
2023-05-06 07:14:37 +00:00
defer db.Close()
2023-08-29 19:03:43 +00:00
feeds := splitFeeds()
2023-08-29 20:23:38 +00:00
2023-05-06 07:14:37 +00:00
if markOnly {
2023-05-06 08:52:30 +00:00
log.Info("marking database")
2023-08-29 19:03:43 +00:00
for _, feed := range feeds {
log.Debugf("marking feed %s", feed)
if err := fetchItems(db, feed, markOnly); err != nil {
log.Fatal(err)
}
2023-05-06 07:14:37 +00:00
}
return
}
2023-05-06 08:52:30 +00:00
log.Debugf("will look for new items every %s\n", interval)
2023-05-06 07:14:37 +00:00
for {
2023-08-29 20:00:05 +00:00
log.Infoln("running fetch loop")
2023-08-29 19:03:43 +00:00
for _, feed := range feeds {
2023-08-29 20:00:05 +00:00
log.Infof("fetching items for feed %s", feed)
2023-08-29 19:03:43 +00:00
if err := fetchItems(db, feed, markOnly); err != nil {
2023-08-29 20:00:05 +00:00
log.Errln(err)
2023-08-29 19:03:43 +00:00
}
2023-05-06 08:52:30 +00:00
}
2023-05-06 07:14:37 +00:00
time.Sleep(interval)
}
2023-08-29 20:08:56 +00:00
log.Infoln("exiting")
2023-05-06 07:14:37 +00:00
}