kyle/nlink
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

initial import

Browse Source
This commit is contained in:
Kyle Isom
2023-05-06 00:14:37 -07:00
commit 353cf2cf8a
14 changed files with 1078 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

119
nomad/db.go Normal file
View File

@@ -0,0 +1,119 @@
package nomad
import (
"database/sql"
"fmt"
_ "github.com/mattn/go-sqlite3"
)
type Executor interface {
Exec(stmt string, args ...interface{}) (sql.Result, error)
Query(stmt string, args ...interface{}) (*sql.Rows, error)
}
func buildArgs(n int) string {
s := ""
for i := 0; i < n; i++ {
s += "?,"
}
return s[:len(s)-1]
}
type DB struct {
path string
db *sql.DB
}
func NewDB(path string) *DB {
return &DB{path: path}
}
func (db *DB) Connect() (err error) {
if db.db != nil {
return nil
}
db.db, err = sql.Open("sqlite3", db.path)
return err
}
func (db *DB) Close() error {
if db.db == nil {
return nil
}
err := db.db.Close()
db.db = nil
return err
}
func (db *DB) Create() (err error) {
err = db.Connect()
if err != nil {
return err
}
_, err = db.db.Exec(createSQL)
return err
}
func (db *DB) Begin() (tx *sql.Tx, err error) {
err = db.Connect()
if err != nil {
return tx, err
}
return db.db.Begin()
}
func (db *DB) executor(tx *sql.Tx) Executor {
if tx != nil {
return tx
}
return db.db
}
func (db *DB) Mark(tx *sql.Tx, item Item) (err error) {
_, err = db.executor(tx).Exec(markSQL, item.URL.ID(), item.Title, item.PubDate)
return err
}
func (db *DB) Filter(tx *sql.Tx, items []Item) (newItems []Item, err error) {
seen := map[string]bool{}
args := buildArgs(len(items))
stmt := fmt.Sprintf(filterSQLTpl, args)
urls := make([]interface{}, 0, len(items))
for _, item := range items {
urls = append(urls, item.URL.ID())
}
rows, err := db.executor(tx).Query(stmt, urls...)
if err != nil {
return nil, err
}
for rows.Next() {
var url string
err = rows.Scan(&url)
if err != nil {
return nil, err
}
seen[url] = true
}
for _, item := range items {
if seen[item.URL.ID()] {
continue
}
newItems = append(newItems, item)
}
// Reverse the list of items so that they are in reverse chronological order.
nitems := len(items) - 1
for i := range items {
j := nitems - i
items[i], items[j] = items[j], items[i]
}
return newItems, nil
}

11
nomad/db_test.go Normal file
View File

@@ -0,0 +1,11 @@
package nomad
import "testing"
func TestBuildArgs(t *testing.T) {
expected := "?,?,?"
args := buildArgs(3)
if expected != args {
t.Fatalf("have '%s', want '%s'", args, expected)
}
}

51
nomad/item.go Normal file
View File

@@ -0,0 +1,51 @@
package nomad
import (
"sort"
"github.com/mmcdole/gofeed"
)
type Item struct {
URL Source
Title string
PubDate int64
}
type itemList []Item
func (l itemList) Len() int {
return len(l)
}
func (l itemList) Less(i, j int) bool {
return l[i].PubDate < l[j].PubDate
}
func (l itemList) Swap(i, j int) {
l[i], l[j] = l[j], l[i]
}
func FetchRSS(feedSource Source) (items []Item, err error) {
rd, err := feedSource.Fetch()
if err != nil {
return nil, err
}
feedParser := gofeed.NewParser()
feed, err := feedParser.ParseString(rd)
if err != nil {
return nil, err
}
for _, item := range feed.Items {
items = append(items, Item{
URL: NewSoupSource(item.Link),
Title: item.Title,
PubDate: item.PublishedParsed.Unix(),
})
}
sort.Sort(itemList(items))
return items, nil
}

104
nomad/item_test.go Normal file
View File

@@ -0,0 +1,104 @@
package nomad
import "testing"
var sampleRSS = `<?xml version='1.0' encoding='UTF-8'?>
<rss xmlns:atom="http://www.w3.org/2005/Atom" xmlns:content="http://purl.org/rss/1.0/modules/content/" version="2.0">
<channel>
<title>kyle@ nomad</title>
<link>https://nomad.wntrmute.net/u/kyle.rss</link>
<description>An experiment in MMS micropublishing</description>
<atom:link href="https://nomad.wntrmute.net/u/kyle.rss" rel="self"/>
<docs>http://www.rssboard.org/rss-specification</docs>
<generator>python-feedgen</generator>
<lastBuildDate>Thu, 04 Nov 2021 16:44:41 +0000</lastBuildDate>
<item>
<title>kyle @ 2021-11-03 10:36:44 PDT: At least not with images, I guess.</title>
<link>https://nomad.wntrmute.net/p/1996</link>
<description>At least not with images, I guess.</description>
<guid isPermaLink="true">https://nomad.wntrmute.net/p/1996</guid>
<pubDate>Wed, 03 Nov 2021 10:36:44 -0700</pubDate>
</item>
<item>
<title>kyle @ 2021-11-03 10:35:53 PDT: This thing isn't working at all...</title>
<link>https://nomad.wntrmute.net/p/1995</link>
<description>This thing isn't working at all...</description>
<guid isPermaLink="true">https://nomad.wntrmute.net/p/1995</guid>
<pubDate>Wed, 03 Nov 2021 10:35:53 -0700</pubDate>
</item>
<item>
<title>kyle @ 2021-11-03 02:38:48 PDT: And a plain text post too...</title>
<link>https://nomad.wntrmute.net/p/1994</link>
<description>And a plain text post too...</description>
<guid isPermaLink="true">https://nomad.wntrmute.net/p/1994</guid>
<pubDate>Wed, 03 Nov 2021 02:38:48 -0700</pubDate>
</item>
<item>
<title>kyle @ 2021-11-03 02:23:29 PDT: Working on a thing that needs a test post so here’s a picture of...</title>
<link>https://nomad.wntrmute.net/p/1993</link>
<description>Working on a thing that needs a test post so here’s a picture of some mead.</description>
<guid isPermaLink="true">https://nomad.wntrmute.net/p/1993</guid>
<enclosure url="https://nomad.sfo2.cdn.digitaloceanspaces.com/MM95aee9af66546a4ef9b66ea8168b359c.jpg" length="194792" type="image/jpeg"/>
<pubDate>Wed, 03 Nov 2021 02:23:29 -0700</pubDate>
</item>
<item>
<title>kyle @ 2021-10-31 22:44:56 PDT: Fall colours in Oakland.</title>
<link>https://nomad.wntrmute.net/p/1991</link>
<description>Fall colours in Oakland.</description>
<guid isPermaLink="true">https://nomad.wntrmute.net/p/1991</guid>
<enclosure url="https://nomad.sfo2.cdn.digitaloceanspaces.com/MM7095b4db336c7d03b23f2cd702eb0e0a.jpg" length="426006" type="image/jpeg"/>
<pubDate>Sun, 31 Oct 2021 22:44:56 -0700</pubDate>
</item>
<item>
<title>kyle @ 2021-10-25 20:02:11 PDT: I am the operator of my pocket ... operator.</title>
<link>https://nomad.wntrmute.net/p/1990</link>
<description>I am the operator of my pocket ... operator.</description>
<guid isPermaLink="true">https://nomad.wntrmute.net/p/1990</guid>
<enclosure url="https://nomad.sfo2.cdn.digitaloceanspaces.com/MMe8e7ad7b08b18e9df53ac97d87c8faa0.jpg" length="232659" type="image/jpeg"/>
<pubDate>Mon, 25 Oct 2021 20:02:11 -0700</pubDate>
</item>
<item>
<title>kyle @ 2021-10-24 19:28:09 PDT: The Yeti being a backup battery that I got with a solar panel fo...</title>
<link>https://nomad.wntrmute.net/p/1989</link>
<description>The Yeti being a backup battery that I got with a solar panel for offgrid stuff, good test for it though </description>
<guid isPermaLink="true">https://nomad.wntrmute.net/p/1989</guid>
<pubDate>Sun, 24 Oct 2021 19:28:09 -0700</pubDate>
</item>
<item>
<title>kyle @ 2021-10-24 19:27:14 PDT: Power's out, fortunately I have the Yeti</title>
<link>https://nomad.wntrmute.net/p/1988</link>
<description>Power's out, fortunately I have the Yeti</description>
<guid isPermaLink="true">https://nomad.wntrmute.net/p/1988</guid>
<pubDate>Sun, 24 Oct 2021 19:27:14 -0700</pubDate>
</item>
<item>
<title>kyle @ 2021-10-24 13:22:47 PDT: Finally exploring the trail behind my apartment, of course in th...</title>
<link>https://nomad.wntrmute.net/p/1986</link>
<description>Finally exploring the trail behind my apartment, of course in the rain.</description>
<guid isPermaLink="true">https://nomad.wntrmute.net/p/1986</guid>
<enclosure url="https://nomad.sfo2.cdn.digitaloceanspaces.com/MM6f13e74faa6d643d654507c5bd9d8f53.jpg" length="622207" type="image/jpeg"/>
<pubDate>Sun, 24 Oct 2021 13:22:47 -0700</pubDate>
</item>
<item>
<title>kyle @ 2021-10-24 09:33:15 PDT: From today’s Readwise email…</title>
<link>https://nomad.wntrmute.net/p/1984</link>
<description>From today’s Readwise email…</description>
<guid isPermaLink="true">https://nomad.wntrmute.net/p/1984</guid>
<enclosure url="https://nomad.sfo2.cdn.digitaloceanspaces.com/MMcc9d528ead93949ea21dd4b4aff21836.jpg" length="53704" type="image/jpeg"/>
<pubDate>Sun, 24 Oct 2021 09:33:15 -0700</pubDate>
</item>
</channel>
</rss>
`
func TestFetchRSS(t *testing.T) {
source := NewStringSource(sampleRSS)
items, err := FetchRSS(source)
if err != nil {
t.Fatal(err)
}
if len(items) == 0 {
t.Fatal("no items parsed from RSS")
}
}

126
nomad/post.go Normal file
View File

@@ -0,0 +1,126 @@
package nomad
import (
"bytes"
"errors"
"fmt"
"path/filepath"
"git.sr.ht/~thrrgilag/woodstock"
"github.com/anaskhan96/soup"
)
func nomadLink(url Source) string {
return fmt.Sprintf("[nomad.wntrmute.net](%s)", url.ID())
}
type sel struct {
selectors []string
}
func selector(selectors ...string) sel {
return sel{selectors: selectors}
}
func find(root soup.Root, attr string, selectors ...sel) (string, bool) {
result := root
for _, selector := range selectors {
result = result.Find(selector.selectors...)
if result.Pointer == nil {
return "", false
}
}
if attr == "" {
text := result.Text()
return text, text != ""
}
value, hasAttr := result.Attrs()[attr]
return value, hasAttr
}
type Post struct {
Image Source
Body string
URL Source
}
func NewPost(item Item) Post {
return Post{
Body: nomadLink(item.URL),
URL: item.URL,
}
}
func (p *Post) Fetch() error {
pageSource, err := p.URL.Fetch()
if err != nil {
return err
}
root := soup.HTMLParse(pageSource)
body, hasBody := find(root, "", selector("p", "class", "entry-body"))
if hasBody {
p.Body = body + " " + p.Body
}
imageURL, hasImageURL := find(
root, "src",
selector("div", "class", "entry-image"),
selector("img"))
if hasImageURL {
p.Image = NewURLSource(imageURL)
}
return nil
}
func (p Post) Post(client *woodstock.Client) error {
pnutPost := woodstock.NewPost{
Text: p.Body,
}
if p.Image != nil {
imageData, err := p.Image.FetchBytes()
if err != nil {
return err
}
params := map[string]string{
"name": filepath.Base(p.Image.ID()),
"is_public": "1",
"type": "photo",
}
r := bytes.NewBuffer(imageData)
file, err := client.CreateFile(params, r)
if err != nil {
panic(err.Error())
return err
}
pnutPost.Raw = []woodstock.Raw{pnutOembedRaw(file)}
}
result, err := client.Post(pnutPost)
if err != nil {
return err
}
if result.Meta.ErrorMessage != "" {
return errors.New("woodstock: " + result.Meta.ErrorMessage)
}
return nil
}
func pnutOembedRaw(file woodstock.FileResult) woodstock.Raw {
fval := map[string]string{
"file_id": file.Data.ID,
"file_token": file.Data.FileToken,
"format": "oembed",
}
value := map[string]map[string]string{"+io.pnut.core.file": fval}
return woodstock.Raw{Type: "io.pnut.core.oembed", Value: value}
}

97
nomad/post_test.go Normal file
View File

@@ -0,0 +1,97 @@
package nomad
import (
"testing"
)
var pageData = `<!DOCTYPE html>
<html lang="en">
<head>
<title>Nomad: An MMS µblog</title>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta property="og:title" content="Finally exploring the trail behind my apartment, of course in the rain." />
<meta property="og:type" content="website" />
<meta property="og:url" content="https://nomad.wntrmute.net/p/1986" />
<meta property="og:image" content="https://nomad.sfo2.cdn.digitaloceanspaces.com/MM6f13e74faa6d643d654507c5bd9d8f53.jpg" />
<link href="/index.rss" type="application/rss+xml" rel="alternate" title="Nomad feed" />
<link href="/static/favicon.ico" rel="shortcut icon">
<link href="/static/style.css" rel="stylesheet">
</head>
<body>
<div id="wrap">
<main>
<header>
<h1><a href="/">Nomad</a></h1>
</header>
<section>
<article class="entry entry-single" id="post-1986">
<div class="entry-details">
<span class="entry-author"><a href="/u/kyle">kyle</a></span>
</div>
<div class="entry-image">
<!-- TODO: how do we provide a useful alt? -->
<a href="https://nomad.sfo2.cdn.digitaloceanspaces.com/MM6f13e74faa6d643d654507c5bd9d8f53.jpg" target="_blank" rel="noopener">
<img src="https://nomad.sfo2.cdn.digitaloceanspaces.com/MM6f13e74faa6d643d654507c5bd9d8f53.jpg" alt="MMS attachment"/>
</a>
</div>
<p class="entry-body">Finally exploring the trail behind my apartment, of course in the rain.</p>
<div class="entry-details">
<time datetime="2021-10-24 13:22:47 PDT">
posted 2021-10-24 13:22:47 PDT
</time>
</div>
</article>
</section>
<footer>
<p>
<a href="/about">Nomad</a> is an experiment in MMS micropublishing,
and is bolted together with
<a href="https://www.palletsprojects.com/p/flask/">Flask</a>
and <a href="https://twilio.com/">Twilio</a>; it was inspired by
<a href="https://craigmod.com/essays/sms_publishing/">Craig Mod's experiment</a>.
It is being built by
<a class="h-card u-url" rel="me" href="https://ai6ua.net/">Kyle</a> and
<a class="h-card u-url" rel="me" href="https://wallyjones.com/">Wally</a>.
</p>
<p><small>Nomad v78</small></p>
</footer>
</main>
</div>
</body>
</html>
`
func TestFetch(t *testing.T) {
p := Post{
Body: nomadLink(NewURLSource("https://localhost")),
URL: NewStringSource(pageData),
}
err := p.Fetch()
if err != nil {
t.Fatal(err)
}
expectedBody := "Finally exploring the trail behind my apartment, of course in the rain. [nomad.wntrmute.net](https://localhost)"
expectedImage := "https://nomad.sfo2.cdn.digitaloceanspaces.com/MM6f13e74faa6d643d654507c5bd9d8f53.jpg"
if p.Body != expectedBody {
t.Fatalf("have '%s', want '%s'", p.Body, expectedBody)
}
if p.Image.ID() != expectedImage {
t.Fatalf("have '%s', want '%s'", p.Image, expectedImage)
}
}

114
nomad/source.go Normal file
View File

@@ -0,0 +1,114 @@
package nomad
import (
"crypto/sha256"
"fmt"
"io/ioutil"
"net/http"
"github.com/anaskhan96/soup"
)
// Source provides data for the various parsing functions.
type Source interface {
Fetch() (string, error)
FetchBytes() ([]byte, error)
ID() string
String() string
}
// URLSource fetches data from an HTTP resource.
type URLSource struct {
url string
}
func (u URLSource) FetchBytes() ([]byte, error) {
resp, err := http.Get(u.url)
if err != nil {
return nil, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
return body, err
}
func (u URLSource) Fetch() (string, error) {
body, err := u.FetchBytes()
if err != nil {
return "", err
}
return string(body), nil
}
func (u URLSource) ID() string {
return u.url
}
func (u URLSource) String() string {
return u.url
}
func NewURLSource(url string) URLSource {
return URLSource{url: url}
}
type StringSource struct {
s string
}
func (s StringSource) Fetch() (string, error) {
return s.String(), nil
}
func (s StringSource) FetchBytes() ([]byte, error) {
return []byte(s.s), nil
}
func NewStringSource(s string) *StringSource {
return &StringSource{s: s}
}
func (s StringSource) ID() string {
h := sha256.Sum256([]byte(s.s))
return fmt.Sprintf("%s", h[:])
}
func (s StringSource) String() string {
return s.s
}
// SoupSource uses the soup package to fetch data.
type SoupSource struct {
url string
}
func (s SoupSource) Fetch() (string, error) {
return soup.Get(s.url)
}
func (s SoupSource) FetchBytes() ([]byte, error) {
body, err := s.Fetch()
if err != nil {
return nil, err
}
return []byte(body), nil
}
func (s SoupSource) ID() string {
return s.url
}
func (s SoupSource) String() string {
return s.url
}
func NewSoupSource(url string) SoupSource {
return SoupSource{url: url}
}

11
nomad/sql.go Normal file
View File

@@ -0,0 +1,11 @@
package nomad
var createSQL = `CREATE TABLE IF NOT EXISTS posts (
url TEXT UNIQUE NOT NULL PRIMARY KEY,
title TEXT UNIQUE NOT NULL,
pub_date INTEGER NOT NULL
);`
var markSQL = `INSERT INTO posts (url, title, pub_date) VALUES (?, ?, ?);`
var filterSQLTpl = `SELECT url FROM posts WHERE url IN (%s)`