From 353cf2cf8a473348278e85d7e08af8435d310a98 Mon Sep 17 00:00:00 2001 From: Kyle Isom Date: Sat, 6 May 2023 00:14:37 -0700 Subject: [PATCH] initial import --- .gitignore | 1 + README.md | 47 ++++++++++++ go.mod | 12 +++ go.sum | 185 +++++++++++++++++++++++++++++++++++++++++++++ main.go | 127 +++++++++++++++++++++++++++++++ nomad/db.go | 119 +++++++++++++++++++++++++++++ nomad/db_test.go | 11 +++ nomad/item.go | 51 +++++++++++++ nomad/item_test.go | 104 +++++++++++++++++++++++++ nomad/post.go | 126 ++++++++++++++++++++++++++++++ nomad/post_test.go | 97 ++++++++++++++++++++++++ nomad/source.go | 114 ++++++++++++++++++++++++++++ nomad/sql.go | 11 +++ storage.go | 73 ++++++++++++++++++ 14 files changed, 1078 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go create mode 100644 nomad/db.go create mode 100644 nomad/db_test.go create mode 100644 nomad/item.go create mode 100644 nomad/item_test.go create mode 100644 nomad/post.go create mode 100644 nomad/post_test.go create mode 100644 nomad/source.go create mode 100644 nomad/sql.go create mode 100644 storage.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4f40057 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +nlink.conf diff --git a/README.md b/README.md new file mode 100644 index 0000000..6b9eb6a --- /dev/null +++ b/README.md @@ -0,0 +1,47 @@ +# nomadlink + +nomadlink is a tool for syncing a Nomad RSS feed to pnut. + +[Nomad](https://nomad.wntrmute.net/) +[Pnut](https://beta.pnut.io/) + +The original version was running as a cronjob on a Raspberry Pi. I've +modified it to run on a gokrazy instance. + +## Previous version instruction + +### Raspberry Pi setup + +1. [Install go](https://www.e-tinkers.com/2019/06/better-way-to-install-golang-go-on-raspberry-pi/). + +2. Install mercurial: `sudo apt install mercurial` + +3. Fetch the repository and `cd` to the directory: `hg clone https://hg.sr.ht/~kisom/nlink && cd nlink` + +4. Make the config directory: `mkdir -p ~/.config/nomadlink` + +5. Create a new Pnut dev app from the [Pnut developer page](https://pnut.io/dev). + +6. Copy the `creds.json` file to `~/.config/nomadlink/creds.json` and fill it out with the correct details from the Pnut dev page. + +7. Run `go build`. + +8. Verify the build works by running `./nlink -i -m`. This will create the database and mark all the posts you currently have on nomad as already posted. + +9. Copy `nlink` to `/usr/local/bin`: `sudo cp nlink /usr/local/bin/` + +10. Edit your crontab (`crontab -e`) and add the following line: + +``` +*/15 * * * * /usr/local/bin/nlink -c /home/$USER/.config/nomadlink/creds.json -d /home/$USER/.config/nomadlink/nomadlink.db +``` + +11. Eventually, posts should start showing up. + +### Optional + +12. Install postfix and set it up as a local site. + +``` +sudo apt install postfix bsd-mailx +``` diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..4965fd0 --- /dev/null +++ b/go.mod @@ -0,0 +1,12 @@ +module git.wntrmute.dev/kyle/nlink + +go 1.15 + +require ( + git.sr.ht/~thrrgilag/woodstock v0.0.0-20210714032038-b22c4f10cc34 + git.wntrmute.dev/kyle/goutils v1.6.6 + github.com/anaskhan96/soup v1.2.4 + github.com/mattn/go-sqlite3 v1.10.0 + github.com/minio/minio-go/v7 v7.0.52 + github.com/mmcdole/gofeed v1.1.3 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..7d0650d --- /dev/null +++ b/go.sum @@ -0,0 +1,185 @@ +bitbucket.org/liamstask/goose v0.0.0-20150115234039-8488cc47d90c/go.mod h1:hSVuE3qU7grINVSwrmzHfpg9k87ALBk+XaualNyUzI4= +git.sr.ht/~thrrgilag/woodstock v0.0.0-20210714032038-b22c4f10cc34 h1:nTH4UWrZTwEhkFvuR4Y4vV6KMSoZuF+3qFmgGs9Vm2I= +git.sr.ht/~thrrgilag/woodstock v0.0.0-20210714032038-b22c4f10cc34/go.mod h1:giZm2mNVkz9BeA31LtUC2gwIYfUSUPQINp+tI8EGFTc= +git.wntrmute.dev/kyle/goutils v1.6.6 h1:CRCBlmSXOTkShbqC6j9lgxh4lb+khzc2zpIJYGQJtnc= +git.wntrmute.dev/kyle/goutils v1.6.6/go.mod h1:p0m2YprqMXkqtxTPKCiRcmgYo/D/9DtAIRfNVFE3JBg= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0= +github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0= +github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE= +github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= +github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= +github.com/anaskhan96/soup v1.2.4 h1:or+sKs9QbzJGZVTYFmTs2VBateEywoq00a6K14z331E= +github.com/anaskhan96/soup v1.2.4/go.mod h1:6YnEp9A2yywlYdM4EgDz9NEHclocMepEtku7wg6Cq3s= +github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo= +github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= +github.com/cloudflare/backoff v0.0.0-20161212185259-647f3cdfc87a/go.mod h1:rzgs2ZOiguV6/NpiDgADjRLPNyZlApIWxKpkT+X8SdY= +github.com/cloudflare/cfssl v1.5.0/go.mod h1:sPPkBS5L8l8sRc/IOO1jG51Xb34u+TYhL6P//JdODMQ= +github.com/cloudflare/go-metrics v0.0.0-20151117154305-6a9aea36fb41/go.mod h1:eaZPlJWD+G9wseg1BuRXlHnjntPMrywMsyxf+LTOdP4= +github.com/cloudflare/redoctober v0.0.0-20171127175943-746a508df14c/go.mod h1:6Se34jNoqrd8bTxrmJB2Bg2aoZ2CdSXonils9NsiNgo= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/daaku/go.zipexe v1.0.0/go.mod h1:z8IiR6TsVLEYKwXAoE/I+8ys/sDkgTzSL0CLnGVd57E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/getsentry/raven-go v0.0.0-20180121060056-563b81fc02b7/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/certificate-transparency-go v1.0.21/go.mod h1:QeJfpSbVSfYc7RgB3gJFj9cbuQMMchQxrWXz8Ruopmg= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jmhodges/clock v0.0.0-20160418191101-880ee4c33548/go.mod h1:hGT6jSUVzF6no3QaDSMLGLEHtHSBSefs+MgcDWnmhmo= +github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/sqlstruct v0.0.0-20150923205031-648daed35d49/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= +github.com/kisom/goutils v1.1.0/go.mod h1:+UBTfd78habUYWFbNWTJNG+jNG/i/lGURakr4A/yNRw= +github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= +github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= +github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/go-gypsy v0.0.0-20160905020020-08cad365cd28/go.mod h1:T/T7jsxVqf9k/zYOqbgNAsANsjxTd1Yq3htjDhQ1H0c= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o= +github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.52 h1:8XhG36F6oKQUDDSuz6dY3rioMzovKjW40W6ANuN0Dps= +github.com/minio/minio-go/v7 v7.0.52/go.mod h1:IbbodHyjUAguneyucUaahv+VMNs/EOTV9du7A7/Z3HU= +github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= +github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= +github.com/mmcdole/gofeed v1.1.3 h1:pdrvMb18jMSLidGp8j0pLvc9IGziX4vbmvVqmLH6z8o= +github.com/mmcdole/gofeed v1.1.3/go.mod h1:QQO3maftbOu+hiVOGOZDRLymqGQCos4zxbA4j89gMrE= +github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf h1:sWGE2v+hO0Nd4yFU/S/mDBM5plIU8v/Qhfz41hkDIAI= +github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf/go.mod h1:pasqhqstspkosTneA62Nc+2p9SOBBYAPbnmRRWPQ0V8= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= +github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E= +github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.12.0/go.mod h1:fUqqXB5vEgVCZ131L+9say31RAri6aF6KDViawhxKK8= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY= +github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/urfave/cli v1.22.3/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/weppos/publicsuffix-go v0.4.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= +github.com/weppos/publicsuffix-go v0.13.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= +github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= +github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54tB79AMBcySS0R2XIyZBAVmeHranShAFELYx7is= +github.com/zmap/zcrypto v0.0.0-20200513165325-16679db567ff/go.mod h1:TxpejqcVKQjQaVVmMGfzx5HnmFMdIU+vLtaCyPBfGI4= +github.com/zmap/zcrypto v0.0.0-20200911161511-43ff0ea04f21/go.mod h1:TxpejqcVKQjQaVVmMGfzx5HnmFMdIU+vLtaCyPBfGI4= +github.com/zmap/zlint/v2 v2.2.1/go.mod h1:ixPWsdq8qLxYRpNUTbcKig3R7WgmspsHGLhCCs6rFAM= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200124225646-8b5121be2f68/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go new file mode 100644 index 0000000..b729ebb --- /dev/null +++ b/main.go @@ -0,0 +1,127 @@ +package main + +import ( + "flag" + "path/filepath" + "time" + + "git.sr.ht/~thrrgilag/woodstock" + "git.wntrmute.dev/kyle/nlink/nomad" + + "git.wntrmute.dev/kyle/goutils/config" + "git.wntrmute.dev/kyle/goutils/die" + log "git.wntrmute.dev/kyle/goutils/syslog" +) + +var defaultRSS = "https://nomad.wntrmute.net/u/kyle.rss" + +func defaultPath(file string) string { + return filepath.Join("/perm", "nlink", file) +} + +func fetchItems(db *nomad.DB, markOnly bool) error { + rssFeed := nomad.NewURLSource(config.Get("local_rss_feed_url")) + items, err := nomad.FetchRSS(rssFeed) + if err != nil { + return nil + } + + items, err = db.Filter(nil, items) + if err != nil { + return err + } + + if items == nil { + return nil + } + + if markOnly { + tx, err := db.Begin() + if err != nil { + return err + } + + for _, item := range items { + if err = db.Mark(tx, item); err != nil { + return err + } + } + + tx.Commit() + return err + } + + client := woodstock.NewClient(config.Get("pnut_id"), config.Get("pnut_secret")) + client.SetAccessToken(config.Get("")) + + for _, item := range items { + p := nomad.NewPost(item) + err = p.Fetch() + die.If(err) + + err = p.Post(client) + die.If(err) + + err = db.Mark(nil, item) + } + + return nil +} + +const interval = 5 * time.Minute + +func main() { + var ( + configFile string + interval time.Duration + initDB bool + level string + markOnly bool + ) + + flag.StringVar(&configFile, "f", defaultPath("nlink.conf"), "`path` to config file") + flag.BoolVar(&initDB, "i", false, "initialize a new DB") + flag.StringVar(&level, "l", "INFO", "log level") + flag.BoolVar(&markOnly, "m", false, "only mark posts as having been posted") + flag.Parse() + + logOpts := &log.Options{ + Level: level, + Tag: "nlink", + Facility: "daemon", + WriteSyslog: true, + } + + if err := log.Setup(logOpts); err != nil { + die.If(err) + } + + log.Debugf("loading config file %s", configFile) + if err := config.LoadFile(configFile); err != nil { + log.Fatal(err) + } + + if initDB { + db := nomad.NewDB(config.Get("local_database")) + err := db.Create() + die.If(err) + db.Close() + } + db := nomad.NewDB(config.Get("local_database")) + err := db.Connect() + die.If(err) + defer db.Close() + + if markOnly { + if err = fetchItems(db, markOnly); err != nil { + log.Fatal(err) + } + return + } + + for { + err = fetchItems(db, markOnly) + log.Info(err) + time.Sleep(interval) + } +} diff --git a/nomad/db.go b/nomad/db.go new file mode 100644 index 0000000..d13e86a --- /dev/null +++ b/nomad/db.go @@ -0,0 +1,119 @@ +package nomad + +import ( + "database/sql" + "fmt" + + _ "github.com/mattn/go-sqlite3" +) + +type Executor interface { + Exec(stmt string, args ...interface{}) (sql.Result, error) + Query(stmt string, args ...interface{}) (*sql.Rows, error) +} + +func buildArgs(n int) string { + s := "" + for i := 0; i < n; i++ { + s += "?," + } + + return s[:len(s)-1] +} + +type DB struct { + path string + db *sql.DB +} + +func NewDB(path string) *DB { + return &DB{path: path} +} + +func (db *DB) Connect() (err error) { + if db.db != nil { + return nil + } + db.db, err = sql.Open("sqlite3", db.path) + return err +} + +func (db *DB) Close() error { + if db.db == nil { + return nil + } + err := db.db.Close() + db.db = nil + return err +} + +func (db *DB) Create() (err error) { + err = db.Connect() + if err != nil { + return err + } + + _, err = db.db.Exec(createSQL) + return err +} + +func (db *DB) Begin() (tx *sql.Tx, err error) { + err = db.Connect() + if err != nil { + return tx, err + } + + return db.db.Begin() +} + +func (db *DB) executor(tx *sql.Tx) Executor { + if tx != nil { + return tx + } + + return db.db +} + +func (db *DB) Mark(tx *sql.Tx, item Item) (err error) { + _, err = db.executor(tx).Exec(markSQL, item.URL.ID(), item.Title, item.PubDate) + return err +} + +func (db *DB) Filter(tx *sql.Tx, items []Item) (newItems []Item, err error) { + seen := map[string]bool{} + args := buildArgs(len(items)) + stmt := fmt.Sprintf(filterSQLTpl, args) + urls := make([]interface{}, 0, len(items)) + for _, item := range items { + urls = append(urls, item.URL.ID()) + } + + rows, err := db.executor(tx).Query(stmt, urls...) + if err != nil { + return nil, err + } + for rows.Next() { + var url string + err = rows.Scan(&url) + if err != nil { + return nil, err + } + seen[url] = true + } + + for _, item := range items { + if seen[item.URL.ID()] { + continue + } + newItems = append(newItems, item) + } + + // Reverse the list of items so that they are in reverse chronological order. + nitems := len(items) - 1 + for i := range items { + j := nitems - i + items[i], items[j] = items[j], items[i] + } + + return newItems, nil +} diff --git a/nomad/db_test.go b/nomad/db_test.go new file mode 100644 index 0000000..1c77a40 --- /dev/null +++ b/nomad/db_test.go @@ -0,0 +1,11 @@ +package nomad + +import "testing" + +func TestBuildArgs(t *testing.T) { + expected := "?,?,?" + args := buildArgs(3) + if expected != args { + t.Fatalf("have '%s', want '%s'", args, expected) + } +} diff --git a/nomad/item.go b/nomad/item.go new file mode 100644 index 0000000..5cfb246 --- /dev/null +++ b/nomad/item.go @@ -0,0 +1,51 @@ +package nomad + +import ( + "sort" + + "github.com/mmcdole/gofeed" +) + +type Item struct { + URL Source + Title string + PubDate int64 +} + +type itemList []Item + +func (l itemList) Len() int { + return len(l) +} + +func (l itemList) Less(i, j int) bool { + return l[i].PubDate < l[j].PubDate +} + +func (l itemList) Swap(i, j int) { + l[i], l[j] = l[j], l[i] +} + +func FetchRSS(feedSource Source) (items []Item, err error) { + rd, err := feedSource.Fetch() + if err != nil { + return nil, err + } + + feedParser := gofeed.NewParser() + feed, err := feedParser.ParseString(rd) + if err != nil { + return nil, err + } + + for _, item := range feed.Items { + items = append(items, Item{ + URL: NewSoupSource(item.Link), + Title: item.Title, + PubDate: item.PublishedParsed.Unix(), + }) + } + + sort.Sort(itemList(items)) + return items, nil +} diff --git a/nomad/item_test.go b/nomad/item_test.go new file mode 100644 index 0000000..cd37722 --- /dev/null +++ b/nomad/item_test.go @@ -0,0 +1,104 @@ +package nomad + +import "testing" + +var sampleRSS = ` + + + kyle@ nomad + https://nomad.wntrmute.net/u/kyle.rss + An experiment in MMS micropublishing + + http://www.rssboard.org/rss-specification + python-feedgen + Thu, 04 Nov 2021 16:44:41 +0000 + + kyle @ 2021-11-03 10:36:44 PDT: At least not with images, I guess. + https://nomad.wntrmute.net/p/1996 + At least not with images, I guess. + https://nomad.wntrmute.net/p/1996 + Wed, 03 Nov 2021 10:36:44 -0700 + + + kyle @ 2021-11-03 10:35:53 PDT: This thing isn't working at all... + https://nomad.wntrmute.net/p/1995 + This thing isn't working at all... + https://nomad.wntrmute.net/p/1995 + Wed, 03 Nov 2021 10:35:53 -0700 + + + kyle @ 2021-11-03 02:38:48 PDT: And a plain text post too... + https://nomad.wntrmute.net/p/1994 + And a plain text post too... + https://nomad.wntrmute.net/p/1994 + Wed, 03 Nov 2021 02:38:48 -0700 + + + kyle @ 2021-11-03 02:23:29 PDT: Working on a thing that needs a test post so here’s a picture of... + https://nomad.wntrmute.net/p/1993 + Working on a thing that needs a test post so here’s a picture of some mead. + https://nomad.wntrmute.net/p/1993 + + Wed, 03 Nov 2021 02:23:29 -0700 + + + kyle @ 2021-10-31 22:44:56 PDT: Fall colours in Oakland. + https://nomad.wntrmute.net/p/1991 + Fall colours in Oakland. + https://nomad.wntrmute.net/p/1991 + + Sun, 31 Oct 2021 22:44:56 -0700 + + + kyle @ 2021-10-25 20:02:11 PDT: I am the operator of my pocket ... operator. + https://nomad.wntrmute.net/p/1990 + I am the operator of my pocket ... operator. + https://nomad.wntrmute.net/p/1990 + + Mon, 25 Oct 2021 20:02:11 -0700 + + + kyle @ 2021-10-24 19:28:09 PDT: The Yeti being a backup battery that I got with a solar panel fo... + https://nomad.wntrmute.net/p/1989 + The Yeti being a backup battery that I got with a solar panel for offgrid stuff, good test for it though + https://nomad.wntrmute.net/p/1989 + Sun, 24 Oct 2021 19:28:09 -0700 + + + kyle @ 2021-10-24 19:27:14 PDT: Power's out, fortunately I have the Yeti + https://nomad.wntrmute.net/p/1988 + Power's out, fortunately I have the Yeti + https://nomad.wntrmute.net/p/1988 + Sun, 24 Oct 2021 19:27:14 -0700 + + + kyle @ 2021-10-24 13:22:47 PDT: Finally exploring the trail behind my apartment, of course in th... + https://nomad.wntrmute.net/p/1986 + Finally exploring the trail behind my apartment, of course in the rain. + https://nomad.wntrmute.net/p/1986 + + Sun, 24 Oct 2021 13:22:47 -0700 + + + kyle @ 2021-10-24 09:33:15 PDT: From today’s Readwise email… + https://nomad.wntrmute.net/p/1984 + From today’s Readwise email… + https://nomad.wntrmute.net/p/1984 + + Sun, 24 Oct 2021 09:33:15 -0700 + + + +` + +func TestFetchRSS(t *testing.T) { + source := NewStringSource(sampleRSS) + items, err := FetchRSS(source) + if err != nil { + t.Fatal(err) + } + + if len(items) == 0 { + t.Fatal("no items parsed from RSS") + } +} diff --git a/nomad/post.go b/nomad/post.go new file mode 100644 index 0000000..54b725e --- /dev/null +++ b/nomad/post.go @@ -0,0 +1,126 @@ +package nomad + +import ( + "bytes" + "errors" + "fmt" + "path/filepath" + + "git.sr.ht/~thrrgilag/woodstock" + "github.com/anaskhan96/soup" +) + +func nomadLink(url Source) string { + return fmt.Sprintf("[nomad.wntrmute.net](%s)", url.ID()) +} + +type sel struct { + selectors []string +} + +func selector(selectors ...string) sel { + return sel{selectors: selectors} +} + +func find(root soup.Root, attr string, selectors ...sel) (string, bool) { + result := root + for _, selector := range selectors { + result = result.Find(selector.selectors...) + if result.Pointer == nil { + return "", false + } + } + + if attr == "" { + text := result.Text() + return text, text != "" + } + + value, hasAttr := result.Attrs()[attr] + return value, hasAttr +} + +type Post struct { + Image Source + Body string + URL Source +} + +func NewPost(item Item) Post { + return Post{ + Body: nomadLink(item.URL), + URL: item.URL, + } +} + +func (p *Post) Fetch() error { + pageSource, err := p.URL.Fetch() + if err != nil { + return err + } + + root := soup.HTMLParse(pageSource) + body, hasBody := find(root, "", selector("p", "class", "entry-body")) + if hasBody { + p.Body = body + " " + p.Body + } + + imageURL, hasImageURL := find( + root, "src", + selector("div", "class", "entry-image"), + selector("img")) + + if hasImageURL { + p.Image = NewURLSource(imageURL) + } + + return nil +} + +func (p Post) Post(client *woodstock.Client) error { + pnutPost := woodstock.NewPost{ + Text: p.Body, + } + + if p.Image != nil { + imageData, err := p.Image.FetchBytes() + if err != nil { + return err + } + + params := map[string]string{ + "name": filepath.Base(p.Image.ID()), + "is_public": "1", + "type": "photo", + } + + r := bytes.NewBuffer(imageData) + file, err := client.CreateFile(params, r) + if err != nil { + panic(err.Error()) + return err + } + pnutPost.Raw = []woodstock.Raw{pnutOembedRaw(file)} + } + + result, err := client.Post(pnutPost) + if err != nil { + return err + } + + if result.Meta.ErrorMessage != "" { + return errors.New("woodstock: " + result.Meta.ErrorMessage) + } + return nil +} + +func pnutOembedRaw(file woodstock.FileResult) woodstock.Raw { + fval := map[string]string{ + "file_id": file.Data.ID, + "file_token": file.Data.FileToken, + "format": "oembed", + } + + value := map[string]map[string]string{"+io.pnut.core.file": fval} + return woodstock.Raw{Type: "io.pnut.core.oembed", Value: value} +} diff --git a/nomad/post_test.go b/nomad/post_test.go new file mode 100644 index 0000000..6adc62b --- /dev/null +++ b/nomad/post_test.go @@ -0,0 +1,97 @@ +package nomad + +import ( + "testing" +) + +var pageData = ` + + + Nomad: An MMS µblog + + + + + + + + + + + + + + +
+
+
+

Nomad

+
+
+ +
+
+ kyle + +
+ +
+ + + MMS attachment + +
+ +

Finally exploring the trail behind my apartment, of course in the rain.

+ +
+ +
+ +
+ +
+ +
+
+ + +` + +func TestFetch(t *testing.T) { + p := Post{ + Body: nomadLink(NewURLSource("https://localhost")), + URL: NewStringSource(pageData), + } + + err := p.Fetch() + if err != nil { + t.Fatal(err) + } + + expectedBody := "Finally exploring the trail behind my apartment, of course in the rain. [nomad.wntrmute.net](https://localhost)" + expectedImage := "https://nomad.sfo2.cdn.digitaloceanspaces.com/MM6f13e74faa6d643d654507c5bd9d8f53.jpg" + if p.Body != expectedBody { + t.Fatalf("have '%s', want '%s'", p.Body, expectedBody) + } + + if p.Image.ID() != expectedImage { + t.Fatalf("have '%s', want '%s'", p.Image, expectedImage) + } +} diff --git a/nomad/source.go b/nomad/source.go new file mode 100644 index 0000000..8401965 --- /dev/null +++ b/nomad/source.go @@ -0,0 +1,114 @@ +package nomad + +import ( + "crypto/sha256" + "fmt" + "io/ioutil" + "net/http" + + "github.com/anaskhan96/soup" +) + +// Source provides data for the various parsing functions. +type Source interface { + Fetch() (string, error) + FetchBytes() ([]byte, error) + ID() string + String() string +} + +// URLSource fetches data from an HTTP resource. +type URLSource struct { + url string +} + +func (u URLSource) FetchBytes() ([]byte, error) { + resp, err := http.Get(u.url) + if err != nil { + return nil, err + } + + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + return body, err +} + +func (u URLSource) Fetch() (string, error) { + body, err := u.FetchBytes() + if err != nil { + return "", err + } + + return string(body), nil +} + +func (u URLSource) ID() string { + return u.url +} + +func (u URLSource) String() string { + return u.url +} + +func NewURLSource(url string) URLSource { + return URLSource{url: url} +} + +type StringSource struct { + s string +} + +func (s StringSource) Fetch() (string, error) { + return s.String(), nil +} + +func (s StringSource) FetchBytes() ([]byte, error) { + return []byte(s.s), nil +} + +func NewStringSource(s string) *StringSource { + return &StringSource{s: s} +} + +func (s StringSource) ID() string { + h := sha256.Sum256([]byte(s.s)) + return fmt.Sprintf("%s", h[:]) +} + +func (s StringSource) String() string { + return s.s +} + +// SoupSource uses the soup package to fetch data. +type SoupSource struct { + url string +} + +func (s SoupSource) Fetch() (string, error) { + return soup.Get(s.url) +} + +func (s SoupSource) FetchBytes() ([]byte, error) { + body, err := s.Fetch() + if err != nil { + return nil, err + } + + return []byte(body), nil +} + +func (s SoupSource) ID() string { + return s.url +} + +func (s SoupSource) String() string { + return s.url +} + +func NewSoupSource(url string) SoupSource { + return SoupSource{url: url} +} diff --git a/nomad/sql.go b/nomad/sql.go new file mode 100644 index 0000000..6c0a676 --- /dev/null +++ b/nomad/sql.go @@ -0,0 +1,11 @@ +package nomad + +var createSQL = `CREATE TABLE IF NOT EXISTS posts ( + url TEXT UNIQUE NOT NULL PRIMARY KEY, + title TEXT UNIQUE NOT NULL, + pub_date INTEGER NOT NULL +);` + +var markSQL = `INSERT INTO posts (url, title, pub_date) VALUES (?, ?, ?);` + +var filterSQLTpl = `SELECT url FROM posts WHERE url IN (%s)` diff --git a/storage.go b/storage.go new file mode 100644 index 0000000..49581bc --- /dev/null +++ b/storage.go @@ -0,0 +1,73 @@ +package main + +import ( + "context" + "io" + "os" + + "git.wntrmute.dev/kyle/goutils/config" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +var minioClient *minio.Client + +func connectMinio() (err error) { + minioClient, err = minio.New(config.Get("minio_host"), &minio.Options{ + Creds: credentials.NewStaticV4( + config.Get("minio_access_key"), + config.Get("minio_secret_key"), ""), + Secure: true, + }) + + return err +} + +func restoreDatabase(path string) error { + obj, err := minioClient.GetObject( + context.Background(), + config.Get("minio_bucket"), + "nlink.db", + minio.GetObjectOptions{}, + ) + + if err != nil { + return err + } + + dbFile, err := os.Create(path) + if err != nil { + return err + } + defer dbFile.Close() + + _, err = io.Copy(dbFile, obj) + return err +} + +func saveDatabase(path string) error { + dbFile, err := os.Open(path) + if err != nil { + return err + } + defer dbFile.Close() + + fi, err := dbFile.Stat() + if err != nil { + return err + } + + _, err = minioClient.PutObject( + context.Background(), + config.Get("minio_bucket"), + "nlink.db", + dbFile, + fi.Size(), + minio.PutObjectOptions{ContentType: "application/octet-stream"}, + ) + + if err != nil { + return err + } + return nil +}