initial import
This commit is contained in:
commit
353cf2cf8a
|
|
@ -0,0 +1 @@
|
||||||
|
nlink.conf
|
||||||
|
|
@ -0,0 +1,47 @@
|
||||||
|
# nomadlink
|
||||||
|
|
||||||
|
nomadlink is a tool for syncing a Nomad RSS feed to pnut.
|
||||||
|
|
||||||
|
[Nomad](https://nomad.wntrmute.net/)
|
||||||
|
[Pnut](https://beta.pnut.io/)
|
||||||
|
|
||||||
|
The original version was running as a cronjob on a Raspberry Pi. I've
|
||||||
|
modified it to run on a gokrazy instance.
|
||||||
|
|
||||||
|
## Previous version instruction
|
||||||
|
|
||||||
|
### Raspberry Pi setup
|
||||||
|
|
||||||
|
1. [Install go](https://www.e-tinkers.com/2019/06/better-way-to-install-golang-go-on-raspberry-pi/).
|
||||||
|
|
||||||
|
2. Install mercurial: `sudo apt install mercurial`
|
||||||
|
|
||||||
|
3. Fetch the repository and `cd` to the directory: `hg clone https://hg.sr.ht/~kisom/nlink && cd nlink`
|
||||||
|
|
||||||
|
4. Make the config directory: `mkdir -p ~/.config/nomadlink`
|
||||||
|
|
||||||
|
5. Create a new Pnut dev app from the [Pnut developer page](https://pnut.io/dev).
|
||||||
|
|
||||||
|
6. Copy the `creds.json` file to `~/.config/nomadlink/creds.json` and fill it out with the correct details from the Pnut dev page.
|
||||||
|
|
||||||
|
7. Run `go build`.
|
||||||
|
|
||||||
|
8. Verify the build works by running `./nlink -i -m`. This will create the database and mark all the posts you currently have on nomad as already posted.
|
||||||
|
|
||||||
|
9. Copy `nlink` to `/usr/local/bin`: `sudo cp nlink /usr/local/bin/`
|
||||||
|
|
||||||
|
10. Edit your crontab (`crontab -e`) and add the following line:
|
||||||
|
|
||||||
|
```
|
||||||
|
*/15 * * * * /usr/local/bin/nlink -c /home/$USER/.config/nomadlink/creds.json -d /home/$USER/.config/nomadlink/nomadlink.db
|
||||||
|
```
|
||||||
|
|
||||||
|
11. Eventually, posts should start showing up.
|
||||||
|
|
||||||
|
### Optional
|
||||||
|
|
||||||
|
12. Install postfix and set it up as a local site.
|
||||||
|
|
||||||
|
```
|
||||||
|
sudo apt install postfix bsd-mailx
|
||||||
|
```
|
||||||
|
|
@ -0,0 +1,12 @@
|
||||||
|
module git.wntrmute.dev/kyle/nlink
|
||||||
|
|
||||||
|
go 1.15
|
||||||
|
|
||||||
|
require (
|
||||||
|
git.sr.ht/~thrrgilag/woodstock v0.0.0-20210714032038-b22c4f10cc34
|
||||||
|
git.wntrmute.dev/kyle/goutils v1.6.6
|
||||||
|
github.com/anaskhan96/soup v1.2.4
|
||||||
|
github.com/mattn/go-sqlite3 v1.10.0
|
||||||
|
github.com/minio/minio-go/v7 v7.0.52
|
||||||
|
github.com/mmcdole/gofeed v1.1.3
|
||||||
|
)
|
||||||
|
|
@ -0,0 +1,185 @@
|
||||||
|
bitbucket.org/liamstask/goose v0.0.0-20150115234039-8488cc47d90c/go.mod h1:hSVuE3qU7grINVSwrmzHfpg9k87ALBk+XaualNyUzI4=
|
||||||
|
git.sr.ht/~thrrgilag/woodstock v0.0.0-20210714032038-b22c4f10cc34 h1:nTH4UWrZTwEhkFvuR4Y4vV6KMSoZuF+3qFmgGs9Vm2I=
|
||||||
|
git.sr.ht/~thrrgilag/woodstock v0.0.0-20210714032038-b22c4f10cc34/go.mod h1:giZm2mNVkz9BeA31LtUC2gwIYfUSUPQINp+tI8EGFTc=
|
||||||
|
git.wntrmute.dev/kyle/goutils v1.6.6 h1:CRCBlmSXOTkShbqC6j9lgxh4lb+khzc2zpIJYGQJtnc=
|
||||||
|
git.wntrmute.dev/kyle/goutils v1.6.6/go.mod h1:p0m2YprqMXkqtxTPKCiRcmgYo/D/9DtAIRfNVFE3JBg=
|
||||||
|
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||||
|
github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0=
|
||||||
|
github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0=
|
||||||
|
github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
|
||||||
|
github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
|
||||||
|
github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c=
|
||||||
|
github.com/anaskhan96/soup v1.2.4 h1:or+sKs9QbzJGZVTYFmTs2VBateEywoq00a6K14z331E=
|
||||||
|
github.com/anaskhan96/soup v1.2.4/go.mod h1:6YnEp9A2yywlYdM4EgDz9NEHclocMepEtku7wg6Cq3s=
|
||||||
|
github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
|
||||||
|
github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
|
||||||
|
github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4=
|
||||||
|
github.com/cloudflare/backoff v0.0.0-20161212185259-647f3cdfc87a/go.mod h1:rzgs2ZOiguV6/NpiDgADjRLPNyZlApIWxKpkT+X8SdY=
|
||||||
|
github.com/cloudflare/cfssl v1.5.0/go.mod h1:sPPkBS5L8l8sRc/IOO1jG51Xb34u+TYhL6P//JdODMQ=
|
||||||
|
github.com/cloudflare/go-metrics v0.0.0-20151117154305-6a9aea36fb41/go.mod h1:eaZPlJWD+G9wseg1BuRXlHnjntPMrywMsyxf+LTOdP4=
|
||||||
|
github.com/cloudflare/redoctober v0.0.0-20171127175943-746a508df14c/go.mod h1:6Se34jNoqrd8bTxrmJB2Bg2aoZ2CdSXonils9NsiNgo=
|
||||||
|
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
|
||||||
|
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||||
|
github.com/daaku/go.zipexe v1.0.0/go.mod h1:z8IiR6TsVLEYKwXAoE/I+8ys/sDkgTzSL0CLnGVd57E=
|
||||||
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||||
|
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||||
|
github.com/getsentry/raven-go v0.0.0-20180121060056-563b81fc02b7/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
|
||||||
|
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
||||||
|
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||||
|
github.com/google/certificate-transparency-go v1.0.21/go.mod h1:QeJfpSbVSfYc7RgB3gJFj9cbuQMMchQxrWXz8Ruopmg=
|
||||||
|
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||||
|
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
|
||||||
|
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||||
|
github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE=
|
||||||
|
github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
|
||||||
|
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
||||||
|
github.com/jmhodges/clock v0.0.0-20160418191101-880ee4c33548/go.mod h1:hGT6jSUVzF6no3QaDSMLGLEHtHSBSefs+MgcDWnmhmo=
|
||||||
|
github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
|
||||||
|
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||||
|
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||||
|
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||||
|
github.com/kisielk/sqlstruct v0.0.0-20150923205031-648daed35d49/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE=
|
||||||
|
github.com/kisom/goutils v1.1.0/go.mod h1:+UBTfd78habUYWFbNWTJNG+jNG/i/lGURakr4A/yNRw=
|
||||||
|
github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4=
|
||||||
|
github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
|
||||||
|
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||||
|
github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||||
|
github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk=
|
||||||
|
github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
|
||||||
|
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||||
|
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
|
||||||
|
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||||
|
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||||
|
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||||
|
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||||
|
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||||
|
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||||
|
github.com/kylelemons/go-gypsy v0.0.0-20160905020020-08cad365cd28/go.mod h1:T/T7jsxVqf9k/zYOqbgNAsANsjxTd1Yq3htjDhQ1H0c=
|
||||||
|
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||||
|
github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||||
|
github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
|
||||||
|
github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o=
|
||||||
|
github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
|
||||||
|
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||||
|
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||||
|
github.com/minio/minio-go/v7 v7.0.52 h1:8XhG36F6oKQUDDSuz6dY3rioMzovKjW40W6ANuN0Dps=
|
||||||
|
github.com/minio/minio-go/v7 v7.0.52/go.mod h1:IbbodHyjUAguneyucUaahv+VMNs/EOTV9du7A7/Z3HU=
|
||||||
|
github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
|
||||||
|
github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
|
||||||
|
github.com/mmcdole/gofeed v1.1.3 h1:pdrvMb18jMSLidGp8j0pLvc9IGziX4vbmvVqmLH6z8o=
|
||||||
|
github.com/mmcdole/gofeed v1.1.3/go.mod h1:QQO3maftbOu+hiVOGOZDRLymqGQCos4zxbA4j89gMrE=
|
||||||
|
github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf h1:sWGE2v+hO0Nd4yFU/S/mDBM5plIU8v/Qhfz41hkDIAI=
|
||||||
|
github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf/go.mod h1:pasqhqstspkosTneA62Nc+2p9SOBBYAPbnmRRWPQ0V8=
|
||||||
|
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||||
|
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
|
||||||
|
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||||
|
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||||
|
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
|
||||||
|
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||||
|
github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8=
|
||||||
|
github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E=
|
||||||
|
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
|
||||||
|
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||||
|
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||||
|
github.com/pkg/sftp v1.12.0/go.mod h1:fUqqXB5vEgVCZ131L+9say31RAri6aF6KDViawhxKK8=
|
||||||
|
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||||
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
|
github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY=
|
||||||
|
github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
|
||||||
|
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||||
|
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
|
||||||
|
github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||||
|
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
|
||||||
|
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||||
|
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
|
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
|
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||||
|
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||||
|
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||||
|
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||||
|
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
|
||||||
|
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||||
|
github.com/urfave/cli v1.22.3/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
|
||||||
|
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||||
|
github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
|
||||||
|
github.com/weppos/publicsuffix-go v0.4.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k=
|
||||||
|
github.com/weppos/publicsuffix-go v0.13.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k=
|
||||||
|
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||||
|
github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0=
|
||||||
|
github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE=
|
||||||
|
github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54tB79AMBcySS0R2XIyZBAVmeHranShAFELYx7is=
|
||||||
|
github.com/zmap/zcrypto v0.0.0-20200513165325-16679db567ff/go.mod h1:TxpejqcVKQjQaVVmMGfzx5HnmFMdIU+vLtaCyPBfGI4=
|
||||||
|
github.com/zmap/zcrypto v0.0.0-20200911161511-43ff0ea04f21/go.mod h1:TxpejqcVKQjQaVVmMGfzx5HnmFMdIU+vLtaCyPBfGI4=
|
||||||
|
github.com/zmap/zlint/v2 v2.2.1/go.mod h1:ixPWsdq8qLxYRpNUTbcKig3R7WgmspsHGLhCCs6rFAM=
|
||||||
|
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||||
|
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||||
|
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||||
|
golang.org/x/crypto v0.0.0-20200124225646-8b5121be2f68/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||||
|
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||||
|
golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||||
|
golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||||
|
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||||
|
golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||||
|
golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc=
|
||||||
|
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
|
||||||
|
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||||
|
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||||
|
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||||
|
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||||
|
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||||
|
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||||
|
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||||
|
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||||
|
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||||
|
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||||
|
golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||||
|
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||||
|
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||||
|
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||||
|
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||||
|
golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
|
||||||
|
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||||
|
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
|
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
|
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
|
||||||
|
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||||
|
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||||
|
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||||
|
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||||
|
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||||
|
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||||
|
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||||
|
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||||
|
golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
|
||||||
|
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||||
|
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||||
|
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||||
|
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||||
|
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||||
|
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
|
google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||||
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
|
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
|
||||||
|
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
|
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
|
||||||
|
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||||
|
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||||
|
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||||
|
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
|
||||||
|
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
|
|
@ -0,0 +1,127 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"flag"
|
||||||
|
"path/filepath"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.sr.ht/~thrrgilag/woodstock"
|
||||||
|
"git.wntrmute.dev/kyle/nlink/nomad"
|
||||||
|
|
||||||
|
"git.wntrmute.dev/kyle/goutils/config"
|
||||||
|
"git.wntrmute.dev/kyle/goutils/die"
|
||||||
|
log "git.wntrmute.dev/kyle/goutils/syslog"
|
||||||
|
)
|
||||||
|
|
||||||
|
var defaultRSS = "https://nomad.wntrmute.net/u/kyle.rss"
|
||||||
|
|
||||||
|
func defaultPath(file string) string {
|
||||||
|
return filepath.Join("/perm", "nlink", file)
|
||||||
|
}
|
||||||
|
|
||||||
|
func fetchItems(db *nomad.DB, markOnly bool) error {
|
||||||
|
rssFeed := nomad.NewURLSource(config.Get("local_rss_feed_url"))
|
||||||
|
items, err := nomad.FetchRSS(rssFeed)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
items, err = db.Filter(nil, items)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if items == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if markOnly {
|
||||||
|
tx, err := db.Begin()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, item := range items {
|
||||||
|
if err = db.Mark(tx, item); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
tx.Commit()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
client := woodstock.NewClient(config.Get("pnut_id"), config.Get("pnut_secret"))
|
||||||
|
client.SetAccessToken(config.Get(""))
|
||||||
|
|
||||||
|
for _, item := range items {
|
||||||
|
p := nomad.NewPost(item)
|
||||||
|
err = p.Fetch()
|
||||||
|
die.If(err)
|
||||||
|
|
||||||
|
err = p.Post(client)
|
||||||
|
die.If(err)
|
||||||
|
|
||||||
|
err = db.Mark(nil, item)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const interval = 5 * time.Minute
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
var (
|
||||||
|
configFile string
|
||||||
|
interval time.Duration
|
||||||
|
initDB bool
|
||||||
|
level string
|
||||||
|
markOnly bool
|
||||||
|
)
|
||||||
|
|
||||||
|
flag.StringVar(&configFile, "f", defaultPath("nlink.conf"), "`path` to config file")
|
||||||
|
flag.BoolVar(&initDB, "i", false, "initialize a new DB")
|
||||||
|
flag.StringVar(&level, "l", "INFO", "log level")
|
||||||
|
flag.BoolVar(&markOnly, "m", false, "only mark posts as having been posted")
|
||||||
|
flag.Parse()
|
||||||
|
|
||||||
|
logOpts := &log.Options{
|
||||||
|
Level: level,
|
||||||
|
Tag: "nlink",
|
||||||
|
Facility: "daemon",
|
||||||
|
WriteSyslog: true,
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := log.Setup(logOpts); err != nil {
|
||||||
|
die.If(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Debugf("loading config file %s", configFile)
|
||||||
|
if err := config.LoadFile(configFile); err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if initDB {
|
||||||
|
db := nomad.NewDB(config.Get("local_database"))
|
||||||
|
err := db.Create()
|
||||||
|
die.If(err)
|
||||||
|
db.Close()
|
||||||
|
}
|
||||||
|
db := nomad.NewDB(config.Get("local_database"))
|
||||||
|
err := db.Connect()
|
||||||
|
die.If(err)
|
||||||
|
defer db.Close()
|
||||||
|
|
||||||
|
if markOnly {
|
||||||
|
if err = fetchItems(db, markOnly); err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
for {
|
||||||
|
err = fetchItems(db, markOnly)
|
||||||
|
log.Info(err)
|
||||||
|
time.Sleep(interval)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,119 @@
|
||||||
|
package nomad
|
||||||
|
|
||||||
|
import (
|
||||||
|
"database/sql"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
_ "github.com/mattn/go-sqlite3"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Executor interface {
|
||||||
|
Exec(stmt string, args ...interface{}) (sql.Result, error)
|
||||||
|
Query(stmt string, args ...interface{}) (*sql.Rows, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildArgs(n int) string {
|
||||||
|
s := ""
|
||||||
|
for i := 0; i < n; i++ {
|
||||||
|
s += "?,"
|
||||||
|
}
|
||||||
|
|
||||||
|
return s[:len(s)-1]
|
||||||
|
}
|
||||||
|
|
||||||
|
type DB struct {
|
||||||
|
path string
|
||||||
|
db *sql.DB
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewDB(path string) *DB {
|
||||||
|
return &DB{path: path}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (db *DB) Connect() (err error) {
|
||||||
|
if db.db != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
db.db, err = sql.Open("sqlite3", db.path)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (db *DB) Close() error {
|
||||||
|
if db.db == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
err := db.db.Close()
|
||||||
|
db.db = nil
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (db *DB) Create() (err error) {
|
||||||
|
err = db.Connect()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = db.db.Exec(createSQL)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (db *DB) Begin() (tx *sql.Tx, err error) {
|
||||||
|
err = db.Connect()
|
||||||
|
if err != nil {
|
||||||
|
return tx, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return db.db.Begin()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (db *DB) executor(tx *sql.Tx) Executor {
|
||||||
|
if tx != nil {
|
||||||
|
return tx
|
||||||
|
}
|
||||||
|
|
||||||
|
return db.db
|
||||||
|
}
|
||||||
|
|
||||||
|
func (db *DB) Mark(tx *sql.Tx, item Item) (err error) {
|
||||||
|
_, err = db.executor(tx).Exec(markSQL, item.URL.ID(), item.Title, item.PubDate)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (db *DB) Filter(tx *sql.Tx, items []Item) (newItems []Item, err error) {
|
||||||
|
seen := map[string]bool{}
|
||||||
|
args := buildArgs(len(items))
|
||||||
|
stmt := fmt.Sprintf(filterSQLTpl, args)
|
||||||
|
urls := make([]interface{}, 0, len(items))
|
||||||
|
for _, item := range items {
|
||||||
|
urls = append(urls, item.URL.ID())
|
||||||
|
}
|
||||||
|
|
||||||
|
rows, err := db.executor(tx).Query(stmt, urls...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for rows.Next() {
|
||||||
|
var url string
|
||||||
|
err = rows.Scan(&url)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
seen[url] = true
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, item := range items {
|
||||||
|
if seen[item.URL.ID()] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
newItems = append(newItems, item)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reverse the list of items so that they are in reverse chronological order.
|
||||||
|
nitems := len(items) - 1
|
||||||
|
for i := range items {
|
||||||
|
j := nitems - i
|
||||||
|
items[i], items[j] = items[j], items[i]
|
||||||
|
}
|
||||||
|
|
||||||
|
return newItems, nil
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,11 @@
|
||||||
|
package nomad
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestBuildArgs(t *testing.T) {
|
||||||
|
expected := "?,?,?"
|
||||||
|
args := buildArgs(3)
|
||||||
|
if expected != args {
|
||||||
|
t.Fatalf("have '%s', want '%s'", args, expected)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,51 @@
|
||||||
|
package nomad
|
||||||
|
|
||||||
|
import (
|
||||||
|
"sort"
|
||||||
|
|
||||||
|
"github.com/mmcdole/gofeed"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Item struct {
|
||||||
|
URL Source
|
||||||
|
Title string
|
||||||
|
PubDate int64
|
||||||
|
}
|
||||||
|
|
||||||
|
type itemList []Item
|
||||||
|
|
||||||
|
func (l itemList) Len() int {
|
||||||
|
return len(l)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l itemList) Less(i, j int) bool {
|
||||||
|
return l[i].PubDate < l[j].PubDate
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l itemList) Swap(i, j int) {
|
||||||
|
l[i], l[j] = l[j], l[i]
|
||||||
|
}
|
||||||
|
|
||||||
|
func FetchRSS(feedSource Source) (items []Item, err error) {
|
||||||
|
rd, err := feedSource.Fetch()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
feedParser := gofeed.NewParser()
|
||||||
|
feed, err := feedParser.ParseString(rd)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, item := range feed.Items {
|
||||||
|
items = append(items, Item{
|
||||||
|
URL: NewSoupSource(item.Link),
|
||||||
|
Title: item.Title,
|
||||||
|
PubDate: item.PublishedParsed.Unix(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
sort.Sort(itemList(items))
|
||||||
|
return items, nil
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,104 @@
|
||||||
|
package nomad
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
var sampleRSS = `<?xml version='1.0' encoding='UTF-8'?>
|
||||||
|
<rss xmlns:atom="http://www.w3.org/2005/Atom" xmlns:content="http://purl.org/rss/1.0/modules/content/" version="2.0">
|
||||||
|
<channel>
|
||||||
|
<title>kyle@ nomad</title>
|
||||||
|
<link>https://nomad.wntrmute.net/u/kyle.rss</link>
|
||||||
|
<description>An experiment in MMS micropublishing</description>
|
||||||
|
<atom:link href="https://nomad.wntrmute.net/u/kyle.rss" rel="self"/>
|
||||||
|
<docs>http://www.rssboard.org/rss-specification</docs>
|
||||||
|
<generator>python-feedgen</generator>
|
||||||
|
<lastBuildDate>Thu, 04 Nov 2021 16:44:41 +0000</lastBuildDate>
|
||||||
|
<item>
|
||||||
|
<title>kyle @ 2021-11-03 10:36:44 PDT: At least not with images, I guess.</title>
|
||||||
|
<link>https://nomad.wntrmute.net/p/1996</link>
|
||||||
|
<description>At least not with images, I guess.</description>
|
||||||
|
<guid isPermaLink="true">https://nomad.wntrmute.net/p/1996</guid>
|
||||||
|
<pubDate>Wed, 03 Nov 2021 10:36:44 -0700</pubDate>
|
||||||
|
</item>
|
||||||
|
<item>
|
||||||
|
<title>kyle @ 2021-11-03 10:35:53 PDT: This thing isn't working at all...</title>
|
||||||
|
<link>https://nomad.wntrmute.net/p/1995</link>
|
||||||
|
<description>This thing isn't working at all...</description>
|
||||||
|
<guid isPermaLink="true">https://nomad.wntrmute.net/p/1995</guid>
|
||||||
|
<pubDate>Wed, 03 Nov 2021 10:35:53 -0700</pubDate>
|
||||||
|
</item>
|
||||||
|
<item>
|
||||||
|
<title>kyle @ 2021-11-03 02:38:48 PDT: And a plain text post too...</title>
|
||||||
|
<link>https://nomad.wntrmute.net/p/1994</link>
|
||||||
|
<description>And a plain text post too...</description>
|
||||||
|
<guid isPermaLink="true">https://nomad.wntrmute.net/p/1994</guid>
|
||||||
|
<pubDate>Wed, 03 Nov 2021 02:38:48 -0700</pubDate>
|
||||||
|
</item>
|
||||||
|
<item>
|
||||||
|
<title>kyle @ 2021-11-03 02:23:29 PDT: Working on a thing that needs a test post so here’s a picture of...</title>
|
||||||
|
<link>https://nomad.wntrmute.net/p/1993</link>
|
||||||
|
<description>Working on a thing that needs a test post so here’s a picture of some mead.</description>
|
||||||
|
<guid isPermaLink="true">https://nomad.wntrmute.net/p/1993</guid>
|
||||||
|
<enclosure url="https://nomad.sfo2.cdn.digitaloceanspaces.com/MM95aee9af66546a4ef9b66ea8168b359c.jpg" length="194792" type="image/jpeg"/>
|
||||||
|
<pubDate>Wed, 03 Nov 2021 02:23:29 -0700</pubDate>
|
||||||
|
</item>
|
||||||
|
<item>
|
||||||
|
<title>kyle @ 2021-10-31 22:44:56 PDT: Fall colours in Oakland.</title>
|
||||||
|
<link>https://nomad.wntrmute.net/p/1991</link>
|
||||||
|
<description>Fall colours in Oakland.</description>
|
||||||
|
<guid isPermaLink="true">https://nomad.wntrmute.net/p/1991</guid>
|
||||||
|
<enclosure url="https://nomad.sfo2.cdn.digitaloceanspaces.com/MM7095b4db336c7d03b23f2cd702eb0e0a.jpg" length="426006" type="image/jpeg"/>
|
||||||
|
<pubDate>Sun, 31 Oct 2021 22:44:56 -0700</pubDate>
|
||||||
|
</item>
|
||||||
|
<item>
|
||||||
|
<title>kyle @ 2021-10-25 20:02:11 PDT: I am the operator of my pocket ... operator.</title>
|
||||||
|
<link>https://nomad.wntrmute.net/p/1990</link>
|
||||||
|
<description>I am the operator of my pocket ... operator.</description>
|
||||||
|
<guid isPermaLink="true">https://nomad.wntrmute.net/p/1990</guid>
|
||||||
|
<enclosure url="https://nomad.sfo2.cdn.digitaloceanspaces.com/MMe8e7ad7b08b18e9df53ac97d87c8faa0.jpg" length="232659" type="image/jpeg"/>
|
||||||
|
<pubDate>Mon, 25 Oct 2021 20:02:11 -0700</pubDate>
|
||||||
|
</item>
|
||||||
|
<item>
|
||||||
|
<title>kyle @ 2021-10-24 19:28:09 PDT: The Yeti being a backup battery that I got with a solar panel fo...</title>
|
||||||
|
<link>https://nomad.wntrmute.net/p/1989</link>
|
||||||
|
<description>The Yeti being a backup battery that I got with a solar panel for offgrid stuff, good test for it though </description>
|
||||||
|
<guid isPermaLink="true">https://nomad.wntrmute.net/p/1989</guid>
|
||||||
|
<pubDate>Sun, 24 Oct 2021 19:28:09 -0700</pubDate>
|
||||||
|
</item>
|
||||||
|
<item>
|
||||||
|
<title>kyle @ 2021-10-24 19:27:14 PDT: Power's out, fortunately I have the Yeti</title>
|
||||||
|
<link>https://nomad.wntrmute.net/p/1988</link>
|
||||||
|
<description>Power's out, fortunately I have the Yeti</description>
|
||||||
|
<guid isPermaLink="true">https://nomad.wntrmute.net/p/1988</guid>
|
||||||
|
<pubDate>Sun, 24 Oct 2021 19:27:14 -0700</pubDate>
|
||||||
|
</item>
|
||||||
|
<item>
|
||||||
|
<title>kyle @ 2021-10-24 13:22:47 PDT: Finally exploring the trail behind my apartment, of course in th...</title>
|
||||||
|
<link>https://nomad.wntrmute.net/p/1986</link>
|
||||||
|
<description>Finally exploring the trail behind my apartment, of course in the rain.</description>
|
||||||
|
<guid isPermaLink="true">https://nomad.wntrmute.net/p/1986</guid>
|
||||||
|
<enclosure url="https://nomad.sfo2.cdn.digitaloceanspaces.com/MM6f13e74faa6d643d654507c5bd9d8f53.jpg" length="622207" type="image/jpeg"/>
|
||||||
|
<pubDate>Sun, 24 Oct 2021 13:22:47 -0700</pubDate>
|
||||||
|
</item>
|
||||||
|
<item>
|
||||||
|
<title>kyle @ 2021-10-24 09:33:15 PDT: From today’s Readwise email…</title>
|
||||||
|
<link>https://nomad.wntrmute.net/p/1984</link>
|
||||||
|
<description>From today’s Readwise email…</description>
|
||||||
|
<guid isPermaLink="true">https://nomad.wntrmute.net/p/1984</guid>
|
||||||
|
<enclosure url="https://nomad.sfo2.cdn.digitaloceanspaces.com/MMcc9d528ead93949ea21dd4b4aff21836.jpg" length="53704" type="image/jpeg"/>
|
||||||
|
<pubDate>Sun, 24 Oct 2021 09:33:15 -0700</pubDate>
|
||||||
|
</item>
|
||||||
|
</channel>
|
||||||
|
</rss>
|
||||||
|
`
|
||||||
|
|
||||||
|
func TestFetchRSS(t *testing.T) {
|
||||||
|
source := NewStringSource(sampleRSS)
|
||||||
|
items, err := FetchRSS(source)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(items) == 0 {
|
||||||
|
t.Fatal("no items parsed from RSS")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,126 @@
|
||||||
|
package nomad
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"path/filepath"
|
||||||
|
|
||||||
|
"git.sr.ht/~thrrgilag/woodstock"
|
||||||
|
"github.com/anaskhan96/soup"
|
||||||
|
)
|
||||||
|
|
||||||
|
func nomadLink(url Source) string {
|
||||||
|
return fmt.Sprintf("[nomad.wntrmute.net](%s)", url.ID())
|
||||||
|
}
|
||||||
|
|
||||||
|
type sel struct {
|
||||||
|
selectors []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func selector(selectors ...string) sel {
|
||||||
|
return sel{selectors: selectors}
|
||||||
|
}
|
||||||
|
|
||||||
|
func find(root soup.Root, attr string, selectors ...sel) (string, bool) {
|
||||||
|
result := root
|
||||||
|
for _, selector := range selectors {
|
||||||
|
result = result.Find(selector.selectors...)
|
||||||
|
if result.Pointer == nil {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if attr == "" {
|
||||||
|
text := result.Text()
|
||||||
|
return text, text != ""
|
||||||
|
}
|
||||||
|
|
||||||
|
value, hasAttr := result.Attrs()[attr]
|
||||||
|
return value, hasAttr
|
||||||
|
}
|
||||||
|
|
||||||
|
type Post struct {
|
||||||
|
Image Source
|
||||||
|
Body string
|
||||||
|
URL Source
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewPost(item Item) Post {
|
||||||
|
return Post{
|
||||||
|
Body: nomadLink(item.URL),
|
||||||
|
URL: item.URL,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *Post) Fetch() error {
|
||||||
|
pageSource, err := p.URL.Fetch()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
root := soup.HTMLParse(pageSource)
|
||||||
|
body, hasBody := find(root, "", selector("p", "class", "entry-body"))
|
||||||
|
if hasBody {
|
||||||
|
p.Body = body + " " + p.Body
|
||||||
|
}
|
||||||
|
|
||||||
|
imageURL, hasImageURL := find(
|
||||||
|
root, "src",
|
||||||
|
selector("div", "class", "entry-image"),
|
||||||
|
selector("img"))
|
||||||
|
|
||||||
|
if hasImageURL {
|
||||||
|
p.Image = NewURLSource(imageURL)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p Post) Post(client *woodstock.Client) error {
|
||||||
|
pnutPost := woodstock.NewPost{
|
||||||
|
Text: p.Body,
|
||||||
|
}
|
||||||
|
|
||||||
|
if p.Image != nil {
|
||||||
|
imageData, err := p.Image.FetchBytes()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
params := map[string]string{
|
||||||
|
"name": filepath.Base(p.Image.ID()),
|
||||||
|
"is_public": "1",
|
||||||
|
"type": "photo",
|
||||||
|
}
|
||||||
|
|
||||||
|
r := bytes.NewBuffer(imageData)
|
||||||
|
file, err := client.CreateFile(params, r)
|
||||||
|
if err != nil {
|
||||||
|
panic(err.Error())
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
pnutPost.Raw = []woodstock.Raw{pnutOembedRaw(file)}
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := client.Post(pnutPost)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if result.Meta.ErrorMessage != "" {
|
||||||
|
return errors.New("woodstock: " + result.Meta.ErrorMessage)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func pnutOembedRaw(file woodstock.FileResult) woodstock.Raw {
|
||||||
|
fval := map[string]string{
|
||||||
|
"file_id": file.Data.ID,
|
||||||
|
"file_token": file.Data.FileToken,
|
||||||
|
"format": "oembed",
|
||||||
|
}
|
||||||
|
|
||||||
|
value := map[string]map[string]string{"+io.pnut.core.file": fval}
|
||||||
|
return woodstock.Raw{Type: "io.pnut.core.oembed", Value: value}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,97 @@
|
||||||
|
package nomad
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
var pageData = `<!DOCTYPE html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<title>Nomad: An MMS µblog</title>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
|
|
||||||
|
<meta property="og:title" content="Finally exploring the trail behind my apartment, of course in the rain." />
|
||||||
|
<meta property="og:type" content="website" />
|
||||||
|
<meta property="og:url" content="https://nomad.wntrmute.net/p/1986" />
|
||||||
|
<meta property="og:image" content="https://nomad.sfo2.cdn.digitaloceanspaces.com/MM6f13e74faa6d643d654507c5bd9d8f53.jpg" />
|
||||||
|
|
||||||
|
<link href="/index.rss" type="application/rss+xml" rel="alternate" title="Nomad feed" />
|
||||||
|
<link href="/static/favicon.ico" rel="shortcut icon">
|
||||||
|
<link href="/static/style.css" rel="stylesheet">
|
||||||
|
</head>
|
||||||
|
|
||||||
|
<body>
|
||||||
|
<div id="wrap">
|
||||||
|
<main>
|
||||||
|
<header>
|
||||||
|
<h1><a href="/">Nomad</a></h1>
|
||||||
|
</header>
|
||||||
|
<section>
|
||||||
|
|
||||||
|
<article class="entry entry-single" id="post-1986">
|
||||||
|
<div class="entry-details">
|
||||||
|
<span class="entry-author"><a href="/u/kyle">kyle</a></span>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="entry-image">
|
||||||
|
<!-- TODO: how do we provide a useful alt? -->
|
||||||
|
<a href="https://nomad.sfo2.cdn.digitaloceanspaces.com/MM6f13e74faa6d643d654507c5bd9d8f53.jpg" target="_blank" rel="noopener">
|
||||||
|
<img src="https://nomad.sfo2.cdn.digitaloceanspaces.com/MM6f13e74faa6d643d654507c5bd9d8f53.jpg" alt="MMS attachment"/>
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<p class="entry-body">Finally exploring the trail behind my apartment, of course in the rain.</p>
|
||||||
|
|
||||||
|
<div class="entry-details">
|
||||||
|
<time datetime="2021-10-24 13:22:47 PDT">
|
||||||
|
posted 2021-10-24 13:22:47 PDT
|
||||||
|
</time>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</article>
|
||||||
|
|
||||||
|
</section>
|
||||||
|
<footer>
|
||||||
|
<p>
|
||||||
|
<a href="/about">Nomad</a> is an experiment in MMS micropublishing,
|
||||||
|
and is bolted together with
|
||||||
|
<a href="https://www.palletsprojects.com/p/flask/">Flask</a>
|
||||||
|
and <a href="https://twilio.com/">Twilio</a>; it was inspired by
|
||||||
|
<a href="https://craigmod.com/essays/sms_publishing/">Craig Mod's experiment</a>.
|
||||||
|
It is being built by
|
||||||
|
<a class="h-card u-url" rel="me" href="https://ai6ua.net/">Kyle</a> and
|
||||||
|
<a class="h-card u-url" rel="me" href="https://wallyjones.com/">Wally</a>.
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<p><small>Nomad v78</small></p>
|
||||||
|
|
||||||
|
</footer>
|
||||||
|
</main>
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
`
|
||||||
|
|
||||||
|
func TestFetch(t *testing.T) {
|
||||||
|
p := Post{
|
||||||
|
Body: nomadLink(NewURLSource("https://localhost")),
|
||||||
|
URL: NewStringSource(pageData),
|
||||||
|
}
|
||||||
|
|
||||||
|
err := p.Fetch()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
expectedBody := "Finally exploring the trail behind my apartment, of course in the rain. [nomad.wntrmute.net](https://localhost)"
|
||||||
|
expectedImage := "https://nomad.sfo2.cdn.digitaloceanspaces.com/MM6f13e74faa6d643d654507c5bd9d8f53.jpg"
|
||||||
|
if p.Body != expectedBody {
|
||||||
|
t.Fatalf("have '%s', want '%s'", p.Body, expectedBody)
|
||||||
|
}
|
||||||
|
|
||||||
|
if p.Image.ID() != expectedImage {
|
||||||
|
t.Fatalf("have '%s', want '%s'", p.Image, expectedImage)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,114 @@
|
||||||
|
package nomad
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
"fmt"
|
||||||
|
"io/ioutil"
|
||||||
|
"net/http"
|
||||||
|
|
||||||
|
"github.com/anaskhan96/soup"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Source provides data for the various parsing functions.
|
||||||
|
type Source interface {
|
||||||
|
Fetch() (string, error)
|
||||||
|
FetchBytes() ([]byte, error)
|
||||||
|
ID() string
|
||||||
|
String() string
|
||||||
|
}
|
||||||
|
|
||||||
|
// URLSource fetches data from an HTTP resource.
|
||||||
|
type URLSource struct {
|
||||||
|
url string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (u URLSource) FetchBytes() ([]byte, error) {
|
||||||
|
resp, err := http.Get(u.url)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
defer resp.Body.Close()
|
||||||
|
body, err := ioutil.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return body, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (u URLSource) Fetch() (string, error) {
|
||||||
|
body, err := u.FetchBytes()
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
return string(body), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (u URLSource) ID() string {
|
||||||
|
return u.url
|
||||||
|
}
|
||||||
|
|
||||||
|
func (u URLSource) String() string {
|
||||||
|
return u.url
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewURLSource(url string) URLSource {
|
||||||
|
return URLSource{url: url}
|
||||||
|
}
|
||||||
|
|
||||||
|
type StringSource struct {
|
||||||
|
s string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s StringSource) Fetch() (string, error) {
|
||||||
|
return s.String(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s StringSource) FetchBytes() ([]byte, error) {
|
||||||
|
return []byte(s.s), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewStringSource(s string) *StringSource {
|
||||||
|
return &StringSource{s: s}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s StringSource) ID() string {
|
||||||
|
h := sha256.Sum256([]byte(s.s))
|
||||||
|
return fmt.Sprintf("%s", h[:])
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s StringSource) String() string {
|
||||||
|
return s.s
|
||||||
|
}
|
||||||
|
|
||||||
|
// SoupSource uses the soup package to fetch data.
|
||||||
|
type SoupSource struct {
|
||||||
|
url string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s SoupSource) Fetch() (string, error) {
|
||||||
|
return soup.Get(s.url)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s SoupSource) FetchBytes() ([]byte, error) {
|
||||||
|
body, err := s.Fetch()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return []byte(body), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s SoupSource) ID() string {
|
||||||
|
return s.url
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s SoupSource) String() string {
|
||||||
|
return s.url
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewSoupSource(url string) SoupSource {
|
||||||
|
return SoupSource{url: url}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,11 @@
|
||||||
|
package nomad
|
||||||
|
|
||||||
|
var createSQL = `CREATE TABLE IF NOT EXISTS posts (
|
||||||
|
url TEXT UNIQUE NOT NULL PRIMARY KEY,
|
||||||
|
title TEXT UNIQUE NOT NULL,
|
||||||
|
pub_date INTEGER NOT NULL
|
||||||
|
);`
|
||||||
|
|
||||||
|
var markSQL = `INSERT INTO posts (url, title, pub_date) VALUES (?, ?, ?);`
|
||||||
|
|
||||||
|
var filterSQLTpl = `SELECT url FROM posts WHERE url IN (%s)`
|
||||||
|
|
@ -0,0 +1,73 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"git.wntrmute.dev/kyle/goutils/config"
|
||||||
|
"github.com/minio/minio-go/v7"
|
||||||
|
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||||
|
)
|
||||||
|
|
||||||
|
var minioClient *minio.Client
|
||||||
|
|
||||||
|
func connectMinio() (err error) {
|
||||||
|
minioClient, err = minio.New(config.Get("minio_host"), &minio.Options{
|
||||||
|
Creds: credentials.NewStaticV4(
|
||||||
|
config.Get("minio_access_key"),
|
||||||
|
config.Get("minio_secret_key"), ""),
|
||||||
|
Secure: true,
|
||||||
|
})
|
||||||
|
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func restoreDatabase(path string) error {
|
||||||
|
obj, err := minioClient.GetObject(
|
||||||
|
context.Background(),
|
||||||
|
config.Get("minio_bucket"),
|
||||||
|
"nlink.db",
|
||||||
|
minio.GetObjectOptions{},
|
||||||
|
)
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
dbFile, err := os.Create(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer dbFile.Close()
|
||||||
|
|
||||||
|
_, err = io.Copy(dbFile, obj)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func saveDatabase(path string) error {
|
||||||
|
dbFile, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer dbFile.Close()
|
||||||
|
|
||||||
|
fi, err := dbFile.Stat()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = minioClient.PutObject(
|
||||||
|
context.Background(),
|
||||||
|
config.Get("minio_bucket"),
|
||||||
|
"nlink.db",
|
||||||
|
dbFile,
|
||||||
|
fi.Size(),
|
||||||
|
minio.PutObjectOptions{ContentType: "application/octet-stream"},
|
||||||
|
)
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue