Add server design doc for sync and web viewing
Covers: gRPC sync API (full notebook push), REST API for web viewing, SVG/JPG/PDF rendering, password + FIDO2/U2F auth via WebAuthn, shareable links with optional expiry, Android app integration points. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
420
docs/SERVER_DESIGN.md
Normal file
420
docs/SERVER_DESIGN.md
Normal file
@@ -0,0 +1,420 @@
|
|||||||
|
# eng-pad Server Design
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
The eng-pad server (`eng-pad-server`) receives notebook data from the
|
||||||
|
Android app via gRPC, stores it, and serves read-only views via a web
|
||||||
|
UI. Users authenticate with password or FIDO2/U2F keys. Shareable links
|
||||||
|
allow unauthenticated read-only access to specific notebooks.
|
||||||
|
|
||||||
|
This is a standalone service. It may integrate with MCIAS in the future
|
||||||
|
but does not depend on it initially.
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
```
|
||||||
|
Android App Server
|
||||||
|
+-----------+ gRPC/TLS +------------------+
|
||||||
|
| eng-pad | ------------> | eng-pad-server |
|
||||||
|
| (writer) | SyncNotebook | (read-only store)|
|
||||||
|
+-----------+ +------------------+
|
||||||
|
|
|
||||||
|
+-----+------+
|
||||||
|
| |
|
||||||
|
REST API Web UI (htmx)
|
||||||
|
(JSON) (SVG rendering)
|
||||||
|
```
|
||||||
|
|
||||||
|
The app is the single writer. The server receives whatever the app
|
||||||
|
pushes and serves it read-only. No conflict resolution needed.
|
||||||
|
|
||||||
|
## Data Model
|
||||||
|
|
||||||
|
### SQLite Schema
|
||||||
|
|
||||||
|
```sql
|
||||||
|
-- Users (password + optional FIDO2 keys)
|
||||||
|
CREATE TABLE users (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
username TEXT NOT NULL UNIQUE,
|
||||||
|
password_hash TEXT NOT NULL, -- Argon2id
|
||||||
|
created_at INTEGER NOT NULL,
|
||||||
|
updated_at INTEGER NOT NULL
|
||||||
|
);
|
||||||
|
|
||||||
|
-- FIDO2/U2F credentials (one user can have many keys)
|
||||||
|
CREATE TABLE webauthn_credentials (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||||
|
credential_id BLOB NOT NULL UNIQUE,
|
||||||
|
public_key BLOB NOT NULL,
|
||||||
|
name TEXT NOT NULL, -- User-assigned label ("YubiKey 5")
|
||||||
|
sign_count INTEGER NOT NULL DEFAULT 0,
|
||||||
|
created_at INTEGER NOT NULL
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Synced notebooks
|
||||||
|
CREATE TABLE notebooks (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||||
|
remote_id INTEGER NOT NULL, -- ID from the Android app's DB
|
||||||
|
title TEXT NOT NULL,
|
||||||
|
page_size TEXT NOT NULL, -- "REGULAR" or "LARGE"
|
||||||
|
synced_at INTEGER NOT NULL,
|
||||||
|
UNIQUE(user_id, remote_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Synced pages
|
||||||
|
CREATE TABLE pages (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
notebook_id INTEGER NOT NULL REFERENCES notebooks(id) ON DELETE CASCADE,
|
||||||
|
remote_id INTEGER NOT NULL, -- ID from the Android app's DB
|
||||||
|
page_number INTEGER NOT NULL,
|
||||||
|
UNIQUE(notebook_id, remote_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Synced strokes
|
||||||
|
CREATE TABLE strokes (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE,
|
||||||
|
pen_size REAL NOT NULL,
|
||||||
|
color INTEGER NOT NULL,
|
||||||
|
style TEXT NOT NULL DEFAULT 'plain',
|
||||||
|
point_data BLOB NOT NULL, -- Packed float array, same format as app
|
||||||
|
stroke_order INTEGER NOT NULL
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Shareable links
|
||||||
|
CREATE TABLE share_links (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
notebook_id INTEGER NOT NULL REFERENCES notebooks(id) ON DELETE CASCADE,
|
||||||
|
token TEXT NOT NULL UNIQUE, -- Random URL-safe token
|
||||||
|
expires_at INTEGER, -- NULL = never expires
|
||||||
|
created_at INTEGER NOT NULL
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX idx_notebooks_user ON notebooks(user_id);
|
||||||
|
CREATE INDEX idx_pages_notebook ON pages(notebook_id);
|
||||||
|
CREATE INDEX idx_strokes_page ON strokes(page_id);
|
||||||
|
CREATE INDEX idx_share_links_token ON share_links(token);
|
||||||
|
CREATE INDEX idx_webauthn_user ON webauthn_credentials(user_id);
|
||||||
|
```
|
||||||
|
|
||||||
|
## gRPC API (Sync)
|
||||||
|
|
||||||
|
Proto definitions in `proto/engpad/v1/`.
|
||||||
|
|
||||||
|
```protobuf
|
||||||
|
syntax = "proto3";
|
||||||
|
package engpad.v1;
|
||||||
|
|
||||||
|
import "google/protobuf/timestamp.proto";
|
||||||
|
|
||||||
|
service EngPadSync {
|
||||||
|
// Push a complete notebook (pages + strokes). Replaces any
|
||||||
|
// existing data for this notebook on the server.
|
||||||
|
rpc SyncNotebook(SyncNotebookRequest) returns (SyncNotebookResponse);
|
||||||
|
|
||||||
|
// Delete a notebook from the server.
|
||||||
|
rpc DeleteNotebook(DeleteNotebookRequest) returns (DeleteNotebookResponse);
|
||||||
|
|
||||||
|
// List notebooks synced by the authenticated user.
|
||||||
|
rpc ListNotebooks(ListNotebooksRequest) returns (ListNotebooksResponse);
|
||||||
|
|
||||||
|
// Generate a shareable link for a notebook.
|
||||||
|
rpc CreateShareLink(CreateShareLinkRequest) returns (CreateShareLinkResponse);
|
||||||
|
|
||||||
|
// Revoke a shareable link.
|
||||||
|
rpc RevokeShareLink(RevokeShareLinkRequest) returns (RevokeShareLinkResponse);
|
||||||
|
|
||||||
|
// List active share links for a notebook.
|
||||||
|
rpc ListShareLinks(ListShareLinksRequest) returns (ListShareLinksResponse);
|
||||||
|
}
|
||||||
|
|
||||||
|
message SyncNotebookRequest {
|
||||||
|
int64 notebook_id = 1; // App-side notebook ID
|
||||||
|
string title = 2;
|
||||||
|
string page_size = 3; // "REGULAR" or "LARGE"
|
||||||
|
repeated PageData pages = 4;
|
||||||
|
}
|
||||||
|
|
||||||
|
message PageData {
|
||||||
|
int64 page_id = 1; // App-side page ID
|
||||||
|
int32 page_number = 2;
|
||||||
|
repeated StrokeData strokes = 3;
|
||||||
|
}
|
||||||
|
|
||||||
|
message StrokeData {
|
||||||
|
float pen_size = 1;
|
||||||
|
int32 color = 2;
|
||||||
|
string style = 3; // "plain", "dashed", "arrow", "double_arrow"
|
||||||
|
bytes point_data = 4; // Packed little-endian floats
|
||||||
|
int32 stroke_order = 5;
|
||||||
|
}
|
||||||
|
|
||||||
|
message SyncNotebookResponse {
|
||||||
|
int64 server_notebook_id = 1;
|
||||||
|
google.protobuf.Timestamp synced_at = 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
message DeleteNotebookRequest {
|
||||||
|
int64 notebook_id = 1; // App-side notebook ID
|
||||||
|
}
|
||||||
|
|
||||||
|
message DeleteNotebookResponse {}
|
||||||
|
|
||||||
|
message ListNotebooksRequest {}
|
||||||
|
|
||||||
|
message ListNotebooksResponse {
|
||||||
|
repeated NotebookSummary notebooks = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
message NotebookSummary {
|
||||||
|
int64 server_id = 1;
|
||||||
|
int64 remote_id = 2;
|
||||||
|
string title = 3;
|
||||||
|
string page_size = 4;
|
||||||
|
int32 page_count = 5;
|
||||||
|
google.protobuf.Timestamp synced_at = 6;
|
||||||
|
}
|
||||||
|
|
||||||
|
message CreateShareLinkRequest {
|
||||||
|
int64 notebook_id = 1; // App-side notebook ID
|
||||||
|
int64 expires_in_seconds = 2; // 0 = never expires
|
||||||
|
}
|
||||||
|
|
||||||
|
message CreateShareLinkResponse {
|
||||||
|
string token = 1;
|
||||||
|
string url = 2; // Full shareable URL
|
||||||
|
google.protobuf.Timestamp expires_at = 3; // Not set if never expires
|
||||||
|
}
|
||||||
|
|
||||||
|
message RevokeShareLinkRequest {
|
||||||
|
string token = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
message RevokeShareLinkResponse {}
|
||||||
|
|
||||||
|
message ListShareLinksRequest {
|
||||||
|
int64 notebook_id = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
message ListShareLinksResponse {
|
||||||
|
repeated ShareLinkInfo links = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
message ShareLinkInfo {
|
||||||
|
string token = 1;
|
||||||
|
string url = 2;
|
||||||
|
google.protobuf.Timestamp created_at = 3;
|
||||||
|
google.protobuf.Timestamp expires_at = 4;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Sync Semantics
|
||||||
|
|
||||||
|
`SyncNotebook` is an upsert: the server replaces all data for the
|
||||||
|
given `notebook_id` (keyed by user + app-side ID). The entire notebook
|
||||||
|
is sent each time — no incremental sync. This is simple and correct
|
||||||
|
since the app is the sole writer.
|
||||||
|
|
||||||
|
For a notebook with ~100 pages and ~500 strokes per page, the payload
|
||||||
|
is roughly 5-10 MB — manageable for manual sync over WiFi.
|
||||||
|
|
||||||
|
## REST API (Web Viewing)
|
||||||
|
|
||||||
|
| Method | Path | Auth | Description |
|
||||||
|
|--------|------|------|-------------|
|
||||||
|
| `POST` | `/v1/auth/login` | None | Password login, returns bearer token |
|
||||||
|
| `POST` | `/v1/auth/webauthn/register/begin` | Bearer | Start FIDO2 registration |
|
||||||
|
| `POST` | `/v1/auth/webauthn/register/finish` | Bearer | Complete FIDO2 registration |
|
||||||
|
| `POST` | `/v1/auth/webauthn/login/begin` | None | Start FIDO2 login |
|
||||||
|
| `POST` | `/v1/auth/webauthn/login/finish` | None | Complete FIDO2 login |
|
||||||
|
| `GET` | `/v1/notebooks` | Bearer | List user's synced notebooks |
|
||||||
|
| `GET` | `/v1/notebooks/:id` | Bearer | Notebook metadata + page list |
|
||||||
|
| `GET` | `/v1/notebooks/:id/pages/:num` | Bearer | Page metadata + stroke count |
|
||||||
|
| `GET` | `/v1/notebooks/:id/pages/:num/svg` | Bearer | Page rendered as SVG |
|
||||||
|
| `GET` | `/v1/notebooks/:id/pages/:num/jpg` | Bearer | Page rendered as JPG (300 DPI) |
|
||||||
|
| `GET` | `/v1/notebooks/:id/pdf` | Bearer | Full notebook as PDF |
|
||||||
|
| `GET` | `/s/:token` | None | Shareable link → notebook view |
|
||||||
|
| `GET` | `/s/:token/pages/:num/svg` | None | Shareable link → page SVG |
|
||||||
|
| `GET` | `/s/:token/pages/:num/jpg` | None | Shareable link → page JPG |
|
||||||
|
| `GET` | `/s/:token/pdf` | None | Shareable link → notebook PDF |
|
||||||
|
|
||||||
|
### SVG Rendering
|
||||||
|
|
||||||
|
Each page is rendered as an SVG document:
|
||||||
|
|
||||||
|
```xml
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg"
|
||||||
|
viewBox="0 0 612 792"
|
||||||
|
width="612" height="792">
|
||||||
|
<!-- Coordinates scaled from 300 DPI to 72 DPI (×0.24) for SVG -->
|
||||||
|
<path d="M x0 y0 L x1 y1 L x2 y2 ..."
|
||||||
|
stroke="black" stroke-width="1.08"
|
||||||
|
stroke-linecap="round" stroke-linejoin="round"
|
||||||
|
fill="none" />
|
||||||
|
<!-- Dashed strokes -->
|
||||||
|
<path d="M x0 y0 L x1 y1"
|
||||||
|
stroke="black" stroke-width="1.08"
|
||||||
|
stroke-dasharray="7.2 4.8"
|
||||||
|
fill="none" />
|
||||||
|
<!-- Arrow heads as separate paths -->
|
||||||
|
</svg>
|
||||||
|
```
|
||||||
|
|
||||||
|
SVG coordinates use 72 DPI (PDF points) for consistency with PDF export.
|
||||||
|
The `viewBox` maps to physical page dimensions. Browsers handle
|
||||||
|
zoom/pan natively via scroll and pinch.
|
||||||
|
|
||||||
|
### JPG and PDF Rendering
|
||||||
|
|
||||||
|
Server-side rendering using Go's `image` package (JPG) and a PDF
|
||||||
|
library like `go-pdf` or `jung-kurt/gofpdf` (PDF). Same rendering
|
||||||
|
logic as the Android app — scale coordinates by 72/300 for PDF,
|
||||||
|
render at 300 DPI for JPG.
|
||||||
|
|
||||||
|
## Authentication
|
||||||
|
|
||||||
|
### Password Auth
|
||||||
|
|
||||||
|
- Argon2id hashing (memory-hard, tuned to hardware)
|
||||||
|
- Bearer tokens returned on login, stored as session cookies for web UI
|
||||||
|
- Token validation via SHA-256 keyed lookup with short TTL cache
|
||||||
|
|
||||||
|
### FIDO2/U2F (WebAuthn)
|
||||||
|
|
||||||
|
Users can register multiple security keys after initial password login.
|
||||||
|
Once registered, keys can be used as an alternative login method.
|
||||||
|
|
||||||
|
Implementation via the `go-webauthn/webauthn` library:
|
||||||
|
|
||||||
|
1. **Registration flow**:
|
||||||
|
- User logs in with password
|
||||||
|
- Calls `POST /v1/auth/webauthn/register/begin` → gets challenge
|
||||||
|
- Browser/client signs challenge with security key
|
||||||
|
- Calls `POST /v1/auth/webauthn/register/finish` with attestation
|
||||||
|
- Server stores credential in `webauthn_credentials` table
|
||||||
|
|
||||||
|
2. **Login flow**:
|
||||||
|
- User calls `POST /v1/auth/webauthn/login/begin` with username → gets challenge
|
||||||
|
- Browser signs challenge with registered key
|
||||||
|
- Calls `POST /v1/auth/webauthn/login/finish` with assertion
|
||||||
|
- Server validates, returns bearer token
|
||||||
|
|
||||||
|
A user can have any number of registered keys. Each key has a
|
||||||
|
user-assigned name for identification.
|
||||||
|
|
||||||
|
## Shareable Links
|
||||||
|
|
||||||
|
Tokens are 32-byte random values encoded as URL-safe base64 (43 chars).
|
||||||
|
Generated via `crypto/rand`.
|
||||||
|
|
||||||
|
- Default expiry: never (0 = no expiry)
|
||||||
|
- Optional expiry set at creation time
|
||||||
|
- Expired links return 410 Gone
|
||||||
|
- Links can be revoked via the gRPC API or web UI
|
||||||
|
- Each link is scoped to a single notebook
|
||||||
|
|
||||||
|
The shareable URL format: `https://pad.metacircular.net/s/{token}`
|
||||||
|
|
||||||
|
The web view for shared links shows the notebook title, page navigation,
|
||||||
|
and export options (SVG view, JPG download, PDF download) — identical
|
||||||
|
to the authenticated view but without the notebook list or share
|
||||||
|
management.
|
||||||
|
|
||||||
|
## Android App Changes
|
||||||
|
|
||||||
|
### Sync Integration
|
||||||
|
|
||||||
|
New files:
|
||||||
|
- `internal/sync/SyncClient.kt` — gRPC client for `EngPadSync`
|
||||||
|
- `internal/sync/SyncManager.kt` — orchestrates notebook serialization
|
||||||
|
and sync
|
||||||
|
|
||||||
|
### UI Changes
|
||||||
|
|
||||||
|
- Notebook overflow menu: add "Sync to server" option
|
||||||
|
- Library toolbar: add "Sync all" button
|
||||||
|
- Settings screen: server URL, credentials
|
||||||
|
- Sync status indicator on notebook cards (synced/unsynced/error)
|
||||||
|
|
||||||
|
### Configuration
|
||||||
|
|
||||||
|
Server URL and auth token stored in `EngPadApp` SharedPreferences.
|
||||||
|
First sync prompts for server URL and login credentials.
|
||||||
|
|
||||||
|
## Server Repository Layout
|
||||||
|
|
||||||
|
```
|
||||||
|
eng-pad-server/
|
||||||
|
├── cmd/
|
||||||
|
│ └── eng-pad-server/ CLI entry point
|
||||||
|
├── internal/
|
||||||
|
│ ├── auth/ Password + WebAuthn auth
|
||||||
|
│ ├── config/ TOML config
|
||||||
|
│ ├── db/ SQLite setup, migrations
|
||||||
|
│ ├── grpcserver/ gRPC sync service
|
||||||
|
│ ├── server/ REST API routes
|
||||||
|
│ ├── render/ SVG, JPG, PDF rendering
|
||||||
|
│ ├── share/ Share link management
|
||||||
|
│ └── webserver/ htmx web UI
|
||||||
|
├── proto/engpad/v1/ Protobuf definitions
|
||||||
|
├── gen/engpad/v1/ Generated Go code
|
||||||
|
├── web/
|
||||||
|
│ ├── templates/ HTML templates
|
||||||
|
│ └── static/ CSS, htmx
|
||||||
|
├── deploy/
|
||||||
|
│ ├── docker/
|
||||||
|
│ ├── systemd/
|
||||||
|
│ └── scripts/
|
||||||
|
├── Makefile
|
||||||
|
├── buf.yaml
|
||||||
|
├── CLAUDE.md
|
||||||
|
├── ARCHITECTURE.md
|
||||||
|
└── eng-pad-server.toml.example
|
||||||
|
```
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[server]
|
||||||
|
listen_addr = ":8443"
|
||||||
|
grpc_addr = ":9443"
|
||||||
|
tls_cert = "/srv/eng-pad-server/certs/cert.pem"
|
||||||
|
tls_key = "/srv/eng-pad-server/certs/key.pem"
|
||||||
|
|
||||||
|
[database]
|
||||||
|
path = "/srv/eng-pad-server/eng-pad-server.db"
|
||||||
|
|
||||||
|
[web]
|
||||||
|
listen_addr = ":8080"
|
||||||
|
base_url = "https://pad.metacircular.net"
|
||||||
|
|
||||||
|
[auth]
|
||||||
|
token_ttl = "24h"
|
||||||
|
argon2_memory = 65536
|
||||||
|
argon2_time = 3
|
||||||
|
argon2_threads = 4
|
||||||
|
|
||||||
|
[webauthn]
|
||||||
|
rp_display_name = "Engineering Pad"
|
||||||
|
rp_id = "pad.metacircular.net"
|
||||||
|
rp_origins = ["https://pad.metacircular.net"]
|
||||||
|
|
||||||
|
[log]
|
||||||
|
level = "info"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Open Questions
|
||||||
|
|
||||||
|
1. Should the web UI show the grid? It's excluded from PDF/JPG export
|
||||||
|
but might be useful for on-screen viewing. Could be a toggle.
|
||||||
|
|
||||||
|
2. Should share links support per-page scope in addition to whole
|
||||||
|
notebook? The user said no for now, but the URL structure supports
|
||||||
|
it (`/s/:token/pages/:num`).
|
||||||
|
|
||||||
|
3. Should the server support deleting individual pages, or only
|
||||||
|
full notebook replacement via sync? Full replacement is simpler
|
||||||
|
and matches the "server receives whatever the app pushes" model.
|
||||||
Reference in New Issue
Block a user