Add server design doc for sync and web viewing
Covers: gRPC sync API (full notebook push), REST API for web viewing, SVG/JPG/PDF rendering, password + FIDO2/U2F auth via WebAuthn, shareable links with optional expiry, Android app integration points. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
420
docs/SERVER_DESIGN.md
Normal file
420
docs/SERVER_DESIGN.md
Normal file
@@ -0,0 +1,420 @@
|
||||
# eng-pad Server Design
|
||||
|
||||
## Overview
|
||||
|
||||
The eng-pad server (`eng-pad-server`) receives notebook data from the
|
||||
Android app via gRPC, stores it, and serves read-only views via a web
|
||||
UI. Users authenticate with password or FIDO2/U2F keys. Shareable links
|
||||
allow unauthenticated read-only access to specific notebooks.
|
||||
|
||||
This is a standalone service. It may integrate with MCIAS in the future
|
||||
but does not depend on it initially.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
Android App Server
|
||||
+-----------+ gRPC/TLS +------------------+
|
||||
| eng-pad | ------------> | eng-pad-server |
|
||||
| (writer) | SyncNotebook | (read-only store)|
|
||||
+-----------+ +------------------+
|
||||
|
|
||||
+-----+------+
|
||||
| |
|
||||
REST API Web UI (htmx)
|
||||
(JSON) (SVG rendering)
|
||||
```
|
||||
|
||||
The app is the single writer. The server receives whatever the app
|
||||
pushes and serves it read-only. No conflict resolution needed.
|
||||
|
||||
## Data Model
|
||||
|
||||
### SQLite Schema
|
||||
|
||||
```sql
|
||||
-- Users (password + optional FIDO2 keys)
|
||||
CREATE TABLE users (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
username TEXT NOT NULL UNIQUE,
|
||||
password_hash TEXT NOT NULL, -- Argon2id
|
||||
created_at INTEGER NOT NULL,
|
||||
updated_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
-- FIDO2/U2F credentials (one user can have many keys)
|
||||
CREATE TABLE webauthn_credentials (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
credential_id BLOB NOT NULL UNIQUE,
|
||||
public_key BLOB NOT NULL,
|
||||
name TEXT NOT NULL, -- User-assigned label ("YubiKey 5")
|
||||
sign_count INTEGER NOT NULL DEFAULT 0,
|
||||
created_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
-- Synced notebooks
|
||||
CREATE TABLE notebooks (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
remote_id INTEGER NOT NULL, -- ID from the Android app's DB
|
||||
title TEXT NOT NULL,
|
||||
page_size TEXT NOT NULL, -- "REGULAR" or "LARGE"
|
||||
synced_at INTEGER NOT NULL,
|
||||
UNIQUE(user_id, remote_id)
|
||||
);
|
||||
|
||||
-- Synced pages
|
||||
CREATE TABLE pages (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
notebook_id INTEGER NOT NULL REFERENCES notebooks(id) ON DELETE CASCADE,
|
||||
remote_id INTEGER NOT NULL, -- ID from the Android app's DB
|
||||
page_number INTEGER NOT NULL,
|
||||
UNIQUE(notebook_id, remote_id)
|
||||
);
|
||||
|
||||
-- Synced strokes
|
||||
CREATE TABLE strokes (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE,
|
||||
pen_size REAL NOT NULL,
|
||||
color INTEGER NOT NULL,
|
||||
style TEXT NOT NULL DEFAULT 'plain',
|
||||
point_data BLOB NOT NULL, -- Packed float array, same format as app
|
||||
stroke_order INTEGER NOT NULL
|
||||
);
|
||||
|
||||
-- Shareable links
|
||||
CREATE TABLE share_links (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
notebook_id INTEGER NOT NULL REFERENCES notebooks(id) ON DELETE CASCADE,
|
||||
token TEXT NOT NULL UNIQUE, -- Random URL-safe token
|
||||
expires_at INTEGER, -- NULL = never expires
|
||||
created_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX idx_notebooks_user ON notebooks(user_id);
|
||||
CREATE INDEX idx_pages_notebook ON pages(notebook_id);
|
||||
CREATE INDEX idx_strokes_page ON strokes(page_id);
|
||||
CREATE INDEX idx_share_links_token ON share_links(token);
|
||||
CREATE INDEX idx_webauthn_user ON webauthn_credentials(user_id);
|
||||
```
|
||||
|
||||
## gRPC API (Sync)
|
||||
|
||||
Proto definitions in `proto/engpad/v1/`.
|
||||
|
||||
```protobuf
|
||||
syntax = "proto3";
|
||||
package engpad.v1;
|
||||
|
||||
import "google/protobuf/timestamp.proto";
|
||||
|
||||
service EngPadSync {
|
||||
// Push a complete notebook (pages + strokes). Replaces any
|
||||
// existing data for this notebook on the server.
|
||||
rpc SyncNotebook(SyncNotebookRequest) returns (SyncNotebookResponse);
|
||||
|
||||
// Delete a notebook from the server.
|
||||
rpc DeleteNotebook(DeleteNotebookRequest) returns (DeleteNotebookResponse);
|
||||
|
||||
// List notebooks synced by the authenticated user.
|
||||
rpc ListNotebooks(ListNotebooksRequest) returns (ListNotebooksResponse);
|
||||
|
||||
// Generate a shareable link for a notebook.
|
||||
rpc CreateShareLink(CreateShareLinkRequest) returns (CreateShareLinkResponse);
|
||||
|
||||
// Revoke a shareable link.
|
||||
rpc RevokeShareLink(RevokeShareLinkRequest) returns (RevokeShareLinkResponse);
|
||||
|
||||
// List active share links for a notebook.
|
||||
rpc ListShareLinks(ListShareLinksRequest) returns (ListShareLinksResponse);
|
||||
}
|
||||
|
||||
message SyncNotebookRequest {
|
||||
int64 notebook_id = 1; // App-side notebook ID
|
||||
string title = 2;
|
||||
string page_size = 3; // "REGULAR" or "LARGE"
|
||||
repeated PageData pages = 4;
|
||||
}
|
||||
|
||||
message PageData {
|
||||
int64 page_id = 1; // App-side page ID
|
||||
int32 page_number = 2;
|
||||
repeated StrokeData strokes = 3;
|
||||
}
|
||||
|
||||
message StrokeData {
|
||||
float pen_size = 1;
|
||||
int32 color = 2;
|
||||
string style = 3; // "plain", "dashed", "arrow", "double_arrow"
|
||||
bytes point_data = 4; // Packed little-endian floats
|
||||
int32 stroke_order = 5;
|
||||
}
|
||||
|
||||
message SyncNotebookResponse {
|
||||
int64 server_notebook_id = 1;
|
||||
google.protobuf.Timestamp synced_at = 2;
|
||||
}
|
||||
|
||||
message DeleteNotebookRequest {
|
||||
int64 notebook_id = 1; // App-side notebook ID
|
||||
}
|
||||
|
||||
message DeleteNotebookResponse {}
|
||||
|
||||
message ListNotebooksRequest {}
|
||||
|
||||
message ListNotebooksResponse {
|
||||
repeated NotebookSummary notebooks = 1;
|
||||
}
|
||||
|
||||
message NotebookSummary {
|
||||
int64 server_id = 1;
|
||||
int64 remote_id = 2;
|
||||
string title = 3;
|
||||
string page_size = 4;
|
||||
int32 page_count = 5;
|
||||
google.protobuf.Timestamp synced_at = 6;
|
||||
}
|
||||
|
||||
message CreateShareLinkRequest {
|
||||
int64 notebook_id = 1; // App-side notebook ID
|
||||
int64 expires_in_seconds = 2; // 0 = never expires
|
||||
}
|
||||
|
||||
message CreateShareLinkResponse {
|
||||
string token = 1;
|
||||
string url = 2; // Full shareable URL
|
||||
google.protobuf.Timestamp expires_at = 3; // Not set if never expires
|
||||
}
|
||||
|
||||
message RevokeShareLinkRequest {
|
||||
string token = 1;
|
||||
}
|
||||
|
||||
message RevokeShareLinkResponse {}
|
||||
|
||||
message ListShareLinksRequest {
|
||||
int64 notebook_id = 1;
|
||||
}
|
||||
|
||||
message ListShareLinksResponse {
|
||||
repeated ShareLinkInfo links = 1;
|
||||
}
|
||||
|
||||
message ShareLinkInfo {
|
||||
string token = 1;
|
||||
string url = 2;
|
||||
google.protobuf.Timestamp created_at = 3;
|
||||
google.protobuf.Timestamp expires_at = 4;
|
||||
}
|
||||
```
|
||||
|
||||
### Sync Semantics
|
||||
|
||||
`SyncNotebook` is an upsert: the server replaces all data for the
|
||||
given `notebook_id` (keyed by user + app-side ID). The entire notebook
|
||||
is sent each time — no incremental sync. This is simple and correct
|
||||
since the app is the sole writer.
|
||||
|
||||
For a notebook with ~100 pages and ~500 strokes per page, the payload
|
||||
is roughly 5-10 MB — manageable for manual sync over WiFi.
|
||||
|
||||
## REST API (Web Viewing)
|
||||
|
||||
| Method | Path | Auth | Description |
|
||||
|--------|------|------|-------------|
|
||||
| `POST` | `/v1/auth/login` | None | Password login, returns bearer token |
|
||||
| `POST` | `/v1/auth/webauthn/register/begin` | Bearer | Start FIDO2 registration |
|
||||
| `POST` | `/v1/auth/webauthn/register/finish` | Bearer | Complete FIDO2 registration |
|
||||
| `POST` | `/v1/auth/webauthn/login/begin` | None | Start FIDO2 login |
|
||||
| `POST` | `/v1/auth/webauthn/login/finish` | None | Complete FIDO2 login |
|
||||
| `GET` | `/v1/notebooks` | Bearer | List user's synced notebooks |
|
||||
| `GET` | `/v1/notebooks/:id` | Bearer | Notebook metadata + page list |
|
||||
| `GET` | `/v1/notebooks/:id/pages/:num` | Bearer | Page metadata + stroke count |
|
||||
| `GET` | `/v1/notebooks/:id/pages/:num/svg` | Bearer | Page rendered as SVG |
|
||||
| `GET` | `/v1/notebooks/:id/pages/:num/jpg` | Bearer | Page rendered as JPG (300 DPI) |
|
||||
| `GET` | `/v1/notebooks/:id/pdf` | Bearer | Full notebook as PDF |
|
||||
| `GET` | `/s/:token` | None | Shareable link → notebook view |
|
||||
| `GET` | `/s/:token/pages/:num/svg` | None | Shareable link → page SVG |
|
||||
| `GET` | `/s/:token/pages/:num/jpg` | None | Shareable link → page JPG |
|
||||
| `GET` | `/s/:token/pdf` | None | Shareable link → notebook PDF |
|
||||
|
||||
### SVG Rendering
|
||||
|
||||
Each page is rendered as an SVG document:
|
||||
|
||||
```xml
|
||||
<svg xmlns="http://www.w3.org/2000/svg"
|
||||
viewBox="0 0 612 792"
|
||||
width="612" height="792">
|
||||
<!-- Coordinates scaled from 300 DPI to 72 DPI (×0.24) for SVG -->
|
||||
<path d="M x0 y0 L x1 y1 L x2 y2 ..."
|
||||
stroke="black" stroke-width="1.08"
|
||||
stroke-linecap="round" stroke-linejoin="round"
|
||||
fill="none" />
|
||||
<!-- Dashed strokes -->
|
||||
<path d="M x0 y0 L x1 y1"
|
||||
stroke="black" stroke-width="1.08"
|
||||
stroke-dasharray="7.2 4.8"
|
||||
fill="none" />
|
||||
<!-- Arrow heads as separate paths -->
|
||||
</svg>
|
||||
```
|
||||
|
||||
SVG coordinates use 72 DPI (PDF points) for consistency with PDF export.
|
||||
The `viewBox` maps to physical page dimensions. Browsers handle
|
||||
zoom/pan natively via scroll and pinch.
|
||||
|
||||
### JPG and PDF Rendering
|
||||
|
||||
Server-side rendering using Go's `image` package (JPG) and a PDF
|
||||
library like `go-pdf` or `jung-kurt/gofpdf` (PDF). Same rendering
|
||||
logic as the Android app — scale coordinates by 72/300 for PDF,
|
||||
render at 300 DPI for JPG.
|
||||
|
||||
## Authentication
|
||||
|
||||
### Password Auth
|
||||
|
||||
- Argon2id hashing (memory-hard, tuned to hardware)
|
||||
- Bearer tokens returned on login, stored as session cookies for web UI
|
||||
- Token validation via SHA-256 keyed lookup with short TTL cache
|
||||
|
||||
### FIDO2/U2F (WebAuthn)
|
||||
|
||||
Users can register multiple security keys after initial password login.
|
||||
Once registered, keys can be used as an alternative login method.
|
||||
|
||||
Implementation via the `go-webauthn/webauthn` library:
|
||||
|
||||
1. **Registration flow**:
|
||||
- User logs in with password
|
||||
- Calls `POST /v1/auth/webauthn/register/begin` → gets challenge
|
||||
- Browser/client signs challenge with security key
|
||||
- Calls `POST /v1/auth/webauthn/register/finish` with attestation
|
||||
- Server stores credential in `webauthn_credentials` table
|
||||
|
||||
2. **Login flow**:
|
||||
- User calls `POST /v1/auth/webauthn/login/begin` with username → gets challenge
|
||||
- Browser signs challenge with registered key
|
||||
- Calls `POST /v1/auth/webauthn/login/finish` with assertion
|
||||
- Server validates, returns bearer token
|
||||
|
||||
A user can have any number of registered keys. Each key has a
|
||||
user-assigned name for identification.
|
||||
|
||||
## Shareable Links
|
||||
|
||||
Tokens are 32-byte random values encoded as URL-safe base64 (43 chars).
|
||||
Generated via `crypto/rand`.
|
||||
|
||||
- Default expiry: never (0 = no expiry)
|
||||
- Optional expiry set at creation time
|
||||
- Expired links return 410 Gone
|
||||
- Links can be revoked via the gRPC API or web UI
|
||||
- Each link is scoped to a single notebook
|
||||
|
||||
The shareable URL format: `https://pad.metacircular.net/s/{token}`
|
||||
|
||||
The web view for shared links shows the notebook title, page navigation,
|
||||
and export options (SVG view, JPG download, PDF download) — identical
|
||||
to the authenticated view but without the notebook list or share
|
||||
management.
|
||||
|
||||
## Android App Changes
|
||||
|
||||
### Sync Integration
|
||||
|
||||
New files:
|
||||
- `internal/sync/SyncClient.kt` — gRPC client for `EngPadSync`
|
||||
- `internal/sync/SyncManager.kt` — orchestrates notebook serialization
|
||||
and sync
|
||||
|
||||
### UI Changes
|
||||
|
||||
- Notebook overflow menu: add "Sync to server" option
|
||||
- Library toolbar: add "Sync all" button
|
||||
- Settings screen: server URL, credentials
|
||||
- Sync status indicator on notebook cards (synced/unsynced/error)
|
||||
|
||||
### Configuration
|
||||
|
||||
Server URL and auth token stored in `EngPadApp` SharedPreferences.
|
||||
First sync prompts for server URL and login credentials.
|
||||
|
||||
## Server Repository Layout
|
||||
|
||||
```
|
||||
eng-pad-server/
|
||||
├── cmd/
|
||||
│ └── eng-pad-server/ CLI entry point
|
||||
├── internal/
|
||||
│ ├── auth/ Password + WebAuthn auth
|
||||
│ ├── config/ TOML config
|
||||
│ ├── db/ SQLite setup, migrations
|
||||
│ ├── grpcserver/ gRPC sync service
|
||||
│ ├── server/ REST API routes
|
||||
│ ├── render/ SVG, JPG, PDF rendering
|
||||
│ ├── share/ Share link management
|
||||
│ └── webserver/ htmx web UI
|
||||
├── proto/engpad/v1/ Protobuf definitions
|
||||
├── gen/engpad/v1/ Generated Go code
|
||||
├── web/
|
||||
│ ├── templates/ HTML templates
|
||||
│ └── static/ CSS, htmx
|
||||
├── deploy/
|
||||
│ ├── docker/
|
||||
│ ├── systemd/
|
||||
│ └── scripts/
|
||||
├── Makefile
|
||||
├── buf.yaml
|
||||
├── CLAUDE.md
|
||||
├── ARCHITECTURE.md
|
||||
└── eng-pad-server.toml.example
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
```toml
|
||||
[server]
|
||||
listen_addr = ":8443"
|
||||
grpc_addr = ":9443"
|
||||
tls_cert = "/srv/eng-pad-server/certs/cert.pem"
|
||||
tls_key = "/srv/eng-pad-server/certs/key.pem"
|
||||
|
||||
[database]
|
||||
path = "/srv/eng-pad-server/eng-pad-server.db"
|
||||
|
||||
[web]
|
||||
listen_addr = ":8080"
|
||||
base_url = "https://pad.metacircular.net"
|
||||
|
||||
[auth]
|
||||
token_ttl = "24h"
|
||||
argon2_memory = 65536
|
||||
argon2_time = 3
|
||||
argon2_threads = 4
|
||||
|
||||
[webauthn]
|
||||
rp_display_name = "Engineering Pad"
|
||||
rp_id = "pad.metacircular.net"
|
||||
rp_origins = ["https://pad.metacircular.net"]
|
||||
|
||||
[log]
|
||||
level = "info"
|
||||
```
|
||||
|
||||
## Open Questions
|
||||
|
||||
1. Should the web UI show the grid? It's excluded from PDF/JPG export
|
||||
but might be useful for on-screen viewing. Could be a toggle.
|
||||
|
||||
2. Should share links support per-page scope in addition to whole
|
||||
notebook? The user said no for now, but the URL structure supports
|
||||
it (`/s/:token/pages/:num`).
|
||||
|
||||
3. Should the server support deleting individual pages, or only
|
||||
full notebook replacement via sync? Full replacement is simpler
|
||||
and matches the "server receives whatever the app pushes" model.
|
||||
Reference in New Issue
Block a user