Delta Synchronization

// Naive approach: Re-fetch everything
const sync = async () => {
  const allPosts = await fetch('/api/posts').then(r => r.json())
  await db.replaceAll('posts', allPosts)  // Slow! 😔
}
 
// Problems:
// - Transfers entire dataset every time
// - Wastes bandwidth (mobile data cost)
// - Slow on large datasets
// - Unnecessary database writes

// Gluonic delta sync
const sync = async () => {
  const since = await storage.getLastSyncId()  // e.g., 14
  
  const delta = await fetch(`/sync/v1/delta?since=${since}`)
  // Server returns only changes since syncId 14
  
  // Apply incremental updates (fast!)
  await store.applyFrames(delta.frames)
}

// Server database
SyncAction table:
┌────┬───────┬─────────┬────────┬──────────────────┐
│ id │ orgId │ model   │ modelId│ op   │ patch    │
├────┼───────┼─────────┼────────┼──────┼──────────┤
│ 12 │ user1 │ post    │ abc    │ i    │ {...}    │
│ 13 │ user1 │ post    │ abc    │ u    │ {...}    │
│ 14 │ user1 │ post    │ def    │ i    │ {...}    │
│ 15 │ user1 │ comment │ xyz    │ i    │ {...}    │ ← Latest
└────┴───────┴─────────┴────────┴──────┴──────────┘

// Client stores last sync ID
await storage.setLastSyncId(15)
 
// Meaning: "I've seen all changes up to #15"

// Client requests changes since last sync
GET /sync/v1/delta?since=15
 
// Server responds with changes after 15:
[
  { sid: 16, t: 'post', id: 'abc', op: 'u', p: { title: 'Updated' } },
  { sid: 17, t: 'user', id: 'u1', op: 'u', p: { name: 'John' } }
]
 
// Client applies and updates lastSyncId to 17

// First time: Get everything
const { rows, lastSid } = await fetch('/sync/v1/bootstrap')
 
// rows: All data for this organization
// lastSid: Latest sync ID (e.g., 1000)
 
await storage.putRows(rows)
await storage.setLastSyncId(lastSid)
 
// Client is now at syncId 1000

// Later: Get only changes
const since = await storage.getLastSyncId()  // 1000
 
const frames = await fetch(`/sync/v1/delta?since=${since}`)
// Returns changes 1001, 1002, 1003, ...
 
await store.applyFrames(frames)
// Applies each change incrementally
 
const maxSid = Math.max(...frames.map(f => f.sid))
await storage.setLastSyncId(maxSid)
 
// Client is now at syncId 1003 (or latest)

// App was offline for a while
// User reconnects
 
// 1. Check current position
const since = await storage.getLastSyncId()  // 1000
 
// 2. Request delta
const frames = await fetch(`/sync/v1/delta?since=${since}`)
// Might return 1001-2500 (1500 changes!)
 
// 3. Apply all changes
await store.applyFrames(frames)
 
// 4. Now caught up!
await storage.setLastSyncId(2500)

// Traditional: Transfer entire dataset
Bootstrap: 10,000 posts × 2KB = 20MB
Every sync: 20MB 😔
 
// Gluonic: Transfer only changes
Bootstrap: 10,000 posts × 2KB = 20MB (one time)
Delta sync: 5 changes × 2KB = 10KB ✓
Savings: 99.95% 🎉

// After initial bootstrap
// User makes 1 edit → 2KB sent
// 10 other users make edits → 20KB received
// Total: 22KB (vs 20MB full sync)

// Delta sync every 5 seconds
setInterval(() => store.catchUp(), 5000)
 
// Each delta only transfers changes
// No performance impact
// Feels real-time ✓

// 10 posts
Delta: 5ms, 1KB
 
// 10,000 posts  
Delta: 5ms, 1KB  // Same! Only changed data
 
// 1,000,000 posts
Delta: 5ms, 1KB  // Still same! 🎉

interface SyncFrame {
  sid: number        // Sync ID (14, 15, 16, ...)
  t: string          // Model type ('post', 'user')
  id: string         // Record ID
  op: 'i'|'u'|'d'    // Operation (insert, update, delete)
  p?: any            // Patch (for insert/update)
  who?: string       // Actor ID (who made the change)
  at: number         // Timestamp
  ctx?: {            // Context
    client_tx_id?: string
  }
}

[
  {
    "sid": 16,
    "t": "post",
    "id": "post-123",
    "op": "u",
    "p": { "title": "Updated Title" },
    "who": "user-1",
    "at": 1698765432000,
    "ctx": { "client_tx_id": "tx-789" }
  },
  {
    "sid": 17,
    "t": "comment",
    "id": "comment-456",
    "op": "i",
    "p": { "text": "Great post!", "postId": "post-123" },
    "who": "user-2",
    "at": 1698765433000
  }
]

// WebSocket connected → instant updates
// WebSocket disconnects → delta sync catches up
// No data loss ✓

// Catch up to latest
await store.catchUp()
 
// Behind the scenes:
async catchUp() {
  const since = await this.storage.getLastSyncId()
  const { frames, sid } = await this.downsync.fetchDelta(since)
  
  await this.applyFrames(frames)
  await this.storage.setLastSyncId(sid)
}

// Delta endpoint
app.get('/sync/v1/delta', async (req, reply) => {
  const orgId = getOrgId(req)
  const since = Number(req.query.since || 0)
  
  // Load changes after 'since'
  const actions = await adapter.loadSyncActions(orgId, since, 5000)
  
  // Convert to frames
  const frames = actions.map(a => ({
    sid: a.id,
    t: a.model,
    id: a.modelId,
    op: a.op,
    p: a.patch,
    who: a.actorId,
    at: a.createdAt.getTime()
  }))
  
  reply.send(frames)
})

import { SyncServer } from '@gluonic/server'
 
// Server configuration
const server = SyncServer({
  database,
  auth,
  maxDelta: 5000  // Max frames per request
  // If more changes, client pages through them
})

const server = SyncServer({
  database,
  auth,
  deltaBackpressure: {
    minIntervalMs: 100,       // Min 100ms between requests
    maxConcurrencyPerOrg: 5   // Max 5 concurrent per org
  }
})

// Instead of:
// fetchDelta(since=100) → 1 frame
// fetchDelta(since=101) → 1 frame
// fetchDelta(since=102) → 1 frame
 
// Gluonic waits 100ms and batches:
// fetchDelta(since=100) → 3 frames ✓