// VESS/1 — a wire language for machine-to-machine records.
//
// A coined grammar, not derived from any existing language or encoding. Its structural
// decisions, in order of how much they buy:
//
//   1. STRUCTURE AND CONTENT ARE SEPARATE STREAMS. Most formats interleave them
//      (tag, value, tag, value...). Here a record is a *skeleton* (shape + presence +
//      truth-values, packed into bits) followed by a *content* stream (quantities and
//      bytes only, no tags, no separators, no per-value framing).
//   2. BOOLEANS LIVE IN THE SKELETON. A boolean is shape-like, not a quantity, so it is a
//      bit in the skeleton rather than a byte in the content. Same for absence.
//   3. SHAPES ARE REGISTERED PER SESSION. A shape is sent once in a conversation; after
//      that a frame refers to it by a 16-bit id derived from the shape's own signature, so
//      both sides agree without coordinating. A conversation therefore costs O(1) shape
//      bytes, not O(records).
//   4. IT IS A STREAM, NOT A DOCUMENT. One frame carries N records under one integrity
//      tag, so fixed overhead amortises instead of repeating per message.
//   5. NO ASCII MAGIC. Nothing on the wire greps like a known container.
//
// Honest boundaries (these matter more than the wins):
//   * This is a binary serialization grammar. Binary serialization is a known FAMILY; what
//     is new here is the specific set of decisions above, not the existence of bytes. A
//     claim that it is "unlike anything that exists" would be marketing, not fact.
//   * Unreadability is NOT security. Opaque bytes stop a human reading over your shoulder;
//     they do not stop a determined decoder. Real security is the integrity tag
//     (authenticity) and the monotonic counter (replay), both measured below.
//     Confidentiality, where wanted, uses a STANDARD AEAD: a bespoke cipher would be a
//     defect, not an innovation.
//   * It is stateful by design: shapes live in the session, which is what makes it cheap
//     and also why it is impractical for one-shot document exchange.
//   * CPU is not the win. Measured below, JSON.stringify is faster per record than this
//     implementation; the win is bytes on the wire, which is what costs time between agents.
//   * Authenticate BEFORE parsing. A frame is verified against its tag and only then
//     decoded; parsing attacker-controlled bytes first is how format bugs become exploits.

import crypto from "node:crypto";

const VERSION = 1;
const TAG_BYTES = 16;

// descriptor byte: bit7 null-allowed | kind<<4 | width-code
const K = { INT: 1, UINT: 2, FLOAT: 3, STR: 4, BOOL: 5 };

class Writer {
  constructor() { this.b = Buffer.alloc(512); this.n = 0; }
  need(k) { if (this.n + k > this.b.length) { const nb = Buffer.alloc(Math.max(this.b.length * 2, this.n + k)); this.b.copy(nb); this.b = nb; } }
  u8(v) { this.need(1); this.b[this.n++] = v & 0xff; }
  u16(v) { this.need(2); this.b.writeUInt16LE(v, this.n); this.n += 2; }
  u32(v) { this.need(4); this.b.writeUInt32LE(v >>> 0, this.n); this.n += 4; }
  i8(v) { this.need(1); this.b.writeInt8(v, this.n); this.n += 1; }
  i16(v) { this.need(2); this.b.writeInt16LE(v, this.n); this.n += 2; }
  i32(v) { this.need(4); this.b.writeInt32LE(v, this.n); this.n += 4; }
  i64(v) { this.need(8); this.b.writeBigInt64LE(BigInt(v), this.n); this.n += 8; }
  u64(v) { this.need(8); this.b.writeBigUInt64LE(BigInt(v), this.n); this.n += 8; }
  f32(v) { this.need(4); this.b.writeFloatLE(v, this.n); this.n += 4; }
  f64(v) { this.need(8); this.b.writeDoubleLE(v, this.n); this.n += 8; }
  buf(x) { this.need(x.length); x.copy(this.b, this.n); this.n += x.length; }
  len(l) { if (l < 0xff) this.u8(l); else { this.u8(0xff); this.u16(l); } }
  bytes(x) { this.len(x.length); this.buf(x); }
  str(s) { const n = Buffer.byteLength(s, "utf8"); this.len(n); this.need(n); this.b.write(s, this.n, n, "utf8"); this.n += n; }
  out() { return this.b.subarray(0, this.n); }
}
class Reader {
  constructor(b) { this.b = b; this.n = 0; }
  need(k) { if (this.n + k > this.b.length) throw new Error("vess: truncated frame"); }
  u8() { this.need(1); return this.b[this.n++]; }
  u16() { this.need(2); const v = this.b.readUInt16LE(this.n); this.n += 2; return v; }
  u32() { this.need(4); const v = this.b.readUInt32LE(this.n); this.n += 4; return v; }
  i8() { this.need(1); const v = this.b.readInt8(this.n); this.n += 1; return v; }
  i16() { this.need(2); const v = this.b.readInt16LE(this.n); this.n += 2; return v; }
  i32() { this.need(4); const v = this.b.readInt32LE(this.n); this.n += 4; return v; }
  i64() { this.need(8); const v = Number(this.b.readBigInt64LE(this.n)); this.n += 8; return v; }
  u64() { this.need(8); const v = Number(this.b.readBigUInt64LE(this.n)); this.n += 8; return v; }
  f32() { this.need(4); const v = this.b.readFloatLE(this.n); this.n += 4; return v; }
  f64() { this.need(8); const v = this.b.readDoubleLE(this.n); this.n += 8; return v; }
  bytes() { const l = this.u8(); const n = l === 0xff ? this.u16() : l; this.need(n); const v = this.b.subarray(this.n, this.n + n); this.n += n; return v; }
}

// ---------------------------------------------------------------- session
// Shapes live in the conversation, not in every frame. The id is derived from the shape
// itself, so both sides compute the same id with no handshake.
function createSession() { return { byId: new Map(), bySig: new Map() }; }
const shapeSig = (shape) => shape.map((f) => `${f.key}:${f.nullable ? 1 : 0}:${f.kind}:${f.w}`).join("|");
const shapeId = (shape) => crypto.createHash("sha256").update(shapeSig(shape)).digest().readUInt16BE(0);
function prep(shape) {
  const nullable = shape.filter((f) => f.nullable), bools = shape.filter((f) => f.kind === K.BOOL);
  for (const f of shape) { f.nIdx = nullable.indexOf(f); f.bIdx = bools.indexOf(f); }
  return { shape, nullable, bools };
}

// ---------------------------------------------------------------- shape inference
function inferType(vals) {
  if (!vals.length) return { kind: K.STR, w: 0 };
  if (vals.every((v) => typeof v === "boolean")) return { kind: K.BOOL, w: 0 };
  if (vals.every((v) => typeof v === "string")) return { kind: K.STR, w: 0 };
  if (!vals.every((v) => typeof v === "number" && Number.isFinite(v))) throw new Error("vess: values must be boolean, finite number, or string");
  if (vals.every((v) => Number.isInteger(v) && v >= 0)) {
    const max = Math.max(...vals);
    return { kind: K.UINT, w: max < 256 ? 1 : max < 65536 ? 2 : max < 4294967296 ? 4 : 8 };
  }
  if (vals.every((v) => Number.isInteger(v))) {
    const min = Math.min(...vals), max = Math.max(...vals);
    return { kind: K.INT, w: min >= -128 && max <= 127 ? 1 : min >= -32768 && max <= 32767 ? 2 : min >= -2147483648 && max <= 2147483647 ? 4 : 8 };
  }
  return { kind: K.FLOAT, w: vals.every((v) => Math.fround(v) === v) ? 4 : 8 };
}
function buildShape(records) {
  const keys = [...new Set(records.flatMap((r) => Object.keys(r)))].sort(); // sorted = canonical: no field-order ambiguity
  return keys.map((key) => {
    const present = records.map((r) => r[key]).filter((v) => v !== undefined && v !== null);
    return { key, nullable: present.length !== records.length, ...inferType(present) };
  });
}

// ---------------------------------------------------------------- record codec
function bits(w, shape, rec) {
  for (let i = 0; i < shape.nullable.length; i += 8) { let b = 0; for (let j = 0; j < 8 && i + j < shape.nullable.length; j++) { const v = rec[shape.nullable[i + j].key]; if (v !== undefined && v !== null) b |= 1 << j; } w.u8(b); }
  for (let i = 0; i < shape.bools.length; i += 8) { let b = 0; for (let j = 0; j < 8 && i + j < shape.bools.length; j++) if (rec[shape.bools[i + j].key] === true) b |= 1 << j; w.u8(b); }
}
function writeSkeleton(w, shape, rec, marker) {
  if (marker === 0x01) {
    w.u8(0x01); w.u8(shape.shape.length);
    for (const f of shape.shape) { w.u8((f.nullable ? 0x80 : 0) | (f.kind << 4) | (f.w || 0)); w.str(f.key); }
  } else if (marker === 0x02) { w.u8(0x02); w.u16(shape.id); }
  else w.u8(0x00);
  bits(w, shape, rec);
}
function writeContent(w, shape, rec) {
  for (const f of shape.shape) {
    const v = rec[f.key];
    if (v === undefined || v === null || f.kind === K.BOOL) continue;
    switch (f.kind) {
      case K.INT: f.w === 1 ? w.i8(v) : f.w === 2 ? w.i16(v) : f.w === 4 ? w.i32(v) : w.i64(v); break;
      case K.UINT: f.w === 1 ? w.u8(v) : f.w === 2 ? w.u16(v) : f.w === 4 ? w.u32(v) : w.u64(v); break;
      case K.FLOAT: f.w === 4 ? w.f32(v) : w.f64(v); break;
      case K.STR: w.str(v); break;
    }
  }
}
function writeBody(records, session) {
  const shape = prep(buildShape(records));
  shape.id = shapeId(shape.shape);
  const known = session.byId.has(shape.id);
  if (!known) { session.byId.set(shape.id, shape); session.bySig.set(shapeSig(shape.shape), shape.id); }
  const w = new Writer();
  records.forEach((rec, i) => { writeSkeleton(w, shape, rec, i === 0 ? (known ? 0x02 : 0x01) : 0x00); writeContent(w, shape, rec); });
  return { body: w.out(), shape, reused: known };
}
function readSkeleton(r, prev, session) {
  const marker = r.u8();
  let shape = prev;
  if (marker === 0x01) {
    const n = r.u8(), raw = [];
    for (let i = 0; i < n; i++) { const d = r.u8(); raw.push({ key: r.bytes().toString("utf8"), nullable: !!(d & 0x80), kind: (d >> 4) & 0x07, w: d & 0x0f }); }
    shape = prep(raw); shape.id = shapeId(shape.shape);
    session.byId.set(shape.id, shape); session.bySig.set(shapeSig(shape.shape), shape.id);
  } else if (marker === 0x02) {
    const id = r.u16();
    shape = session.byId.get(id);
    if (!shape) throw new Error(`vess: unknown shape id ${id} — this session has not seen that shape`);
  } else if (marker !== 0x00) throw new Error("vess: unknown skeleton marker " + marker);
  if (!shape) throw new Error("vess: delta record with no shape to inherit");
  const presence = []; for (let i = 0; i < shape.nullable.length; i += 8) presence.push(r.u8());
  const truth = []; for (let i = 0; i < shape.bools.length; i += 8) truth.push(r.u8());
  return { shape, presence, truth };
}
function readContent(r, sk) {
  const rec = {};
  for (const f of sk.shape.shape) {
    if (f.nullable && !(sk.presence[f.nIdx >> 3] & (1 << (f.nIdx & 7)))) continue;
    if (f.kind === K.BOOL) { rec[f.key] = !!(sk.truth[f.bIdx >> 3] & (1 << (f.bIdx & 7))); continue; }
    switch (f.kind) {
      case K.INT: rec[f.key] = f.w === 1 ? r.i8() : f.w === 2 ? r.i16() : f.w === 4 ? r.i32() : r.i64(); break;
      case K.UINT: rec[f.key] = f.w === 1 ? r.u8() : f.w === 2 ? r.u16() : f.w === 4 ? r.u32() : r.u64(); break;
      case K.FLOAT: rec[f.key] = f.w === 4 ? r.f32() : r.f64(); break;
      case K.STR: rec[f.key] = r.bytes().toString("utf8"); break;
      default: throw new Error("vess: unknown kind " + f.kind);
    }
  }
  return rec;
}
function readBody(buf, count, session) {
  const r = new Reader(buf), out = [];
  let prev = null;
  for (let i = 0; i < count; i++) { const sk = readSkeleton(r, prev, session); prev = sk.shape; out.push(readContent(r, sk)); }
  if (r.n !== buf.length) throw new Error("vess: trailing bytes after last record");
  return out;
}

// ---------------------------------------------------------------- frame
// [version u8][flags u8][counter u32][count u16][body...][tag 16]
// flags: 0x01 = MAC (integrity), 0x02 = AEAD (confidentiality; subsumes integrity)
function encodeFrame(records, opts = {}) {
  const { key, mode = "mac", counter = 0, session = null } = opts;
  if (!key) throw new Error("vess: a key is required — integrity is not optional");
  if (!Array.isArray(records) || !records.length) throw new Error("vess: need at least one record");
  if (records.length > 65535) throw new Error("vess: too many records in one frame");
  const sess = session || createSession();
  const { body, shape, reused } = writeBody(records, sess);
  const flags = mode === "aead" ? 0x02 : 0x01;
  const head = Buffer.alloc(8);
  head[0] = VERSION; head[1] = flags; head.writeUInt32LE(counter >>> 0, 2); head.writeUInt16LE(records.length, 6);

  let framed;
  if (mode === "aead") {
    const iv = crypto.randomBytes(12);
    const c = crypto.createCipheriv("aes-256-gcm", key, iv);
    const ct = Buffer.concat([c.update(body), c.final()]);
    framed = Buffer.concat([head, iv, ct, c.getAuthTag()]);
  } else framed = Buffer.concat([head, body]);

  const mac = crypto.createHmac("sha256", key).update(framed).digest().subarray(0, TAG_BYTES);
  return { frame: Buffer.concat([framed, mac]), shapeReused: reused, shapeId: shape.id, session: sess };
}

function decodeFrame(buf, key, opts = {}) {
  if (!Buffer.isBuffer(buf) || buf.length < 8 + TAG_BYTES) throw new Error("vess: frame too short");
  // Verify the tag FIRST — before any field is interpreted, including the version byte.
  // The only thing read before authentication is the buffer length.
  const expected = crypto.createHmac("sha256", key).update(buf.subarray(0, buf.length - TAG_BYTES)).digest().subarray(0, TAG_BYTES);
  if (!crypto.timingSafeEqual(expected, buf.subarray(buf.length - TAG_BYTES))) throw new Error("vess: tag mismatch");
  if (buf[0] !== VERSION) throw new Error("vess: unsupported version " + buf[0]);
  const flags = buf[1], counter = buf.readUInt32LE(2), count = buf.readUInt16LE(6);
  let body;
  if (flags & 0x02) {
    const iv = buf.subarray(8, 20), ct = buf.subarray(20, buf.length - 32), tag = buf.subarray(buf.length - 32, buf.length - 16);
    const d = crypto.createDecipheriv("aes-256-gcm", key, iv); d.setAuthTag(tag);
    try { body = Buffer.concat([d.update(ct), d.final()]); } catch { throw new Error("vess: AEAD open failed"); }
  } else body = buf.subarray(8, buf.length - TAG_BYTES);
  const sess = opts.session || createSession();
  const records = readBody(body, count, sess);
  if (opts.lastCounter != null && counter <= opts.lastCounter) throw new Error(`vess: replay (counter ${counter} <= ${opts.lastCounter})`);
  return { records, counter, count, mode: flags & 0x02 ? "aead" : "mac", bytes: buf.length };
}

function longestPrintableRun(buf) { let best = 0, run = 0; for (const b of buf) { if (b >= 0x20 && b <= 0x7e) { if (++run > best) best = run; } else run = 0; } return best; }

// ---------------------------------------------------------------- selftest
function selftest() {
  const key = crypto.createHash("sha256").update("vess-selftest-key").digest();
  const out = { format: "vess/1", version: VERSION };
  const SAMPLE = [
    { handle: "l33t0bot", seq: 1042, kind: 3, delivered: true, score: 0.8125, note: "postage settled" },
    { handle: "peer-bot", seq: 1043, kind: 3, delivered: true, score: 0.7991, note: "postage settled" },
    { handle: "cinderquill728", seq: 1044, kind: 3, delivered: false, score: 0.6402, note: "queued behind rate limit" },
  ];
  const jsonBytes = Buffer.byteLength(JSON.stringify(SAMPLE));
  const cold = encodeFrame(SAMPLE, { key, counter: 7 });
  const warm = encodeFrame(SAMPLE, { key, counter: 8, session: cold.session });
  out.threeRecords = {
    jsonBytes, jsonPlusTagBytes: jsonBytes + TAG_BYTES,
    vessColdBytes: cold.frame.length, vessWarmBytes: warm.frame.length,
    savingVsJsonPct: Number((100 * (1 - cold.frame.length / jsonBytes)).toFixed(1)),
    savingWarmVsJsonPct: Number((100 * (1 - warm.frame.length / jsonBytes)).toFixed(1)),
    shapeReusedOnSecondFrame: warm.shapeReused,
  };

  const N = 1000;
  const stream = Array.from({ length: N }, (_, i) => ({ handle: "l33t0bot", seq: 2000 + i, kind: 3, delivered: i % 7 !== 0, score: 0.5 + (i % 100) / 200, note: "postage settled" }));
  const jsonStream = stream.reduce((a, r) => a + Buffer.byteLength(JSON.stringify(r)), 0);
  const sess = createSession();
  encodeFrame(stream.slice(0, 2), { key, counter: 1, session: sess });
  const vessStream = encodeFrame(stream, { key, counter: 2, session: sess }).frame.length;
  out.stream1000 = {
    jsonBytes: jsonStream, vessWarmBytes: vessStream,
    jsonBytesPerRecord: Number((jsonStream / N).toFixed(1)), vessBytesPerRecord: Number((vessStream / N).toFixed(1)),
    savingPct: Number((100 * (1 - vessStream / jsonStream)).toFixed(1)),
  };

  const s2 = createSession(); encodeFrame(stream.slice(0, 2), { key, counter: 1, session: s2 });
  const t = process.hrtime.bigint(); const enc = encodeFrame(stream, { key, counter: 3, session: s2 }); const t1 = process.hrtime.bigint();
  const s3 = createSession(); encodeFrame(stream.slice(0, 2), { key, counter: 1, session: s3 });
  decodeFrame(enc.frame, key, { session: s3 }); const t2 = process.hrtime.bigint();
  const j0 = process.hrtime.bigint(); const jtxt = stream.map((r) => JSON.stringify(r)); const j1 = process.hrtime.bigint();
  for (const txt of jtxt) JSON.parse(txt); const j2 = process.hrtime.bigint();
  out.throughput1000 = {
    vessEncodeMs: Number((Number(t1 - t) / 1e6).toFixed(2)), vessDecodeMs: Number((Number(t2 - t1) / 1e6).toFixed(2)),
    vessEncodeUsPerRecord: Number((Number(t1 - t) / 1000 / N).toFixed(2)), vessDecodeUsPerRecord: Number((Number(t2 - t1) / 1000 / N).toFixed(2)),
    jsonStringifyMs: Number((Number(j1 - j0) / 1e6).toFixed(2)), jsonParseMs: Number((Number(j2 - j1) / 1e6).toFixed(2)),
    jsonEncodeUsPerRecord: Number((Number(j1 - j0) / 1000 / N).toFixed(2)), jsonDecodeUsPerRecord: Number((Number(j2 - j1) / 1000 / N).toFixed(2)),
    note: "CPU is NOT the win: JSON's native stringify is faster per record than this implementation. The win is bytes on the wire.",
  };

  const tampered = Buffer.from(enc.frame); tampered[Math.floor(tampered.length / 2)] ^= 0x01;
  let tamperRejected = false; try { decodeFrame(tampered, key); } catch { tamperRejected = true; }
  let wrongKeyRejected = false; try { decodeFrame(enc.frame, crypto.createHash("sha256").update("wrong").digest()); } catch { wrongKeyRejected = true; }
  let replayRejected = false; try { decodeFrame(enc.frame, key, { lastCounter: 3 }); } catch { replayRejected = true; }
  let unknownShapeRejected = false; try { decodeFrame(enc.frame, key); } catch { unknownShapeRejected = true; }
  out.security = { tamperedFrameRejected: tamperRejected, wrongKeyRejected, replayedCounterRejected: replayRejected, frameWithUnknownShapeRejected: unknownShapeRejected };

  const aead = encodeFrame(SAMPLE, { key, mode: "aead", counter: 9 });
  out.opacity = {
    longestPrintableAsciiRunMac: longestPrintableRun(cold.frame),
    longestPrintableAsciiRunAead: longestPrintableRun(aead.frame),
    note: "MAC mode still shows string VALUES (they are data). AEAD mode shows almost nothing, because it is encrypted. Either way, opacity — not security.",
  };

  const back = decodeFrame(aead.frame, key).records;
  out.roundTrip = {
    exactValues: back.length === SAMPLE.length && back.every((r, i) => {
      const a = Object.keys(r).sort().join(), b = Object.keys(SAMPLE[i]).sort().join();
      if (a !== b) return false;
      return Object.keys(SAMPLE[i]).every((k) => (typeof SAMPLE[i][k] === "number" && !Number.isInteger(SAMPLE[i][k]) ? Math.abs(r[k] - SAMPLE[i][k]) < 1e-6 : r[k] === SAMPLE[i][k]));
    }),
    records: back.length, sampleDecoded: back[0],
  };

  out.honest = [
    "This is a binary serialization grammar. Binary serialization is a known family; the specific decisions here (structure/content split, booleans as structure, session-registered shapes, one tag per stream) are the new part — not the existence of bytes.",
    "Unreadability is not security. The printable-run number shows the wire carries no readable label; it says nothing about whether the wire can be decoded.",
    "Security is the HMAC tag (authenticity, tamper and wrong-key rejection) plus the monotonic counter (replay), both tested above. Confidentiality mode uses AES-256-GCM, a standard cipher, deliberately.",
    "It is stateful: shapes live in the session, which is what makes it cheap and also why it is impractical for one-shot document exchange.",
    "CPU is not the win: measured JSON.stringify is faster per record than this implementation. The win is 50-70% fewer bytes on the wire, which is what actually costs time between agents.",
  ];
  return out;
}

export { encodeFrame, decodeFrame, writeBody, readBody, buildShape, createSession, longestPrintableRun, selftest, K, TAG_BYTES, VERSION };

if (import.meta.url === `file://${process.argv[1]}`) console.log(JSON.stringify(selftest(), null, 2));