ndjson-source

#!/usr/bin/env node
/* ndjson-source.js — validate and normalize an NDJSON stream into canonical JSONL.
   Dependency-free, deterministic, pure. Runs in Node or a browser. MIT.

   WHAT IT IS. A SOURCE: it turns NDJSON — newline-delimited JSON, one JSON value per
   line, the shape a log, an export, or a `jq -c` stream so often arrives in — into the
   front of a pipe: one canonical compact JSON value per line (JSONL), ready to feed INTO
   the fold/filter/transform gifts. Give it a file (or pipe text on stdin) and it parses
   each line as its own JSON value and re-emits it canonically:

       {"id":1}                   ->  {"id":1}
       {"id":2}                       {"id":2}

       { "a": 1e3 }               ->  {"a":1000}     (whitespace + number tokens normalized)
       [1, 2, 3]                      [1,2,3]

   WHERE IT SITS NEXT TO json-source (the honest complement). json-source streams a
   TOP-LEVEL JSON ARRAY — one `[ ... ]` document — and REFUSES a non-array top-level,
   because a single value is not a stream. NDJSON is exactly the OTHER shape: a stream is
   already spelled out, one value per line, no enclosing array. ndjson-source is the honest
   reader for that shape. json-source unwraps an array into a stream; ndjson-source
   validates and canonicalizes a stream that is already line-delimited. It reads NDJSON,
   not a JSON array (use json-source) and not JSON5.

   PER-LINE HONESTY (the honesty axis). Each non-blank line MUST be exactly one valid JSON
   value. A line that is malformed JSON — or that carries a second value after the first —
   is a HARD ERROR (exit 2), naming the 1-based line number, never a skipped line and never
   a partial or repaired parse. ndjson-source does not quietly drop the bad record and keep
   going (that silently changes your data); it stops and tells you which line. Blank lines
   (empty or whitespace-only) carry no record and are skipped — that is not a guess, a
   blank line is unambiguously not a value. CRLF and LF line endings are both accepted (a
   trailing carriage return is stripped before parsing).

   CANONICAL RE-SERIALIZATION (declared, so it is pinnable). Each value is emitted via
   canonical compact JSON.stringify — object key order is preserved from the input, but
   input WHITESPACE and NUMBER TOKENS are normalized to canonical JSON form (`1e3` -> 1000,
   `1.0` -> 1, `{ "a" : 1 }` -> {"a":1}). This is the JSON *value*, losslessly; it is not
   the input line's exact bytes. So ndjson-source is a VALIDATOR and NORMALIZER: valid but
   sloppy NDJSON comes out as canonical JSONL, byte-identical every run and every machine.

   THE MODEL
     [FILE]   The NDJSON file to read. If omitted, read stdin. One JSON value per line;
              blank lines are skipped; each non-blank line becomes one output line.

   DETERMINISM. parse(text) is a pure function — no clock, no randomness, no network — so
   the same text yields byte-identical output every run.

   USAGE
     node ndjson-source.js data.ndjson
     cat data.ndjson | node ndjson-source.js
     node ndjson-source.js --help

   Exit codes: 0 success (including an empty stream from empty or all-blank input) · 2 input
   error (a malformed JSON line, an unknown option, a second positional file, or an
   unreadable file). Always a clean one-line message on stderr, never a stack trace.

   Released under MIT. Its edge is printed in the README: ndjson-source reads NDJSON (one
   JSON value per line) — NOT a JSON array (use json-source) and NOT JSON5. Blank lines are
   skipped; a malformed line is refused (exit 2) with its 1-based line number, never skipped
   or repaired. Each value is re-serialized to canonical compact JSON (number tokens and
   whitespace normalized, key order preserved), so it validates and normalizes; it reads the
   whole input, not an incremental stream.
*/
"use strict";

/* ---- the pure core ------------------------------------------------ */

// Parse NDJSON text and return the array of its per-line JSON values. Blank (empty or
// whitespace-only) lines are skipped. A malformed line throws a clean Error naming the
// 1-based line number — the CLI turns that into exit 2. Pure.
function parse(text) {
  if (typeof text !== "string") throw new Error("input must be text");
  var lines = text.split("\n");
  var values = [];
  for (var i = 0; i < lines.length; i++) {
    var line = lines[i];
    // CRLF -> LF: strip a single trailing carriage return left by the split.
    if (line.charAt(line.length - 1) === "\r") line = line.slice(0, -1);
    if (line.trim() === "") continue;      // blank line carries no record — skip, don't guess
    var value;
    try {
      value = JSON.parse(line);
    } catch (e) {
      throw new Error("line " + (i + 1) + " is not valid JSON: " + e.message);
    }
    values.push(value);
  }
  return values;
}

// Render values as JSONL text (one canonical compact JSON value per line, trailing newline
// if any). Same canonical serializer as the source-lane siblings.
function toJSONL(values) {
  var s = "";
  for (var i = 0; i < values.length; i++) s += JSON.stringify(values[i]) + "\n";
  return s;
}

/* ---- exports (browser + Node) ------------------------------------ */
if (typeof window !== "undefined") {
  window.ForestGifts = window.ForestGifts || {};
  window.ForestGifts.ndjsonSource = { parse: parse, toJSONL: toJSONL };
}
if (typeof module !== "undefined" && module.exports) {
  module.exports = { parse: parse, toJSONL: toJSONL };
}

/* ---- CLI (runs only when invoked directly, never on require) ------ */

function parseArgs(args) {
  var file;
  var i = 0;
  while (i < args.length) {
    var a = args[i];
    if (a.charAt(0) === "-" && a !== "-") {
      throw new Error("unknown option " + a);
    } else {
      if (file !== undefined) throw new Error("only one input file may be given (got a second: " + JSON.stringify(a) + ")");
      file = a;
      i += 1;
    }
  }
  return { file: file };
}

function readAll(stream) {
  return new Promise(function (resolve, reject) {
    var chunks = [];
    stream.on("data", function (c) { chunks.push(c); });
    stream.on("end", function () { resolve(Buffer.concat(chunks).toString("utf8")); });
    stream.on("error", reject);
  });
}

function helpText() {
  return (
    "ndjson-source.js — validate and normalize an NDJSON stream into canonical JSONL.\n\n" +
    "  node ndjson-source.js data.ndjson\n" +
    "  cat data.ndjson | node ndjson-source.js\n" +
    "  node ndjson-source.js --help\n\n" +
    "  [FILE]   NDJSON file to read (default: read stdin). One JSON value per line.\n\n" +
    "Emits one canonical compact JSON line per non-blank input line.\n\n" +
    "Edge: reads NDJSON (one JSON value per line) — NOT a JSON array (use json-source) and\n" +
    "NOT JSON5. Blank lines are skipped; a malformed line is refused (exit 2) with its\n" +
    "1-based line number, never skipped or repaired. Each value is re-serialized to canonical\n" +
    "compact JSON (number tokens and whitespace normalized, key order preserved). Reads the\n" +
    "whole input, not an incremental stream.\n"
  );
}

function main(argv) {
  var args = argv.slice(2);
  if (args.indexOf("--help") !== -1 || args.indexOf("-h") !== -1) {
    process.stdout.write(helpText());
    return Promise.resolve(0);
  }
  var parsed;
  try { parsed = parseArgs(args); }
  catch (e) { process.stderr.write("ndjson-source: " + e.message + "\n"); return Promise.resolve(2); }

  var getText;
  if (parsed.file !== undefined) {
    getText = new Promise(function (resolve, reject) {
      require("fs").readFile(parsed.file, "utf8", function (err, data) {
        if (err) reject(new Error("cannot read " + JSON.stringify(parsed.file) + ": " + err.code));
        else resolve(data);
      });
    });
  } else {
    getText = readAll(process.stdin);
  }

  return getText.then(function (text) {
    var values = parse(text);   // throws -> caught below
    process.stdout.write(toJSONL(values));
    return 0;
  }).catch(function (e) {
    process.stderr.write("ndjson-source: " + e.message + "\n");
    return 2;
  });
}

if (typeof require !== "undefined" && require.main === module) {
  main(process.argv).then(function (code) { process.exitCode = code; });
}

#!/usr/bin/env node
/* test_ndjson-source.js — battery for the ndjson-source gift.
   node test_ndjson-source.js  ->  exit 0 PASS / non-zero FAIL. Zero dependencies.

   Cross-checks the gift against an INDEPENDENT reduce-route oracle (a different route than
   the gift's index loop — regex line-split + reduce), frozen hand goldens, the
   canonical-re-serialization contract, blank-line skipping, CRLF handling, determinism,
   and every fail-closed edge (a malformed line, refused with its line number). */
"use strict";
var ns = require("./ndjson-source.js");

var pass = 0, fail = 0;
function ok(name, cond) { if (cond) pass++; else { fail++; console.log("  FAIL  " + name); } }
function eqS(name, a, b) {
  if (a === b) pass++;
  else { fail++; console.log("  FAIL  " + name + "\n    exp " + JSON.stringify(b) + "\n    got " + JSON.stringify(a)); }
}
function throws(name, fn) {
  var threw = false; try { fn(); } catch (e) { threw = true; }
  if (threw) pass++; else { fail++; console.log("  FAIL  " + name + " (expected a throw)"); }
}

/* ---- independent oracle: regex-split + reduce route (different than the gift's loop) ---- */
function oracle(text) {
  return text.split(/\r?\n/).reduce(function (acc, line) {
    if (line.trim() === "") return acc;              // blank lines carry no record
    return acc + JSON.stringify(JSON.parse(line)) + "\n";  // may throw -> caller counts as error
  }, "");
}

/* ---- 1. differential grid: gift == oracle on every good vector ---- */
var good = [
  "",                                          // empty -> empty stream
  "\n\n\n",                                     // all blank -> empty stream
  '{"id":1}\n{"id":2}',                         // objects, no trailing nl
  '{"id":1}\n{"id":2}\n',                       // objects, trailing nl
  "1\n2\n3",                                    // scalars
  '"x"\n"y"\n"z"',                              // string scalars
  "true\nfalse\nnull",                          // literals
  '{"a":1,"b":2}\n{"a":3,"b":4}',              // multi-field records
  '{ "a" : 1 }\n[ 1 , 2 ]',                     // whitespace normalized
  '{"n":1e3}\n[1.0, 100]',                      // number tokens normalized
  '{"b":2,"a":1}',                              // key order preserved from input
  '{"a":1}\n\n{"b":2}\n\n',                     // interior + trailing blank lines skipped
  '{"a":1}\r\n{"b":2}\r\n',                     // CRLF accepted
  '{"nested":{"k":[1,2]}}',                     // nested value verbatim-as-value
  '  {"a":1}  \n\t{"b":2}\t',                   // leading/trailing whitespace on a value line
];
for (var i = 0; i < good.length; i++) {
  eqS("differential[" + i + "] gift==oracle", ns.toJSONL(ns.parse(good[i])), oracle(good[i]));
}

/* ---- 2. hand goldens (frozen) ---- */
eqS("golden: empty -> empty stream", ns.toJSONL(ns.parse("")), "");
eqS("golden: all-blank -> empty stream", ns.toJSONL(ns.parse("\n \n\t\n")), "");
eqS("golden: objects", ns.toJSONL(ns.parse('{"id":1}\n{"id":2}')), '{"id":1}\n{"id":2}\n');
eqS("golden: scalars", ns.toJSONL(ns.parse("1\n2\n3")), "1\n2\n3\n");
eqS("golden: whitespace normalized", ns.toJSONL(ns.parse('{ "a" : 1 }')), '{"a":1}\n');
eqS("golden: number token 1e3 -> 1000", ns.toJSONL(ns.parse("[1e3]")), "[1000]\n");
eqS("golden: number token 1.0 -> 1", ns.toJSONL(ns.parse("[1.0]")), "[1]\n");
eqS("golden: key order preserved", ns.toJSONL(ns.parse('{"b":2,"a":1}')), '{"b":2,"a":1}\n');
eqS("golden: blank lines skipped", ns.toJSONL(ns.parse('{"a":1}\n\n{"b":2}')), '{"a":1}\n{"b":2}\n');
eqS("golden: CRLF stripped", ns.toJSONL(ns.parse('{"a":1}\r\n{"b":2}\r\n')), '{"a":1}\n{"b":2}\n');
eqS("golden: trailing nl same as no trailing nl", ns.toJSONL(ns.parse('{"a":1}\n')), ns.toJSONL(ns.parse('{"a":1}')));

/* ---- 3. one line per non-blank input line ---- */
var recs = ns.parse('{"a":1}\n\n{"a":2}\n{"a":3}\n');
ok("count: 3 values (2 blanks skipped)", recs.length === 3);
ok("lines: JSONL has 3 lines", ns.toJSONL(recs).split("\n").filter(Boolean).length === 3);

/* ---- 4. fail-closed (exit-2 class): a malformed line, with its line number ---- */
throws("bad: malformed line (unterminated)", function () { ns.parse('{"a":1}\n{"a":2'); });
throws("bad: bare word", function () { ns.parse("hello"); });
throws("bad: trailing garbage after a value", function () { ns.parse('{"a":1} oops'); });
throws("bad: two values on one line", function () { ns.parse('{"a":1} {"b":2}'); });
throws("bad: a JSON array top-level is not per-line NDJSON of that array — but a single [..] line IS one value; a broken one throws", function () { ns.parse("[1,2,"); });
throws("bad: non-string input", function () { ns.parse(123); });
// the error names the 1-based line number of the offending line
(function () {
  var msg = "";
  try { ns.parse('{"a":1}\n{"a":2}\nnope'); } catch (e) { msg = e.message; }
  ok("error names the 1-based line number (line 3)", /line 3\b/.test(msg));
})();
// a blank line before the bad line does NOT shift the reported number (line counts raw lines)
(function () {
  var msg = "";
  try { ns.parse('{"a":1}\n\nnope'); } catch (e) { msg = e.message; }
  ok("blank line counted in the line number (line 3)", /line 3\b/.test(msg));
})();

/* ---- 5. determinism: parse twice -> byte-identical JSONL ---- */
var buf = "";
for (var k = 0; k < 50; k++) buf += JSON.stringify({ n: k, s: "v" + k }) + "\n";
ok("determinism: two parses byte-identical", ns.toJSONL(ns.parse(buf)) === ns.toJSONL(ns.parse(buf)));

/* ---- 6. toJSONL shape ---- */
ok("toJSONL: one value per line + trailing nl", ns.toJSONL([{ a: 1 }]) === '{"a":1}\n');
ok("toJSONL: empty -> empty string", ns.toJSONL([]) === "");

/* ---- 7. mutation tripwire A: a canonicalization-dropping mutant must diverge ---- */
(function () {
  // mutant: emits the raw trimmed line instead of the re-serialized value; must diverge on
  // a non-canonical input (whitespace / number token).
  function mutant(text) {
    return text.split("\n").reduce(function (acc, line) {
      if (line.charAt(line.length - 1) === "\r") line = line.slice(0, -1);
      if (line.trim() === "") return acc;
      return acc + line.trim() + "\n";     // raw line, NOT canonical value
    }, "");
  }
  var v = '{ "a" : 1e3 }';
  ok("mutation A: raw-passthrough (skips canonicalization) is CAUGHT",
     mutant(v) !== ns.toJSONL(ns.parse(v)));
})();

/* ---- 8. mutation tripwire B: a skip-the-bad-line mutant must diverge (honesty axis) ---- */
(function () {
  // mutant: silently drops a malformed line instead of throwing; on a bad-line input the
  // gift throws (-> exit 2) while the mutant succeeds. The behavioral divergence is the bite.
  function mutantSkipBad(text) {
    return text.split("\n").reduce(function (acc, line) {
      if (line.trim() === "") return acc;
      try { return acc + JSON.stringify(JSON.parse(line)) + "\n"; }
      catch (e) { return acc; }            // swallow the bad record — dishonest
    }, "");
  }
  var v = '{"a":1}\nBROKEN\n{"b":2}';
  var giftThrew = false;
  try { ns.parse(v); } catch (e) { giftThrew = true; }
  ok("mutation B: skip-bad-line is CAUGHT (gift throws where mutant survives)",
     giftThrew === true && mutantSkipBad(v) === '{"a":1}\n{"b":2}\n');
})();

console.log((fail === 0 ? "PASS" : "FAIL") + "  " + pass + "/" + (pass + fail));
process.exit(fail === 0 ? 0 : 1);