schema-filter

#!/usr/bin/env node
/* schema-filter.js — pass only the JSONL records that match a declared JSON shape.
   Dependency-free, deterministic, one pass, order-stable. Runs in Node or a browser. MIT.

   WHAT IT IS. Give it a stream of records — one JSON object per line (JSONL) — and a
   SHAPE you declare as a set of typed fields, and it emits, verbatim and in input
   order, exactly the records that match EVERY declared field. Everything else is
   dropped. It is a SUBSET filter: the output is a subset of the input, byte-for-byte
   per surviving record, so `schema-filter` in a pipe never invents or reshapes a
   record — it only decides which ones pass.

   A FIELD CONSTRAINT names one field and the JSON TYPE it must hold:

       --field    NAME TYPE     require NAME present AND of TYPE (fail-closed if absent)
       --optional NAME TYPE     if NAME is present it must be TYPE; if absent, it passes

   TYPE is one of a documented, closed set of seven JSON types:

       string   a JSON string
       number   a finite JSON number (integer or fractional)
       integer  a finite JSON number with no fractional part
       boolean  true or false
       object   a JSON object (not an array, not null)
       array    a JSON array
       null     the JSON literal null

   Declare more than one field and they AND together: a record survives only if it
   matches all of them. Declare none and every well-formed record passes (the identity
   filter) — a deliberate, documented default, not an error.

   THE MISSING-FIELD POLICY (the whole honesty story). A `--field` constraint is a
   claim the record must satisfy, and a record that LACKS that field cannot be shown to
   satisfy it, so it is DROPPED. This is fail-closed — absence is not a match. Use
   `--optional` for the "if this field is present it must be a date, otherwise leave the
   record alone" shape: an optional field that is absent passes; an optional field that
   is present but the WRONG type still fails. The choice is per field, declared once, so
   a stream is filtered under one explicit shape.

   TYPE, NOT VALUE. schema-filter tests a value's TYPE, never its meaning. A record with
   the right types passes even if the values are nonsense: `{"age": -999}` matches
   `--field age integer`. It does not coerce (the string "5" does NOT match `number`),
   does not infer a schema from the data, does not reshape, sort, or de-duplicate
   surviving records. `number` excludes non-finite values by construction (valid JSON
   cannot carry NaN or Infinity, so this only ever matters for a hostile embedder).
   `integer` is a type test (no fractional part), never a rounding.

   DETERMINISM. Records are tested and emitted in input order; a surviving record is
   written back byte-for-byte as it arrived (the original line, trailing \r trimmed), so
   the same stream and the same shape yield byte-identical output on every machine and
   every run.

   STRICTNESS. A filter is only trustworthy if it refuses to guess:
     - Every non-blank line must be a JSON OBJECT. A line that is not valid JSON, or is
       valid JSON but not an object (a number, string, array, null), is a HARD ERROR
       (exit 2) naming the line — never a silent skip.
     - An unknown TYPE token in a `--field`/`--optional` declaration is a hard error
       (exit 2) at startup, before any record is read — a shape you cannot express is a
       bug in the caller's declaration, surfaced loudly, not an empty result.

   USAGE
     cat people.jsonl | node schema-filter.js --field name string --field age integer
     node schema-filter.js --field id string --optional email string in.jsonl
     node schema-filter.js --field tags array < items.jsonl
     node schema-filter.js --help

   Each non-blank line is one JSON object. Blank lines are skipped. A trailing \r
   (CRLF files) is trimmed. Surviving records are emitted one per line, verbatim.

   Exit codes: 0 success (with or without survivors) · 2 input error (missing file, a
   directory, a malformed constraint, an unknown type, or a line that is not a JSON
   object). Always a clean one-line message on stderr, never a stack trace.

   Released under MIT. Its edge is printed in the README: this is a SUBSET filter over a
   DECLARED shape. It filters on the fields and types YOU name — it does not infer a
   schema, does not reshape or reformat surviving records, does not sort, and does not
   de-duplicate. A record matching every declared type passes even if its values are
   wrong: the constraint tests a value's TYPE, never its truth.
*/
"use strict";

/* ---- the pure core ------------------------------------------------ */

// The closed set of declarable JSON types. A shape you cannot express is refused.
var TYPES = {
  string: 1, number: 1, integer: 1, boolean: 1, object: 1, array: 1, "null": 1
};

// Does one JSON value match one declared type? Total over the seven types;
// an out-of-set type never matches (the CLI refuses those before this runs).
function matchType(val, type) {
  switch (type) {
    case "string":  return typeof val === "string";
    case "number":  return typeof val === "number" && isFinite(val);
    case "integer": return typeof val === "number" && isFinite(val) && Math.floor(val) === val;
    case "boolean": return typeof val === "boolean";
    case "object":  return val !== null && typeof val === "object" && !Array.isArray(val);
    case "array":   return Array.isArray(val);
    case "null":    return val === null;
  }
  return false;
}

// A parsed constraint: { field, type, optional }. Does one record satisfy it?
function satisfies(rec, c) {
  var has = Object.prototype.hasOwnProperty.call(rec, c.field);
  if (!has) return !!c.optional;            // missing: required -> fail, optional -> pass
  return matchType(rec[c.field], c.type);   // present: must match the declared type
}

// The public filter: JSONL text + parsed constraints -> the surviving records,
// in input order, each as { line, record }.
//   constraints: [ {field, type, optional}, ... ]
// Throws a clean, line-named Error on a line that is not a JSON object.
function filter(text, constraints) {
  var lines = String(text).split("\n");
  var out = [];
  var i, line, rec;

  for (i = 0; i < lines.length; i++) {
    line = lines[i];
    if (line.charCodeAt(line.length - 1) === 0x0d) line = line.slice(0, -1); // trim \r
    if (line.length === 0) continue; // blank line is not a record

    try { rec = JSON.parse(line); }
    catch (e) {
      throw new Error("line " + (i + 1) + " is not valid JSON: " +
        JSON.stringify(line.slice(0, 40)));
    }
    if (rec === null || typeof rec !== "object" || Array.isArray(rec)) {
      throw new Error("line " + (i + 1) + " is not a JSON object (got " +
        (rec === null ? "null" : Array.isArray(rec) ? "array" : typeof rec) + "): " +
        JSON.stringify(line.slice(0, 40)));
    }

    var pass = true;
    for (var k = 0; k < constraints.length; k++) {
      if (!satisfies(rec, constraints[k])) { pass = false; break; }
    }
    if (pass) out.push({ line: line, record: rec });
  }
  return out;
}

/* ---- exports (browser + Node) ------------------------------------ */
if (typeof window !== "undefined") {
  window.ForestGifts = window.ForestGifts || {};
  window.ForestGifts.schemaFilter = filter;
  window.ForestGifts.schemaFilterMatchType = matchType;
}
if (typeof module !== "undefined" && module.exports) {
  module.exports = { filter: filter, satisfies: satisfies, matchType: matchType };
}

/* ---- CLI (runs only when invoked directly, never on require) ------ */

// Parse argv into { constraints, files }. Throws a clean Error on a bad declaration.
function parseArgs(args) {
  var constraints = [];
  var files = [];
  var i = 0;
  while (i < args.length) {
    var a = args[i];
    if (a === "--field" || a === "--optional") {
      var optional = a === "--optional";
      var field = args[i + 1], type = args[i + 2];
      if (field === undefined || type === undefined || field.charAt(0) === "-") {
        throw new Error(a + " requires FIELD TYPE (e.g. " + a + " age integer)");
      }
      if (!Object.prototype.hasOwnProperty.call(TYPES, type)) {
        throw new Error(a + " " + field + ": unknown type " + JSON.stringify(type) +
          " (declare one of: string number integer boolean object array null)");
      }
      constraints.push({ field: field, type: type, optional: optional });
      i += 3;
    } else if (a.charAt(0) === "-") {
      throw new Error("unknown option " + a);
    } else {
      files.push(a); i += 1;
    }
  }
  return { constraints: constraints, files: files };
}

function run(text, constraints) {
  var survivors = filter(text, constraints);
  var s = "";
  for (var i = 0; i < survivors.length; i++) s += survivors[i].line + "\n";
  return s;
}

function main(argv) {
  var args = argv.slice(2);
  if (args.indexOf("--help") !== -1 || args.indexOf("-h") !== -1) {
    process.stdout.write(
      "schema-filter.js — pass only the JSONL records that match a declared JSON shape.\n\n" +
      "  cat people.jsonl | node schema-filter.js --field name string --field age integer\n" +
      "  node schema-filter.js --field id string --optional email string in.jsonl\n" +
      "  node schema-filter.js --field tags array < items.jsonl\n" +
      "  node schema-filter.js --help\n\n" +
      "  --field    NAME TYPE   require NAME present AND of TYPE (fail-closed if absent)\n" +
      "  --optional NAME TYPE   if NAME is present it must be TYPE; if absent, it passes\n\n" +
      "TYPE is one of: string number integer boolean object array null\n" +
      "Multiple fields AND together. No fields = every well-formed record passes.\n" +
      "Surviving records are emitted VERBATIM, in input order.\n\n" +
      "Edge: this is a SUBSET filter over a DECLARED shape. It tests a value's TYPE,\n" +
      "never its meaning; it does not coerce, infer a schema, reshape, sort, or\n" +
      "de-duplicate. Absence fails a --field and passes an --optional. A non-object\n" +
      "line or an unknown type is a hard error (exit 2).\n"
    );
    return 0;
  }

  var parsed;
  try { parsed = parseArgs(args); }
  catch (e) { process.stderr.write("schema-filter: " + e.message + "\n"); return 2; }

  function emit(text) {
    try { process.stdout.write(run(text, parsed.constraints)); return 0; }
    catch (e) { process.stderr.write("schema-filter: " + e.message + "\n"); return 2; }
  }

  if (parsed.files.length > 0) {
    var fs = require("fs");
    var text;
    try { text = fs.readFileSync(parsed.files[0], "utf8"); }
    catch (e) {
      process.stderr.write("schema-filter: cannot read " + parsed.files[0] +
        " (" + (e.code === "EISDIR" ? "is a directory" : (e.code || "read error")) + ")\n");
      return 2;
    }
    return emit(text);
  }

  // stdin
  var chunks = [];
  process.stdin.on("data", function (d) { chunks.push(d); });
  process.stdin.on("end", function () {
    process.exitCode = emit(Buffer.concat(chunks).toString("utf8"));
  });
  return 0;
}

if (typeof require !== "undefined" && require.main === module) {
  process.exitCode = main(process.argv);
}

#!/usr/bin/env node
/* test_schema-filter.js — out-of-band battery for the schema-filter gift.

   Exercises the exported pure core (filter / satisfies / matchType) against an
   INDEPENDENTLY authored oracle — a second, from-spec type checker written with a
   different route (a typeof/predicate map, not the gift's switch) — plus hand goldens
   and every documented edge. Node only; no dependencies. Exit 0 all-pass / 1 fail.
*/
"use strict";
var sf = require("./schema-filter.js");

var pass = 0, fail = 0;
function check(name, cond) {
  if (cond) { pass++; }
  else { fail++; console.log("  FAIL  " + name); }
}
function J(v) { return JSON.stringify(v); }

/* ---- independent oracle: a second, from-spec type checker ---------------- */
// Different route than the gift's switch: an explicit predicate table.
var ORACLE = {
  string:  function (v) { return typeof v === "string"; },
  number:  function (v) { return typeof v === "number" && v === v && v !== Infinity && v !== -Infinity; },
  integer: function (v) { return typeof v === "number" && v === v && v !== Infinity && v !== -Infinity && parseInt(v, 10) === v && v % 1 === 0; },
  boolean: function (v) { return v === true || v === false; },
  object:  function (v) { return typeof v === "object" && v !== null && !(v instanceof Array); },
  array:   function (v) { return v instanceof Array; },
  "null":  function (v) { return v === null; }
};
function oracleSatisfies(rec, c) {
  var has = Object.prototype.hasOwnProperty.call(rec, c.field);
  if (!has) return !!c.optional;
  return ORACLE[c.type](rec[c.field]);
}
function oracleSurvivors(recs, constraints) {
  return recs.filter(function (r) {
    return constraints.every(function (c) { return oracleSatisfies(r, c); });
  }).map(function (r) { return JSON.stringify(r); });
}
// Drive the gift's pure filter over a JSONL rendering of the records.
function giftSurvivors(recs, constraints) {
  var text = recs.map(function (r) { return JSON.stringify(r); }).join("\n") + "\n";
  return sf.filter(text, constraints).map(function (s) { return s.line; });
}

/* ---- matchType: every type, positive + negative ------------------------- */
var typeVectors = [
  ["string",  "hi", true], ["string", 5, false], ["string", null, false],
  ["number",  5, true], ["number", 5.5, true], ["number", "5", false], ["number", true, false],
  ["integer", 5, true], ["integer", 5.0, true], ["integer", 5.5, false], ["integer", "5", false],
  ["boolean", true, true], ["boolean", false, true], ["boolean", 0, false], ["boolean", "true", false],
  ["object",  {}, true], ["object", { a: 1 }, true], ["object", [], false], ["object", null, false],
  ["array",   [], true], ["array", [1, 2], true], ["array", {}, false], ["array", "x", false],
  ["null",    null, true], ["null", 0, false], ["null", "", false], ["null", {}, false]
];
typeVectors.forEach(function (v) {
  check("matchType(" + J(v[1]) + ", " + v[0] + ") === " + v[2], sf.matchType(v[1], v[0]) === v[2]);
  // cross-check against the independent oracle
  check("matchType == oracle [" + v[0] + ", " + J(v[1]) + "]", sf.matchType(v[1], v[0]) === ORACLE[v[0]](v[1]));
});

/* ---- the integer/number honesty distinction (the gift's sharp edge) ------ */
check("3.5 is number, NOT integer", sf.matchType(3.5, "number") === true && sf.matchType(3.5, "integer") === false);
check("3.0 IS integer (no fractional part)", sf.matchType(3.0, "integer") === true);
check("string \"3\" is NEITHER number nor integer (no coercion)",
  sf.matchType("3", "number") === false && sf.matchType("3", "integer") === false);

/* ---- scenario battery: gift filter == independent oracle ----------------- */
var scenarios = [
  {
    name: "single --field integer",
    recs: [{ n: 1 }, { n: 2.5 }, { n: "3" }, { n: 4 }],
    constraints: [{ field: "n", type: "integer", optional: false }]
  },
  {
    name: "two --field AND (name string + age integer)",
    recs: [
      { name: "a", age: 30 }, { name: "b", age: 30.5 }, { name: 5, age: 30 },
      { name: "c" }, { age: 40 }, { name: "d", age: 40 }
    ],
    constraints: [
      { field: "name", type: "string", optional: false },
      { field: "age", type: "integer", optional: false }
    ]
  },
  {
    name: "--field required missing DROPS",
    recs: [{ id: "has", x: "y" }, { id: "missing" }],
    constraints: [{ field: "x", type: "string", optional: false }]
  },
  {
    name: "--optional absent PASSES, present-wrong FAILS",
    recs: [{ id: "absent" }, { id: "ok", email: "e" }, { id: "wrong", email: 5 }],
    constraints: [{ field: "email", type: "string", optional: true }]
  },
  {
    name: "array + object + boolean + null types",
    recs: [
      { tags: [], meta: {}, active: true, deleted: null },
      { tags: {}, meta: {}, active: true, deleted: null },
      { tags: [], meta: [], active: true, deleted: null },
      { tags: [], meta: {}, active: 1, deleted: null },
      { tags: [], meta: {}, active: true, deleted: 0 }
    ],
    constraints: [
      { field: "tags", type: "array", optional: false },
      { field: "meta", type: "object", optional: false },
      { field: "active", type: "boolean", optional: false },
      { field: "deleted", type: "null", optional: false }
    ]
  },
  {
    name: "no constraints = identity",
    recs: [{ a: 1 }, { b: 2 }, { c: 3 }],
    constraints: []
  }
];
scenarios.forEach(function (s) {
  check("gift == oracle [" + s.name + "]", J(giftSurvivors(s.recs, s.constraints)) === J(oracleSurvivors(s.recs, s.constraints)));
});

/* ---- hand goldens (independent of the oracle) ---------------------------- */
check("golden: integer keeps [1,4], drops 2.5 and \"3\"",
  J(giftSurvivors([{ n: 1 }, { n: 2.5 }, { n: "3" }, { n: 4 }], [{ field: "n", type: "integer", optional: false }]))
  === J([JSON.stringify({ n: 1 }), JSON.stringify({ n: 4 })]));
check("golden: optional absent kept, wrong-type dropped",
  J(giftSurvivors([{ id: "absent" }, { id: "wrong", email: 5 }], [{ field: "email", type: "string", optional: true }]))
  === J([JSON.stringify({ id: "absent" })]));

/* ---- verbatim passthrough + order-stability ------------------------------ */
(function () {
  var text = '{"z":1,"a":2}\n{"b":3}\n';  // key order preserved verbatim, not re-serialized
  var got = sf.filter(text, [{ field: "z", type: "integer", optional: false }]).map(function (s) { return s.line; });
  check("verbatim: original line returned (key order preserved)", J(got) === J(['{"z":1,"a":2}']));
})();
(function () {
  var recs = [{ n: 5 }, { n: 1 }, { n: 9 }, { n: 2 }];
  var got = giftSurvivors(recs, [{ field: "n", type: "integer", optional: false }]);
  check("order-stable (5,1,9,2 preserved)",
    J(got) === J([JSON.stringify({ n: 5 }), JSON.stringify({ n: 1 }), JSON.stringify({ n: 9 }), JSON.stringify({ n: 2 })]));
})();

/* ---- CRLF + blank lines -------------------------------------------------- */
(function () {
  var text = '{"n":1}\r\n\r\n{"n":2}\r\n';
  var got = sf.filter(text, [{ field: "n", type: "integer", optional: false }]).map(function (s) { return s.line; });
  check("CRLF trimmed, blank lines skipped", J(got) === J(['{"n":1}', '{"n":2}']));
})();

/* ---- strictness: non-object lines throw --------------------------------- */
function throws(text, constraints) {
  try { sf.filter(text, constraints); return false; } catch (e) { return true; }
}
check("bare number line throws", throws("42\n", []));
check("array line throws", throws("[1,2]\n", []));
check("string line throws", throws('"hi"\n', []));
check("null line throws", throws("null\n", []));
check("malformed JSON throws", throws("{bad\n", []));
check("well-formed object does NOT throw", !throws('{"a":1}\n', []));

/* ---- determinism: two runs identical ------------------------------------ */
(function () {
  var recs = scenarios[1].recs, cs = scenarios[1].constraints;
  check("deterministic across two filter() calls", J(giftSurvivors(recs, cs)) === J(giftSurvivors(recs, cs)));
})();

console.log("");
var verdict = fail === 0 ? "PASS" : "FAIL";
console.log(verdict + ": " + pass + " checks passed, " + fail + " failed  [test_schema-filter]");
process.exit(fail === 0 ? 0 : 1);