Files
racket-wiki/static/js/cmap/model/interchange.js
T

535 lines
24 KiB
JavaScript

/**
* Versioned JSON interchange for the Racket Wiki CMap model.
*
* The bundle separates shared concept content from map-local placement data.
* `buildBundle` follows map and page references, embeds the referenced page
* attachments, and validates the resulting object. Import code calls
* `validateBundle` first and then uses `preparedMapDocument` to join shared
* concepts back into a map document. The helpers in this module deliberately
* return detached JSON values so callers cannot mutate an input map or bundle.
*/
const FORMAT = "racket-wiki-cmap-bundle";
const FORMAT_VERSION = 1;
const SCHEMA = "/schemas/racket-wiki-cmap-bundle-v1.schema.json";
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
const TEMPORARY_ID = /^new:[A-Za-z0-9][A-Za-z0-9._-]{0,119}$/;
const SLUG = /^[\p{L}\p{N}][\p{L}\p{N}._-]{0,119}$/u;
const CONCEPT_KEYS = [
"id", "label", "synopsis", "aspects", "tags", "descriptionPageSlug",
"pageSlug", "cmapSlug", "externalUrl", "imageSource"
];
const PLACEMENT_CONTENT_KEYS = new Set(CONCEPT_KEYS.filter((key) => key !== "id"));
/** Return a detached JSON-compatible copy while preserving undefined. */
function clone(value) {
return value === undefined ? undefined : JSON.parse(JSON.stringify(value));
}
/**
* Check a wiki page reference in either `slug` or `namespace:slug` form.
* The check is shared by concept links, map metadata and exported pages.
*/
function validPageReference(value) {
if (typeof value !== "string") return false;
const separator = value.indexOf(":");
if (separator < 0) return SLUG.test(value);
const namespace = value.slice(0, separator);
const slug = value.slice(separator + 1);
return namespace.length <= 80 && SLUG.test(namespace) && SLUG.test(slug);
}
/** Accept only complete HTTP(S) URLs for external concept links. */
function validExternalUrl(value) {
try {
const url = new URL(String(value));
return url.protocol === "http:" || url.protocol === "https:";
} catch (_error) {
return false;
}
}
/**
* Decode the one or two historical JSON string wrappers used by storage.
* Invalid or non-object values become an empty document; bundle validation
* remains responsible for rejecting malformed imported data.
*/
function decodedDocument(value) {
let documentValue = value;
for (let attempt = 0; attempt < 2 && typeof documentValue === "string"; attempt += 1) {
documentValue = JSON.parse(documentValue);
}
return documentValue && typeof documentValue === "object" && !Array.isArray(documentValue) ?
documentValue : {};
}
/** Copy only shared concept fields into the bundle-level concept table. */
function conceptContent(value) {
const result = {};
for (const key of CONCEPT_KEYS) {
if (Object.prototype.hasOwnProperty.call(value || {}, key)) result[key] = clone(value[key]);
}
return result;
}
/** Index valid document concepts by their stable identity. */
function conceptsById(documentValue) {
return new Map((Array.isArray(documentValue.concepts) ? documentValue.concepts : [])
.filter((concept) => concept && typeof concept.id === "string" && concept.id)
.map((concept) => [concept.id, concept]));
}
/** Return unique concept ids used by non-phrase placements. */
function itemConceptIds(documentValue) {
return [...new Set((Array.isArray(documentValue.items) ? documentValue.items : [])
.filter((item) => item && item.kind !== "phrase" && typeof item.conceptId === "string" && item.conceptId)
.map((item) => item.conceptId))];
}
/** Find linked CMap slugs in the concepts actually placed on a document. */
function linkedMapSlugs(documentValue) {
const concepts = conceptsById(documentValue);
return [...new Set(itemConceptIds(documentValue)
.map((id) => String(concepts.get(id)?.cmapSlug || "").trim())
.filter(Boolean))];
}
/**
* Collect page references from map metadata and placed concepts.
* Unplaced concept records are intentionally ignored because they are not
* part of the visible map dependency graph.
*/
function linkedPageReferences(documentValue) {
const references = new Set();
const metadata = documentValue.metadata && typeof documentValue.metadata === "object" ?
documentValue.metadata : {};
if (typeof metadata.explanationPageSlug === "string" && metadata.explanationPageSlug.trim()) {
references.add(metadata.explanationPageSlug.trim());
}
const concepts = conceptsById(documentValue);
for (const id of itemConceptIds(documentValue)) {
const concept = concepts.get(id) || {};
for (const key of ["pageSlug", "descriptionPageSlug"]) {
if (typeof concept[key] === "string" && concept[key].trim()) references.add(concept[key].trim());
}
}
return references;
}
/**
* Extract local upload URLs from Markdown and HTML-like markup.
* A Set removes duplicates; the final prefix check removes a shorter URL
* accidentally captured from a URL containing a space.
*/
function attachmentUrls(markdown) {
const source = String(markdown || "");
const urls = new Set();
const add = (value) => {
const url = String(value || "").trim();
if (url.startsWith("/uploads/") && url.split("/").length >= 4) urls.add(url);
};
for (const match of source.matchAll(/!?\[[^\]]*\]\((\/uploads\/[^)]*)\)/g)) add(match[1]);
for (const match of source.matchAll(/(?:src|href)\s*=\s*["'](\/uploads\/[^"']+)["']/gi)) add(match[1]);
for (const match of source.matchAll(/\/uploads\/[^\s"'<>\\)]+/g)) add(match[0]);
const collected = [...urls];
return collected.filter((url) => !collected.some((other) =>
other !== url && other.startsWith(`${url} `)));
}
/** Derive a readable fallback filename from an upload URL. */
function attachmentName(url) {
const encoded = String(url || "").split("/").at(-1) || "attachment.bin";
try {
return decodeURIComponent(encoded) || "attachment.bin";
} catch (_error) {
return encoded || "attachment.bin";
}
}
/** Replace upload URLs after imported attachments receive new server URLs. */
function replaceAttachmentUrls(markdown, replacements) {
let result = String(markdown || "");
const entries = replacements instanceof Map ? [...replacements.entries()] :
Object.entries(replacements || {});
entries.sort(([left], [right]) => right.length - left.length);
for (const [source, target] of entries) {
if (!source || source === target) continue;
result = result.split(source).join(String(target));
}
return result;
}
/**
* Strip shared concept fields from placements while preserving layout data.
* It also keeps only connectors whose two local item endpoints still exist,
* because export must not emit dangling layout relations.
*/
function placementDocument(documentValue) {
const documentCopy = clone(decodedDocument(documentValue));
const ids = itemConceptIds(documentCopy);
for (const concept of (Array.isArray(documentCopy.concepts) ? documentCopy.concepts : [])) {
if (concept && typeof concept.id === "string" && !ids.includes(concept.id)) ids.push(concept.id);
}
documentCopy.items = (Array.isArray(documentCopy.items) ? documentCopy.items : []).map((item) => {
if (!item || item.kind === "phrase") return item;
return Object.fromEntries(Object.entries(item)
.filter(([key]) => !PLACEMENT_CONTENT_KEYS.has(key)));
});
documentCopy.concepts = ids.map((id) => ({ id }));
const itemIds = new Set();
for (const item of documentCopy.items) {
const itemId = Number(item?.id);
if (Number.isInteger(itemId)) itemIds.add(itemId);
}
documentCopy.connectors = (Array.isArray(documentCopy.connectors) ?
documentCopy.connectors : []).filter((connector) => {
const sourceExists = itemIds.has(Number(connector?.sourceId));
const targetExists = itemIds.has(Number(connector?.targetId));
return sourceExists && targetExists;
});
return documentCopy;
}
/**
* Convert one wiki page and all uploads referenced by its Markdown into a
* bundle page record. The attachment loader is called once per unique URL.
*/
async function pageRecord(page, requestedReference, loadAttachment) {
const markdown = String(page.markdown || "");
const attachments = [];
for (const url of attachmentUrls(markdown)) {
if (typeof loadAttachment !== "function") {
throw new Error(`Attachment loader is required for ${url}.`);
}
const loaded = await loadAttachment(url, requestedReference);
if (!loaded || typeof loaded.contentBase64 !== "string") {
throw new Error(`Attachment ${url} did not provide base64 content.`);
}
attachments.push({
url,
name: String(loaded.name || attachmentName(url)),
mimeType: String(loaded.mimeType || "application/octet-stream"),
contentBase64: loaded.contentBase64
});
}
return {
reference: String(requestedReference),
title: String(page.title || page.slug || requestedReference),
markdown,
tags: Array.isArray(page.tags) ? page.tags.map(String) : [],
attachments
};
}
/**
* Build and validate a complete export bundle.
*
* @param {object} options Export options and asynchronous repository loaders.
* @param {object} options.rootMap Root map with `slug`, `title` and `document`.
* @param {Function} options.loadConceptMap Loads a linked map by slug.
* @param {Function} options.loadWikiPage Loads a linked wiki page by reference.
* @param {Function} [options.loadAttachment] Loads base64 upload content.
* @param {number} [options.maxDepth=0] Maximum depth for linked CMaps.
* @returns {Promise<object>} A validated, detached interchange bundle.
* @throws {Error} When required loaders are absent or a loaded record is invalid.
* @sideeffects Calls the supplied map, page and attachment loaders.
*
* `collectMap` traverses map dependencies and uses `placementDocument` for
* layout-only map records. It collects shared concepts and page references
* separately; page records are then built and the final aggregate is checked
* by `validateBundle` before it is returned.
*/
async function buildBundle(options) {
if (!options?.rootMap?.slug) throw new Error("A root CMap is required.");
if (typeof options.loadConceptMap !== "function") throw new Error("loadConceptMap is required.");
if (typeof options.loadWikiPage !== "function") throw new Error("loadWikiPage is required.");
const maximumDepth = Math.max(0, Math.min(10, Number(options.maxDepth) || 0));
const maps = [];
const concepts = new Map();
const pageReferences = new Set();
const visited = new Set();
const missingMaps = new Set();
async function collectMap(map, depth) {
if (!map?.slug || visited.has(map.slug)) return;
visited.add(map.slug);
const documentValue = decodedDocument(map.document);
maps.push({
slug: String(map.slug),
title: String(map.title || map.slug),
document: placementDocument(documentValue)
});
for (const concept of conceptsById(documentValue).values()) {
if (!concepts.has(concept.id)) concepts.set(concept.id, conceptContent(concept));
}
for (const reference of linkedPageReferences(documentValue)) pageReferences.add(reference);
const sourceSlug = String(documentValue.derivedView?.sourceCmapSlug || "").trim();
if (sourceSlug && !visited.has(sourceSlug)) {
try {
await collectMap(await options.loadConceptMap(sourceSlug), depth);
} catch (_error) {
missingMaps.add(sourceSlug);
}
}
if (depth >= maximumDepth) return;
for (const slug of linkedMapSlugs(documentValue)) {
if (visited.has(slug)) continue;
try {
await collectMap(await options.loadConceptMap(slug), depth + 1);
} catch (_error) {
missingMaps.add(slug);
}
}
}
await collectMap(options.rootMap, 0);
const pages = [];
const missingPages = [];
for (const reference of [...pageReferences].sort()) {
try {
pages.push(await pageRecord(
await options.loadWikiPage(reference), reference, options.loadAttachment));
} catch (_error) {
missingPages.push(reference);
}
}
const bundle = {
$schema: SCHEMA,
format: FORMAT,
formatVersion: FORMAT_VERSION,
exportedAt: options.exportedAt || new Date().toISOString(),
generator: options.generator || "Racket Wiki",
rootCmapSlug: String(options.rootMap.slug),
cmaps: maps,
concepts: [...concepts.values()],
pages,
missing: { cmaps: [...missingMaps].sort(), pages: missingPages }
};
validateBundle(bundle);
return bundle;
}
/**
* Validate the complete bundle contract and return the original bundle.
*
* @param {object} bundle Candidate bundle to validate.
* @returns {object} The same bundle object after successful validation.
* @throws {Error} With `validationErrors` when one or more contract checks fail.
*
* Validation also builds the concept, map, item and page-reference indexes
* needed to detect duplicate identities, dangling references and missing
* linked pages. Up to twenty errors are included in the message while the
* complete list remains available on `error.validationErrors`.
*/
function validateBundle(bundle) {
const errors = [];
const issue = (path, message) => errors.push(`${path}: ${message}`);
if (!bundle || typeof bundle !== "object" || Array.isArray(bundle)) {
throw new Error("The import must be a JSON object.");
}
if (bundle.format !== FORMAT) issue("format", `must be ${FORMAT}`);
if (bundle.formatVersion !== FORMAT_VERSION) issue("formatVersion", `must be ${FORMAT_VERSION}`);
if (!Array.isArray(bundle.cmaps) || !bundle.cmaps.length) issue("cmaps", "must contain at least one CMap");
if (!Array.isArray(bundle.concepts)) issue("concepts", "must be an array");
if (!Array.isArray(bundle.pages)) issue("pages", "must be an array");
// Validate the shared concept table and collect its page dependencies.
const conceptIds = new Set();
const conceptLabels = new Set();
const usedConceptIds = new Set();
const linkedPageReferences = new Set();
for (const [index, concept] of (Array.isArray(bundle.concepts) ? bundle.concepts : []).entries()) {
const path = `concepts[${index}]`;
if (!concept || typeof concept !== "object" || Array.isArray(concept)) {
issue(path, "must be an object");
continue;
}
if (typeof concept.id !== "string" || !(UUID.test(concept.id) || TEMPORARY_ID.test(concept.id))) {
issue(`${path}.id`, "must be a UUID or a new:<name> temporary id");
} else if (conceptIds.has(concept.id)) {
issue(`${path}.id`, "is duplicated");
} else conceptIds.add(concept.id);
if (typeof concept.label !== "string" || !concept.label.trim()) issue(`${path}.label`, "is required");
else {
const name = concept.label.trim().toLocaleLowerCase();
if (conceptLabels.has(name)) issue(`${path}.label`, "duplicates another concept name");
else conceptLabels.add(name);
}
for (const key of ["pageSlug", "descriptionPageSlug"]) {
if (typeof concept[key] === "string" && concept[key].trim()) {
linkedPageReferences.add(concept[key].trim());
if (!validPageReference(concept[key].trim())) issue(`${path}.${key}`, "must be a valid wiki page reference");
}
}
if (typeof concept.cmapSlug === "string" && concept.cmapSlug.trim() && !SLUG.test(concept.cmapSlug.trim())) {
issue(`${path}.cmapSlug`, "must be a valid CMap slug");
}
if (concept.externalUrl !== undefined && concept.externalUrl !== null &&
(typeof concept.externalUrl !== "string" ||
!concept.externalUrl.trim() || !validExternalUrl(concept.externalUrl.trim()))) {
issue(`${path}.externalUrl`, "must be a complete http or https URL");
}
}
// Validate map identities, placements, connectors and map-level page links.
const mapSlugs = new Set();
for (const [mapIndex, cmap] of (Array.isArray(bundle.cmaps) ? bundle.cmaps : []).entries()) {
const path = `cmaps[${mapIndex}]`;
if (!cmap || typeof cmap !== "object" || Array.isArray(cmap)) {
issue(path, "must be an object");
continue;
}
if (typeof cmap.slug !== "string" || !SLUG.test(cmap.slug)) issue(`${path}.slug`, "must be a valid CMap slug");
else if (mapSlugs.has(cmap.slug)) issue(`${path}.slug`, "is duplicated");
else mapSlugs.add(cmap.slug);
if (typeof cmap.title !== "string" || !cmap.title.trim()) issue(`${path}.title`, "is required");
const documentValue = cmap.document;
if (!documentValue || typeof documentValue !== "object" || Array.isArray(documentValue)) {
issue(`${path}.document`, "must be an object");
continue;
}
const items = Array.isArray(documentValue.items) ? documentValue.items : [];
const metadataReference = documentValue.metadata?.explanationPageSlug;
if (typeof metadataReference === "string" && metadataReference.trim()) {
linkedPageReferences.add(metadataReference.trim());
if (!validPageReference(metadataReference.trim())) {
issue(`${path}.document.metadata.explanationPageSlug`, "must be a valid wiki page reference");
}
}
const documentConceptIds = new Set();
for (const [referenceIndex, reference] of (Array.isArray(documentValue.concepts) ?
documentValue.concepts : []).entries()) {
const referencePath = `${path}.document.concepts[${referenceIndex}].id`;
if (!reference || typeof reference.id !== "string" || !conceptIds.has(reference.id)) {
issue(referencePath, "must reference a concept in concepts[]");
} else if (documentConceptIds.has(reference.id)) issue(referencePath, "is duplicated within the CMap");
else documentConceptIds.add(reference.id);
}
const itemIds = new Set();
for (const [itemIndex, item] of items.entries()) {
const itemPath = `${path}.document.items[${itemIndex}]`;
if (!item || typeof item !== "object" || Array.isArray(item)) {
issue(itemPath, "must be an object");
continue;
}
if (!Number.isInteger(Number(item.id))) issue(`${itemPath}.id`, "must be an integer");
else if (itemIds.has(Number(item.id))) issue(`${itemPath}.id`, "is duplicated within the CMap");
else itemIds.add(Number(item.id));
if (!Number.isFinite(Number(item.x))) issue(`${itemPath}.x`, "must be a number");
if (!Number.isFinite(Number(item.y))) issue(`${itemPath}.y`, "must be a number");
if (item.kind !== "phrase") {
if (typeof item.conceptId !== "string" || !conceptIds.has(item.conceptId)) {
issue(`${itemPath}.conceptId`, "must reference a concept in concepts[]");
} else usedConceptIds.add(item.conceptId);
}
}
for (const [connectorIndex, connector] of (Array.isArray(documentValue.connectors) ?
documentValue.connectors : []).entries()) {
const connectorPath = `${path}.document.connectors[${connectorIndex}]`;
if (!itemIds.has(Number(connector?.sourceId))) issue(`${connectorPath}.sourceId`, "references an unknown item");
if (!itemIds.has(Number(connector?.targetId))) issue(`${connectorPath}.targetId`, "references an unknown item");
}
}
if (typeof bundle.rootCmapSlug !== "string" || !mapSlugs.has(bundle.rootCmapSlug)) {
issue("rootCmapSlug", "must reference a CMap in cmaps[]");
}
for (const id of conceptIds) {
if (!usedConceptIds.has(id)) issue(`concepts[id=${id}]`, "must occur as a diagram placement");
}
// Validate page records and their embedded attachment payloads.
const pageReferences = new Set();
for (const [index, page] of (Array.isArray(bundle.pages) ? bundle.pages : []).entries()) {
const path = `pages[${index}]`;
if (!page || typeof page !== "object" || Array.isArray(page)) {
issue(path, "must be an object");
continue;
}
if (typeof page.reference !== "string" || !validPageReference(page.reference)) issue(`${path}.reference`, "must be a valid wiki page reference");
else if (pageReferences.has(page.reference)) issue(`${path}.reference`, "is duplicated");
else pageReferences.add(page.reference);
if (typeof page.title !== "string" || !page.title.trim()) issue(`${path}.title`, "is required");
if (typeof page.markdown !== "string") issue(`${path}.markdown`, "must be a string");
if (!Array.isArray(page.tags) || !page.tags.every((tag) => typeof tag === "string")) {
issue(`${path}.tags`, "must be an array of strings");
}
if (page.attachments !== undefined && !Array.isArray(page.attachments)) {
issue(`${path}.attachments`, "must be an array");
}
const attachmentReferences = new Set();
for (const [attachmentIndex, attachment] of (Array.isArray(page.attachments) ?
page.attachments : []).entries()) {
const attachmentPath = `${path}.attachments[${attachmentIndex}]`;
if (!attachment || typeof attachment !== "object" || Array.isArray(attachment)) {
issue(attachmentPath, "must be an object");
continue;
}
if (typeof attachment.url !== "string" || !attachment.url.startsWith("/uploads/")) {
issue(`${attachmentPath}.url`, "must be a local /uploads/ URL");
} else if (attachmentReferences.has(attachment.url)) {
issue(`${attachmentPath}.url`, "is duplicated within the page");
} else attachmentReferences.add(attachment.url);
if (typeof attachment.name !== "string" || !attachment.name.trim()) {
issue(`${attachmentPath}.name`, "is required");
}
if (typeof attachment.mimeType !== "string" || !attachment.mimeType.trim()) {
issue(`${attachmentPath}.mimeType`, "is required");
}
if (typeof attachment.contentBase64 !== "string" ||
!/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(attachment.contentBase64)) {
issue(`${attachmentPath}.contentBase64`, "must be valid base64");
}
}
}
// Reconcile all collected page dependencies with present or declared-missing pages.
const explicitlyMissingPages = new Set(Array.isArray(bundle.missing?.pages) ? bundle.missing.pages : []);
for (const reference of linkedPageReferences) {
if (!pageReferences.has(reference) && !explicitlyMissingPages.has(reference)) {
issue(`pages[reference=${reference}]`, "is required by a linked concept or CMap explanation");
}
}
// Report all discovered issues together so import callers can repair a bundle in one pass.
if (errors.length) {
const error = new Error(`Invalid CMap bundle:\n${errors.slice(0, 20).join("\n")}`);
error.validationErrors = errors;
throw error;
}
return bundle;
}
/**
* Rejoin bundle-level concept content with one placement-only map document.
*
* @param {object} bundle A bundle that satisfies `validateBundle`.
* @param {object} cmap One entry from `bundle.cmaps`.
* @returns {object} A detached document suitable for repository import.
* @throws {Error} When the bundle is invalid or a referenced concept is absent.
*/
function preparedMapDocument(bundle, cmap) {
validateBundle(bundle);
const byId = new Map(bundle.concepts.map((concept) => [concept.id, concept]));
const documentValue = clone(cmap.document);
const ids = itemConceptIds(documentValue);
for (const reference of (Array.isArray(documentValue.concepts) ? documentValue.concepts : [])) {
if (reference?.id && !ids.includes(reference.id)) ids.push(reference.id);
}
documentValue.concepts = ids.map((id) => clone(byId.get(id)));
return documentValue;
}
export {
FORMAT,
FORMAT_VERSION,
SCHEMA,
attachmentUrls,
buildBundle,
decodedDocument,
placementDocument,
preparedMapDocument,
replaceAttachmentUrls,
validateBundle
};