refactoring
This commit is contained in:
@@ -0,0 +1,446 @@
|
||||
/* Versioned JSON interchange for Racket Wiki concept maps. */
|
||||
|
||||
const FORMAT = "racket-wiki-cmap-bundle";
|
||||
const FORMAT_VERSION = 1;
|
||||
const SCHEMA = "/schemas/racket-wiki-cmap-bundle-v1.schema.json";
|
||||
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
||||
const TEMPORARY_ID = /^new:[A-Za-z0-9][A-Za-z0-9._-]{0,119}$/;
|
||||
const SLUG = /^[\p{L}\p{N}][\p{L}\p{N}._-]{0,119}$/u;
|
||||
const CONCEPT_KEYS = [
|
||||
"id", "label", "synopsis", "aspects", "tags", "descriptionPageSlug",
|
||||
"pageSlug", "cmapSlug", "externalUrl", "imageSource"
|
||||
];
|
||||
const PLACEMENT_CONTENT_KEYS = new Set(CONCEPT_KEYS.filter((key) => key !== "id"));
|
||||
|
||||
function clone(value) {
|
||||
return value === undefined ? undefined : JSON.parse(JSON.stringify(value));
|
||||
}
|
||||
|
||||
function validPageReference(value) {
|
||||
if (typeof value !== "string") return false;
|
||||
const separator = value.indexOf(":");
|
||||
if (separator < 0) return SLUG.test(value);
|
||||
const namespace = value.slice(0, separator);
|
||||
const slug = value.slice(separator + 1);
|
||||
return namespace.length <= 80 && SLUG.test(namespace) && SLUG.test(slug);
|
||||
}
|
||||
|
||||
function validExternalUrl(value) {
|
||||
try {
|
||||
const url = new URL(String(value));
|
||||
return url.protocol === "http:" || url.protocol === "https:";
|
||||
} catch (_error) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function decodedDocument(value) {
|
||||
let documentValue = value;
|
||||
for (let attempt = 0; attempt < 2 && typeof documentValue === "string"; attempt += 1) {
|
||||
documentValue = JSON.parse(documentValue);
|
||||
}
|
||||
return documentValue && typeof documentValue === "object" && !Array.isArray(documentValue) ?
|
||||
documentValue : {};
|
||||
}
|
||||
|
||||
function conceptContent(value) {
|
||||
const result = {};
|
||||
for (const key of CONCEPT_KEYS) {
|
||||
if (Object.prototype.hasOwnProperty.call(value || {}, key)) result[key] = clone(value[key]);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function conceptsById(documentValue) {
|
||||
return new Map((Array.isArray(documentValue.concepts) ? documentValue.concepts : [])
|
||||
.filter((concept) => concept && typeof concept.id === "string" && concept.id)
|
||||
.map((concept) => [concept.id, concept]));
|
||||
}
|
||||
|
||||
function itemConceptIds(documentValue) {
|
||||
return [...new Set((Array.isArray(documentValue.items) ? documentValue.items : [])
|
||||
.filter((item) => item && item.kind !== "phrase" && typeof item.conceptId === "string" && item.conceptId)
|
||||
.map((item) => item.conceptId))];
|
||||
}
|
||||
|
||||
function linkedMapSlugs(documentValue) {
|
||||
const concepts = conceptsById(documentValue);
|
||||
return [...new Set(itemConceptIds(documentValue)
|
||||
.map((id) => String(concepts.get(id)?.cmapSlug || "").trim())
|
||||
.filter(Boolean))];
|
||||
}
|
||||
|
||||
function linkedPageReferences(documentValue) {
|
||||
const references = new Set();
|
||||
const metadata = documentValue.metadata && typeof documentValue.metadata === "object" ?
|
||||
documentValue.metadata : {};
|
||||
if (typeof metadata.explanationPageSlug === "string" && metadata.explanationPageSlug.trim()) {
|
||||
references.add(metadata.explanationPageSlug.trim());
|
||||
}
|
||||
const concepts = conceptsById(documentValue);
|
||||
for (const id of itemConceptIds(documentValue)) {
|
||||
const concept = concepts.get(id) || {};
|
||||
for (const key of ["pageSlug", "descriptionPageSlug"]) {
|
||||
if (typeof concept[key] === "string" && concept[key].trim()) references.add(concept[key].trim());
|
||||
}
|
||||
}
|
||||
return references;
|
||||
}
|
||||
|
||||
function attachmentUrls(markdown) {
|
||||
const source = String(markdown || "");
|
||||
const urls = new Set();
|
||||
const add = (value) => {
|
||||
const url = String(value || "").trim();
|
||||
if (url.startsWith("/uploads/") && url.split("/").length >= 4) urls.add(url);
|
||||
};
|
||||
for (const match of source.matchAll(/!?\[[^\]]*\]\((\/uploads\/[^)]*)\)/g)) add(match[1]);
|
||||
for (const match of source.matchAll(/(?:src|href)\s*=\s*["'](\/uploads\/[^"']+)["']/gi)) add(match[1]);
|
||||
for (const match of source.matchAll(/\/uploads\/[^\s"'<>\\)]+/g)) add(match[0]);
|
||||
const collected = [...urls];
|
||||
return collected.filter((url) => !collected.some((other) =>
|
||||
other !== url && other.startsWith(`${url} `)));
|
||||
}
|
||||
|
||||
function attachmentName(url) {
|
||||
const encoded = String(url || "").split("/").at(-1) || "attachment.bin";
|
||||
try {
|
||||
return decodeURIComponent(encoded) || "attachment.bin";
|
||||
} catch (_error) {
|
||||
return encoded || "attachment.bin";
|
||||
}
|
||||
}
|
||||
|
||||
function replaceAttachmentUrls(markdown, replacements) {
|
||||
let result = String(markdown || "");
|
||||
const entries = replacements instanceof Map ? [...replacements.entries()] :
|
||||
Object.entries(replacements || {});
|
||||
entries.sort(([left], [right]) => right.length - left.length);
|
||||
for (const [source, target] of entries) {
|
||||
if (!source || source === target) continue;
|
||||
result = result.split(source).join(String(target));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function placementDocument(documentValue) {
|
||||
const documentCopy = clone(decodedDocument(documentValue));
|
||||
const ids = itemConceptIds(documentCopy);
|
||||
for (const concept of (Array.isArray(documentCopy.concepts) ? documentCopy.concepts : [])) {
|
||||
if (concept && typeof concept.id === "string" && !ids.includes(concept.id)) ids.push(concept.id);
|
||||
}
|
||||
documentCopy.items = (Array.isArray(documentCopy.items) ? documentCopy.items : []).map((item) => {
|
||||
if (!item || item.kind === "phrase") return item;
|
||||
return Object.fromEntries(Object.entries(item)
|
||||
.filter(([key]) => !PLACEMENT_CONTENT_KEYS.has(key)));
|
||||
});
|
||||
documentCopy.concepts = ids.map((id) => ({ id }));
|
||||
const itemIds = new Set();
|
||||
for (const item of documentCopy.items) {
|
||||
const itemId = Number(item?.id);
|
||||
if (Number.isInteger(itemId)) itemIds.add(itemId);
|
||||
}
|
||||
documentCopy.connectors = (Array.isArray(documentCopy.connectors) ?
|
||||
documentCopy.connectors : []).filter((connector) => {
|
||||
const sourceExists = itemIds.has(Number(connector?.sourceId));
|
||||
const targetExists = itemIds.has(Number(connector?.targetId));
|
||||
return sourceExists && targetExists;
|
||||
});
|
||||
return documentCopy;
|
||||
}
|
||||
|
||||
async function pageRecord(page, requestedReference, loadAttachment) {
|
||||
const markdown = String(page.markdown || "");
|
||||
const attachments = [];
|
||||
for (const url of attachmentUrls(markdown)) {
|
||||
if (typeof loadAttachment !== "function") {
|
||||
throw new Error(`Attachment loader is required for ${url}.`);
|
||||
}
|
||||
const loaded = await loadAttachment(url, requestedReference);
|
||||
if (!loaded || typeof loaded.contentBase64 !== "string") {
|
||||
throw new Error(`Attachment ${url} did not provide base64 content.`);
|
||||
}
|
||||
attachments.push({
|
||||
url,
|
||||
name: String(loaded.name || attachmentName(url)),
|
||||
mimeType: String(loaded.mimeType || "application/octet-stream"),
|
||||
contentBase64: loaded.contentBase64
|
||||
});
|
||||
}
|
||||
return {
|
||||
reference: String(requestedReference),
|
||||
title: String(page.title || page.slug || requestedReference),
|
||||
markdown,
|
||||
tags: Array.isArray(page.tags) ? page.tags.map(String) : [],
|
||||
attachments
|
||||
};
|
||||
}
|
||||
|
||||
async function buildBundle(options) {
|
||||
if (!options?.rootMap?.slug) throw new Error("A root CMap is required.");
|
||||
if (typeof options.loadConceptMap !== "function") throw new Error("loadConceptMap is required.");
|
||||
if (typeof options.loadWikiPage !== "function") throw new Error("loadWikiPage is required.");
|
||||
const maximumDepth = Math.max(0, Math.min(10, Number(options.maxDepth) || 0));
|
||||
const maps = [];
|
||||
const concepts = new Map();
|
||||
const pageReferences = new Set();
|
||||
const visited = new Set();
|
||||
const missingMaps = new Set();
|
||||
|
||||
async function collectMap(map, depth) {
|
||||
if (!map?.slug || visited.has(map.slug)) return;
|
||||
visited.add(map.slug);
|
||||
const documentValue = decodedDocument(map.document);
|
||||
maps.push({
|
||||
slug: String(map.slug),
|
||||
title: String(map.title || map.slug),
|
||||
document: placementDocument(documentValue)
|
||||
});
|
||||
for (const concept of conceptsById(documentValue).values()) {
|
||||
if (!concepts.has(concept.id)) concepts.set(concept.id, conceptContent(concept));
|
||||
}
|
||||
for (const reference of linkedPageReferences(documentValue)) pageReferences.add(reference);
|
||||
|
||||
const sourceSlug = String(documentValue.derivedView?.sourceCmapSlug || "").trim();
|
||||
if (sourceSlug && !visited.has(sourceSlug)) {
|
||||
try {
|
||||
await collectMap(await options.loadConceptMap(sourceSlug), depth);
|
||||
} catch (_error) {
|
||||
missingMaps.add(sourceSlug);
|
||||
}
|
||||
}
|
||||
if (depth >= maximumDepth) return;
|
||||
for (const slug of linkedMapSlugs(documentValue)) {
|
||||
if (visited.has(slug)) continue;
|
||||
try {
|
||||
await collectMap(await options.loadConceptMap(slug), depth + 1);
|
||||
} catch (_error) {
|
||||
missingMaps.add(slug);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await collectMap(options.rootMap, 0);
|
||||
const pages = [];
|
||||
const missingPages = [];
|
||||
for (const reference of [...pageReferences].sort()) {
|
||||
try {
|
||||
pages.push(await pageRecord(
|
||||
await options.loadWikiPage(reference), reference, options.loadAttachment));
|
||||
} catch (_error) {
|
||||
missingPages.push(reference);
|
||||
}
|
||||
}
|
||||
|
||||
const bundle = {
|
||||
$schema: SCHEMA,
|
||||
format: FORMAT,
|
||||
formatVersion: FORMAT_VERSION,
|
||||
exportedAt: options.exportedAt || new Date().toISOString(),
|
||||
generator: options.generator || "Racket Wiki",
|
||||
rootCmapSlug: String(options.rootMap.slug),
|
||||
cmaps: maps,
|
||||
concepts: [...concepts.values()],
|
||||
pages,
|
||||
missing: { cmaps: [...missingMaps].sort(), pages: missingPages }
|
||||
};
|
||||
validateBundle(bundle);
|
||||
return bundle;
|
||||
}
|
||||
|
||||
function validateBundle(bundle) {
|
||||
const errors = [];
|
||||
const issue = (path, message) => errors.push(`${path}: ${message}`);
|
||||
if (!bundle || typeof bundle !== "object" || Array.isArray(bundle)) {
|
||||
throw new Error("The import must be a JSON object.");
|
||||
}
|
||||
if (bundle.format !== FORMAT) issue("format", `must be ${FORMAT}`);
|
||||
if (bundle.formatVersion !== FORMAT_VERSION) issue("formatVersion", `must be ${FORMAT_VERSION}`);
|
||||
if (!Array.isArray(bundle.cmaps) || !bundle.cmaps.length) issue("cmaps", "must contain at least one CMap");
|
||||
if (!Array.isArray(bundle.concepts)) issue("concepts", "must be an array");
|
||||
if (!Array.isArray(bundle.pages)) issue("pages", "must be an array");
|
||||
|
||||
const conceptIds = new Set();
|
||||
const conceptLabels = new Set();
|
||||
const usedConceptIds = new Set();
|
||||
const linkedPageReferences = new Set();
|
||||
for (const [index, concept] of (Array.isArray(bundle.concepts) ? bundle.concepts : []).entries()) {
|
||||
const path = `concepts[${index}]`;
|
||||
if (!concept || typeof concept !== "object" || Array.isArray(concept)) {
|
||||
issue(path, "must be an object");
|
||||
continue;
|
||||
}
|
||||
if (typeof concept.id !== "string" || !(UUID.test(concept.id) || TEMPORARY_ID.test(concept.id))) {
|
||||
issue(`${path}.id`, "must be a UUID or a new:<name> temporary id");
|
||||
} else if (conceptIds.has(concept.id)) {
|
||||
issue(`${path}.id`, "is duplicated");
|
||||
} else conceptIds.add(concept.id);
|
||||
if (typeof concept.label !== "string" || !concept.label.trim()) issue(`${path}.label`, "is required");
|
||||
else {
|
||||
const name = concept.label.trim().toLocaleLowerCase();
|
||||
if (conceptLabels.has(name)) issue(`${path}.label`, "duplicates another concept name");
|
||||
else conceptLabels.add(name);
|
||||
}
|
||||
for (const key of ["pageSlug", "descriptionPageSlug"]) {
|
||||
if (typeof concept[key] === "string" && concept[key].trim()) {
|
||||
linkedPageReferences.add(concept[key].trim());
|
||||
if (!validPageReference(concept[key].trim())) issue(`${path}.${key}`, "must be a valid wiki page reference");
|
||||
}
|
||||
}
|
||||
if (typeof concept.cmapSlug === "string" && concept.cmapSlug.trim() && !SLUG.test(concept.cmapSlug.trim())) {
|
||||
issue(`${path}.cmapSlug`, "must be a valid CMap slug");
|
||||
}
|
||||
if (concept.externalUrl !== undefined && concept.externalUrl !== null &&
|
||||
(typeof concept.externalUrl !== "string" ||
|
||||
!concept.externalUrl.trim() || !validExternalUrl(concept.externalUrl.trim()))) {
|
||||
issue(`${path}.externalUrl`, "must be a complete http or https URL");
|
||||
}
|
||||
}
|
||||
|
||||
const mapSlugs = new Set();
|
||||
for (const [mapIndex, cmap] of (Array.isArray(bundle.cmaps) ? bundle.cmaps : []).entries()) {
|
||||
const path = `cmaps[${mapIndex}]`;
|
||||
if (!cmap || typeof cmap !== "object" || Array.isArray(cmap)) {
|
||||
issue(path, "must be an object");
|
||||
continue;
|
||||
}
|
||||
if (typeof cmap.slug !== "string" || !SLUG.test(cmap.slug)) issue(`${path}.slug`, "must be a valid CMap slug");
|
||||
else if (mapSlugs.has(cmap.slug)) issue(`${path}.slug`, "is duplicated");
|
||||
else mapSlugs.add(cmap.slug);
|
||||
if (typeof cmap.title !== "string" || !cmap.title.trim()) issue(`${path}.title`, "is required");
|
||||
const documentValue = cmap.document;
|
||||
if (!documentValue || typeof documentValue !== "object" || Array.isArray(documentValue)) {
|
||||
issue(`${path}.document`, "must be an object");
|
||||
continue;
|
||||
}
|
||||
const items = Array.isArray(documentValue.items) ? documentValue.items : [];
|
||||
const metadataReference = documentValue.metadata?.explanationPageSlug;
|
||||
if (typeof metadataReference === "string" && metadataReference.trim()) {
|
||||
linkedPageReferences.add(metadataReference.trim());
|
||||
if (!validPageReference(metadataReference.trim())) {
|
||||
issue(`${path}.document.metadata.explanationPageSlug`, "must be a valid wiki page reference");
|
||||
}
|
||||
}
|
||||
const documentConceptIds = new Set();
|
||||
for (const [referenceIndex, reference] of (Array.isArray(documentValue.concepts) ?
|
||||
documentValue.concepts : []).entries()) {
|
||||
const referencePath = `${path}.document.concepts[${referenceIndex}].id`;
|
||||
if (!reference || typeof reference.id !== "string" || !conceptIds.has(reference.id)) {
|
||||
issue(referencePath, "must reference a concept in concepts[]");
|
||||
} else if (documentConceptIds.has(reference.id)) issue(referencePath, "is duplicated within the CMap");
|
||||
else documentConceptIds.add(reference.id);
|
||||
}
|
||||
const itemIds = new Set();
|
||||
for (const [itemIndex, item] of items.entries()) {
|
||||
const itemPath = `${path}.document.items[${itemIndex}]`;
|
||||
if (!item || typeof item !== "object" || Array.isArray(item)) {
|
||||
issue(itemPath, "must be an object");
|
||||
continue;
|
||||
}
|
||||
if (!Number.isInteger(Number(item.id))) issue(`${itemPath}.id`, "must be an integer");
|
||||
else if (itemIds.has(Number(item.id))) issue(`${itemPath}.id`, "is duplicated within the CMap");
|
||||
else itemIds.add(Number(item.id));
|
||||
if (!Number.isFinite(Number(item.x))) issue(`${itemPath}.x`, "must be a number");
|
||||
if (!Number.isFinite(Number(item.y))) issue(`${itemPath}.y`, "must be a number");
|
||||
if (item.kind !== "phrase") {
|
||||
if (typeof item.conceptId !== "string" || !conceptIds.has(item.conceptId)) {
|
||||
issue(`${itemPath}.conceptId`, "must reference a concept in concepts[]");
|
||||
} else usedConceptIds.add(item.conceptId);
|
||||
}
|
||||
}
|
||||
for (const [connectorIndex, connector] of (Array.isArray(documentValue.connectors) ?
|
||||
documentValue.connectors : []).entries()) {
|
||||
const connectorPath = `${path}.document.connectors[${connectorIndex}]`;
|
||||
if (!itemIds.has(Number(connector?.sourceId))) issue(`${connectorPath}.sourceId`, "references an unknown item");
|
||||
if (!itemIds.has(Number(connector?.targetId))) issue(`${connectorPath}.targetId`, "references an unknown item");
|
||||
}
|
||||
}
|
||||
if (typeof bundle.rootCmapSlug !== "string" || !mapSlugs.has(bundle.rootCmapSlug)) {
|
||||
issue("rootCmapSlug", "must reference a CMap in cmaps[]");
|
||||
}
|
||||
for (const id of conceptIds) {
|
||||
if (!usedConceptIds.has(id)) issue(`concepts[id=${id}]`, "must occur as a diagram placement");
|
||||
}
|
||||
|
||||
const pageReferences = new Set();
|
||||
for (const [index, page] of (Array.isArray(bundle.pages) ? bundle.pages : []).entries()) {
|
||||
const path = `pages[${index}]`;
|
||||
if (!page || typeof page !== "object" || Array.isArray(page)) {
|
||||
issue(path, "must be an object");
|
||||
continue;
|
||||
}
|
||||
if (typeof page.reference !== "string" || !validPageReference(page.reference)) issue(`${path}.reference`, "must be a valid wiki page reference");
|
||||
else if (pageReferences.has(page.reference)) issue(`${path}.reference`, "is duplicated");
|
||||
else pageReferences.add(page.reference);
|
||||
if (typeof page.title !== "string" || !page.title.trim()) issue(`${path}.title`, "is required");
|
||||
if (typeof page.markdown !== "string") issue(`${path}.markdown`, "must be a string");
|
||||
if (!Array.isArray(page.tags) || !page.tags.every((tag) => typeof tag === "string")) {
|
||||
issue(`${path}.tags`, "must be an array of strings");
|
||||
}
|
||||
if (page.attachments !== undefined && !Array.isArray(page.attachments)) {
|
||||
issue(`${path}.attachments`, "must be an array");
|
||||
}
|
||||
const attachmentReferences = new Set();
|
||||
for (const [attachmentIndex, attachment] of (Array.isArray(page.attachments) ?
|
||||
page.attachments : []).entries()) {
|
||||
const attachmentPath = `${path}.attachments[${attachmentIndex}]`;
|
||||
if (!attachment || typeof attachment !== "object" || Array.isArray(attachment)) {
|
||||
issue(attachmentPath, "must be an object");
|
||||
continue;
|
||||
}
|
||||
if (typeof attachment.url !== "string" || !attachment.url.startsWith("/uploads/")) {
|
||||
issue(`${attachmentPath}.url`, "must be a local /uploads/ URL");
|
||||
} else if (attachmentReferences.has(attachment.url)) {
|
||||
issue(`${attachmentPath}.url`, "is duplicated within the page");
|
||||
} else attachmentReferences.add(attachment.url);
|
||||
if (typeof attachment.name !== "string" || !attachment.name.trim()) {
|
||||
issue(`${attachmentPath}.name`, "is required");
|
||||
}
|
||||
if (typeof attachment.mimeType !== "string" || !attachment.mimeType.trim()) {
|
||||
issue(`${attachmentPath}.mimeType`, "is required");
|
||||
}
|
||||
if (typeof attachment.contentBase64 !== "string" ||
|
||||
!/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(attachment.contentBase64)) {
|
||||
issue(`${attachmentPath}.contentBase64`, "must be valid base64");
|
||||
}
|
||||
}
|
||||
}
|
||||
const explicitlyMissingPages = new Set(Array.isArray(bundle.missing?.pages) ? bundle.missing.pages : []);
|
||||
for (const reference of linkedPageReferences) {
|
||||
if (!pageReferences.has(reference) && !explicitlyMissingPages.has(reference)) {
|
||||
issue(`pages[reference=${reference}]`, "is required by a linked concept or CMap explanation");
|
||||
}
|
||||
}
|
||||
|
||||
if (errors.length) {
|
||||
const error = new Error(`Invalid CMap bundle:\n${errors.slice(0, 20).join("\n")}`);
|
||||
error.validationErrors = errors;
|
||||
throw error;
|
||||
}
|
||||
return bundle;
|
||||
}
|
||||
|
||||
function preparedMapDocument(bundle, cmap) {
|
||||
validateBundle(bundle);
|
||||
const byId = new Map(bundle.concepts.map((concept) => [concept.id, concept]));
|
||||
const documentValue = clone(cmap.document);
|
||||
const ids = itemConceptIds(documentValue);
|
||||
for (const reference of (Array.isArray(documentValue.concepts) ? documentValue.concepts : [])) {
|
||||
if (reference?.id && !ids.includes(reference.id)) ids.push(reference.id);
|
||||
}
|
||||
documentValue.concepts = ids.map((id) => clone(byId.get(id)));
|
||||
return documentValue;
|
||||
}
|
||||
|
||||
export {
|
||||
FORMAT,
|
||||
FORMAT_VERSION,
|
||||
SCHEMA,
|
||||
attachmentUrls,
|
||||
buildBundle,
|
||||
decodedDocument,
|
||||
placementDocument,
|
||||
preparedMapDocument,
|
||||
replaceAttachmentUrls,
|
||||
validateBundle
|
||||
};
|
||||
Reference in New Issue
Block a user