eval/inject improved; testing won t fail on non existing javascript engine

README.md

@@ -54,8 +54,10 @@ The test framework looks for JavaScript engines such as `node`, `deno`,
 or when `JSMAKER_BROWSER_FALLBACK=1` is set.
 
 When no JavaScript engine is available, the tests generate the JavaScript test
-files and print warnings, but do not fail unless `JSMAKER_REQUIRE_ENGINE` or
+files and use an explicit `non-failing-javascript-stub`.  The stub prints notes
-`JSMAKER_REQUIRE_NODE` is set.
+to stdout, does not execute the generated JavaScript, and succeeds unless
+`JSMAKER_REQUIRE_ENGINE` or `JSMAKER_REQUIRE_NODE` is set.  This avoids package
+server failures caused solely by a missing JavaScript runtime.
 
 Useful environment variables:
 
@@ -105,8 +107,12 @@ Belangrijk:
 
 This build includes the `with-handlers` callee-position fix for inline lambda
 handlers, including rest-argument handlers such as `(lambda args ...)`. It also
-adds a Racket-like division-by-zero runtime check for `/`, so the generic
+fixes top-level `js` statement-context handling for `with-handlers`, so a
-`exn?` handler subset can catch `(/ 10 0)`.
+side-effecting catch handler does not prematurely return from the surrounding
+JavaScript wrapper.  Use `js/expression` when the value of a `with-handlers`
+form itself is needed.  The build also adds a Racket-like division-by-zero
+runtime check for `/`, so the generic `exn?` handler subset can catch
+`(/ 10 0)`.
 
 ## JavaScript use case demos
 

main.rkt

@@ -8,9 +8,10 @@
 
 (provide js js/expression)
 
-;; Convert a Racket value produced by (eval <racket-expr>) inside a js form to
+;; Convert a Racket value produced by (inject <racket-expr>) or the
-;; a JavaScript literal string.  This is a Racket -> JavaScript interpolation
+;; historical alias (eval <racket-expr>) inside a js form to a JavaScript
-;; boundary, not JavaScript eval.
+;; literal string.  This is a Racket -> JavaScript interpolation boundary,
+;; not JavaScript eval.
 (define (jsmaker-runtime-escape-js-string s)
   (define out (open-output-string))
   (for ([ch (in-string s)])
@@ -261,8 +262,9 @@
     (format "__~a_~a" prefix tmp-counter))
 
   ;; Runtime interpolation escape hatch.  Before compiling the syntax to a
-  ;; datum, the public macros replace each (eval <racket-expr>) subform by a
+  ;; datum, the public macros replace each (inject <racket-expr>) subform by a
-  ;; unique placeholder symbol.  The generated macro result is then a Racket
+  ;; unique placeholder symbol.  The historical alias (eval <racket-expr>) is
+  ;; kept for compatibility.  The generated macro result is then a Racket
   ;; expression that replaces those placeholders by JavaScript literals computed
   ;; from <racket-expr> in the use-site lexical context.  This is intentionally
   ;; different from JavaScript eval, which remains available as
@@ -276,27 +278,49 @@
     (define (fresh-eval-placeholder)
       (set! counter (add1 counter))
       (string->symbol (format "__rkt_eval_slot_~a__" counter)))
-    (define (walk d)
+    (define (syntax-head-symbol x)
-      (match d
+      (define xs (syntax->list x))
-        [(list 'quote _) d]
+      (and xs
-        [(list 'quasiquote _) d]
+           (pair? xs)
-        [(list 'eval expr)
+           (identifier? (car xs))
+           (syntax-e (car xs))))
+    (define (walk-pair p)
+      (cond
+        [(null? p) '()]
+        [(pair? p) (cons (walk (car p)) (walk-pair (cdr p)))]
+        [else (walk p)]))
+    (define (walk x)
+      (define e (if (syntax? x) (syntax-e x) x))
+      (cond
+        [(syntax? x)
+         (case (syntax-head-symbol x)
+           [(quote quasiquote) (syntax->datum x)]
+           [(inject eval)
+            (define xs (syntax->list x))
+            (cond
+              [(and xs (= (length xs) 2))
                (define ph (fresh-eval-placeholder))
-         (set! evals (append evals (list (list (symbol->string ph) expr))))
+               (set! evals (append evals (list (list (symbol->string ph) (cadr xs)))))
                ph]
-        [(? pair?) (cons (walk (car d)) (walk (cdr d)))]
+              [else (walk-pair e)])]
-        [(? vector?) (list->vector (map walk (vector->list d)))]
+           [else
-        [_ d]))
+            (cond
-    (values (walk (syntax->datum stx)) evals))
+              [(pair? e) (walk-pair e)]
+              [(vector? e) (list->vector (map walk (vector->list e)))]
+              [else e])])]
+        [(pair? e) (walk-pair e)]
+        [(vector? e) (list->vector (map walk (vector->list e)))]
+        [else e]))
+    (values (walk stx) evals))
 
   (define (wrap-compiled-js stx compiled evals)
     (for/fold ([acc #`#,compiled]) ([ev (in-list evals)])
       (define placeholder (first ev))
-      (define racket-expr-datum (second ev))
+      (define racket-expr-stx (second ev))
       #`(string-replace #,acc
                          #,placeholder
                          (jsmaker-runtime-value->js
-                          #,(datum->syntax stx racket-expr-datum)))))
+                          #,racket-expr-stx))))
 
   ;; Racket conditionals treat only #f as false.  JavaScript would also
   ;; reject 0, "", null and undefined in condition position, so every
@@ -1408,6 +1432,8 @@ if (~a !== false) return ~a;" tmp (compile-expr arg) tmp tmp)))
        (format "(~a = ~a)" (compile-assignment-target target) (compile-expr rhs))]
       [(list 'return) "undefined"]
       [(list 'return e) (compile-expr e)]
+      [(list 'inject racket-expr)
+       (compile-racket-eval racket-expr)]
       [(list 'eval racket-expr)
        (compile-racket-eval racket-expr)]
       [(list f args ...) (compile-call f args)]
@@ -1424,6 +1450,12 @@ if (~a !== false) return ~a;" tmp (compile-expr arg) tmp tmp)))
       [(list 'delete-prop! _ ...) #f]
       [(list 'while _ ...) #f]
       [(list 'for _ ...) #f]
+      ;; Top-level js is statement/program output.  with-handlers may contain
+      ;; a handler used only for side effects; compiling it as an implicit
+      ;; tail return would emit return statements inside the generated
+      ;; try/catch and prematurely leave the caller's wrapper.  Use
+      ;; js/expression when the value of with-handlers is needed.
+      [(list 'with-handlers _ ...) #f]
       [_ #t]))
 
   (define (compile-top forms)

scrbl/js-maker.scrbl

@@ -22,10 +22,13 @@
 
 @bold{js-maker} is a small, syntax-driven JavaScript generator for writing a
 practical JavaScript subset in Racket notation.  It provides two macros,
-@racket[js] and @racket[js/expression].  Both macros run at expansion time and
+@racket[js] and @racket[js/expression].  In the ordinary case the macros
-return JavaScript source code as a string.  The generated JavaScript can then be
+expand to a string containing JavaScript source code.  When the source contains
-embedded in a page, written to a file, tested with Node or another JavaScript
+@racket[(inject racket-expr)] or its historical alias @racket[(eval racket-expr)],
-engine, or used as part of a larger code-generation workflow.
+the macros instead expand to a Racket expression that computes the JavaScript
+source string at run time.  The generated JavaScript can then be embedded in a
+page, written to a file, tested with Node or another JavaScript engine, or used
+as part of a larger code-generation workflow.
 
 The package is deliberately not a full Racket compiler.  It recognizes a
 well-defined set of Racket-like forms and maps them to JavaScript while trying
@@ -60,9 +63,12 @@ Inside function bodies, the last expression is returned automatically unless it
 is already a statement form such as @racket[return], @racket[define],
 @racket[set!], @racket[while], or @racket[for].  At the top level of a
 @racket[js] form, js-maker also returns the value of a final value-producing
-form, such as @racket[let], @racket[begin], @racket[if], or
+form, such as @racket[let], @racket[begin], or @racket[if].  This makes
-@racket[with-handlers].  This makes @racket[js] output suitable as the body of a
+@racket[js] output suitable as the body of a WebView @tt{runJavaScript}
-WebView @tt{runJavaScript} wrapper function.
+wrapper function.  A top-level @racket[with-handlers] is treated as a statement
+so that a catch handler used for side effects does not prematurely return from
+the surrounding wrapper.  Use @racket[js/expression] when the value of a
+@racket[with-handlers] form itself is needed.
 
 For example:
 
@@ -71,7 +77,7 @@ For example:
   (displayln
    (js
     (let ([el (send document getElementById 'test)])
-      (set! (js-dot el innerHTML) (eval html))
+      (set! (js-dot el innerHTML) (inject html))
       #t))))
 ]
 
@@ -107,12 +113,14 @@ self-call.
 @section{Mental model}
 
 The generator is best understood as a source-to-source translator over syntax.
-The input is converted to datum form and matched against the supported surface
+Most input is converted to datum form and matched against the supported surface
 language.  The Racket code is normally not evaluated.  The deliberate exception
-is @racket[(eval racket-expr)], which interpolates a Racket value into the
+is @racket[(inject racket-expr)], and the compatible older spelling
-JavaScript source text.  The expression is evaluated in the lexical context of
+@racket[(eval racket-expr)].  These forms mark a Racket expression whose value
-the @racket[js] or @racket[js/expression] use and its value is emitted as a
+must be serialized as a JavaScript literal while the generated source string is
-JavaScript literal.  This has a few important consequences:
+being constructed.  The expression is evaluated in the lexical context of the
+@racket[js] or @racket[js/expression] use.  This has a few important
+consequences:
 
 @compact-items[
  @item{Only forms known to js-maker are translated specially.  Unknown calls are
@@ -128,19 +136,24 @@ JavaScript literal.  This has a few important consequences:
        helper functions/IIFEs to preserve semantics.}
 ]
 
-@subsection{Racket value interpolation with eval}
+@subsection{Racket value interpolation with inject}
 
-The form @racket[(eval racket-expr)] is an interpolation escape hatch inherited
+The form @racket[(inject racket-expr)] is the preferred interpolation escape
-from the original transformer.  It evaluates @racket[racket-expr] as Racket in
+hatch.  It evaluates @racket[racket-expr] as Racket in the use-site lexical
-the use-site lexical context and then splices the resulting value into the
+context and then splices the resulting value into the JavaScript source as a
-JavaScript source as a literal.  This makes surrounding Racket bindings visible
+literal.  This makes surrounding Racket bindings visible to the interpolation
-to the interpolation expression.
+expression.
+
+The older spelling @racket[(eval racket-expr)] is still accepted as an alias for
+compatibility with existing code.  New code should prefer @racket[inject],
+because the form is not JavaScript @tt{eval} and does not evaluate JavaScript
+source text.
 
 @(rkt+js
 #<<RKT
 (let ([x 10]
       [y 20])
-  (js (let ([a (eval (* x y))])
+  (js (let ([a (inject (* x y))])
         (return (* a a)))))
 RKT
 #<<JS
@@ -153,20 +166,26 @@ JS
 
 @(rkt+js
 #<<RKT
-(js/expression (array (eval (+ 1 2))
+(js/expression (array (inject (+ 1 2))
-                      (eval (string-append "a" "b"))))
+                      (inject (string-append "a" "b"))))
 RKT
 #<<JS
 [3, "ab"]
 JS
 )
 
-This is not JavaScript @tt{eval}.  To call JavaScript @tt{eval}, call the
+Neither @racket[inject] nor its alias @racket[eval] is JavaScript @tt{eval}.
-JavaScript function explicitly, for example @racket[(send window eval "1 + 2")].
+To call JavaScript @tt{eval}, call the JavaScript function explicitly, for
-Racket-side @racket[eval] is best used for constants, generated literal data,
+example @racket[(send window eval "1 + 2")].  Racket-side interpolation is best
-and small configuration values that are known while the JavaScript source is
+used for constants, generated literal data, and small configuration values that
-being constructed.  It should not be used for run-time browser state, DOM access,
+are known while the JavaScript source is being constructed.  It should not be
-or user input.
+used for run-time browser state, DOM access, or untrusted user input.
+
+The name @racket[inject] was chosen over a name such as @racket[subst] because
+it is not raw textual substitution.  js-maker serializes the Racket value to a
+JavaScript literal, including string escaping, booleans, numbers, lists, vectors,
+hashes, symbols, keywords, characters, @racket[#f], @racket[#t], @racket[null],
+and @racket[(void)].
 
 @section{Supported expression and statement forms}
 
@@ -184,7 +203,9 @@ both, depending on context:
  @item{@racket[when], @racket[unless], @racket[while], @racket[for],
        @racket[for/list], @racket[for/vector], and @racket[for/fold].}
  @item{@racket[with-handlers] for the generic @racket[exn?] case.}
- @item{@racket[(eval racket-expr)] for Racket-side value interpolation into JavaScript literals.}
+ @item{@racket[(inject racket-expr)] for Racket-side value interpolation into
+       JavaScript literals; @racket[(eval racket-expr)] is accepted as a
+       compatibility alias.}
  @item{Interop forms such as @racket[send], @racket[new], @racket[js-ref],
        @racket[js-dot], @racket[set-prop!], @racket[delete-prop!],
        @racket[js-delete], @racket[array], @racket[object],
@@ -728,11 +749,13 @@ with a JavaScript executor.  The executor module searches for engines such as
 Node, Deno, Bun, QuickJS, V8 @tt{d8}, JavaScriptCore @tt{jsc}, SpiderMonkey
 @tt{js}, and an optional Chromium fallback.  Node is the preferred default.
 
-If no JavaScript engine is available, tests are generated but execution is
+If no JavaScript engine is available, tests are generated and the framework
-skipped with clear warnings and a successful exit status.  This is intentional
+uses an explicit @tt{non-failing-javascript-stub}.  The stub prints notes to
-so package tests do not fail merely because a JavaScript runtime is missing.
+stdout, does not execute the generated JavaScript, and exits successfully.
-Set @tt{JSMAKER_REQUIRE_ENGINE=1} or @tt{JSMAKER_REQUIRE_NODE=1} to make a
+This is intentional so package tests do not fail merely because a JavaScript
-missing engine a hard failure.
+runtime is missing, especially in package-server environments.  Set
+@tt{JSMAKER_REQUIRE_ENGINE=1} or @tt{JSMAKER_REQUIRE_NODE=1} to make a missing
+engine a hard failure.
 
 Useful commands are:
 

testing/jsmaker-regression.rkt

@@ -21,6 +21,48 @@
 (unless (regexp-match? #rx"__cmp" effectful-chain)
   (error 'jsmaker-regression "effectful chained comparison should use temporaries, got: ~a" effectful-chain))
 
+(define-syntax with-id->el
+  (syntax-rules ()
+    ((_ id el expr)
+     (js (let ((el (send document getElementById (eval id))))
+           expr)))))
+
+(define (runtime-eval-macro-function id html)
+  (with-id->el id el
+    (begin
+      (set! (js-dot el innerHTML) (eval html))
+      #t)))
+
+(define runtime-eval-macro-program
+  (runtime-eval-macro-function 't "BEE"))
+(unless (and (regexp-match? #rx"getElementById\\(\"t\"\\)" runtime-eval-macro-program)
+             (regexp-match? #rx"innerHTML = \"BEE\"" runtime-eval-macro-program)
+             (regexp-match? #rx"return true;" runtime-eval-macro-program))
+  (error 'jsmaker-regression
+         "runtime eval inside a user macro should keep use-site lexical bindings, got: ~a"
+         runtime-eval-macro-program))
+
+(define-syntax with-id->el/inject
+  (syntax-rules ()
+    ((_ id el expr)
+     (js (let ((el (send document getElementById (inject id))))
+           expr)))))
+
+(define (runtime-inject-macro-function id html)
+  (with-id->el/inject id el
+    (begin
+      (set! (js-dot el innerHTML) (inject html))
+      #t)))
+
+(define runtime-inject-macro-program
+  (runtime-inject-macro-function 't "BEE"))
+(unless (and (regexp-match? #rx"getElementById\\(\"t\"\\)" runtime-inject-macro-program)
+             (regexp-match? #rx"innerHTML = \"BEE\"" runtime-inject-macro-program)
+             (regexp-match? #rx"return true;" runtime-inject-macro-program))
+  (error 'jsmaker-regression
+         "runtime inject inside a user macro should keep use-site lexical bindings, got: ~a"
+         runtime-inject-macro-program))
+
 (define simple-let-star
   (js (let* ((x 10)
              (y (+ x x)))
@@ -75,6 +117,11 @@
      (js-expression-test 'runtime-eval-lexical-let
                          (js/expression (let ([a (eval (* x y))]) (* a a)))
                          "40000"))
+   (let ([x 10]
+         [y 20])
+     (js-expression-test 'runtime-inject-lexical-let
+                         (js/expression (let ([a (inject (* x y))]) (* a a)))
+                         "40000"))
    (js-expression-test 'substring (js/expression (substring "abcdef" 1 4)) "\"bcd\"")
    (js-expression-test 'equal-list (js/expression (equal? (list 1 2) (list 1 2))) "true")
    (js-expression-test 'cond-test-only (js/expression (cond [0] [else 2])) "0")

testing/jsmaker-test-framework.rkt

@@ -8,8 +8,12 @@
          js-program-test
          write-js-test-file
          run-jsmaker-regression
+         notice-line
          warning-line)
 
+(define (notice-line who fmt . args)
+  (displayln (string-append "NOTE: " (apply format fmt args))))
+
 (define (warning-line who fmt . args)
   (displayln (string-append "WARNING: " (apply format fmt args))
              (current-error-port)))
@@ -79,10 +83,16 @@
        (error who "JavaScript regression failed with ~a; see ~a"
               (js-engine-name engine) js-path))]
     [else
+     (if (or (getenv "JSMAKER_REQUIRE_ENGINE") (getenv "JSMAKER_REQUIRE_NODE"))
+         (begin
            (warning-line who "No JavaScript engine was found; regression tests were generated but not executed.")
            (warning-line who "Generated JavaScript test file: ~a" js-path)
            (warning-line who "Tried engines: ~a" (string-join (map symbol->string (known-js-engine-names)) ", "))
            (warning-line who "Set JSMAKER_ENGINE to auto/node/deno/bun/qjs/d8/jsc/js/chromium or set JSMAKER_ENGINE_PATH.")
            (warning-line who "For backwards compatibility, JSMAKER_NODE can point to a Node executable.")
-     (when (or (getenv "JSMAKER_REQUIRE_ENGINE") (getenv "JSMAKER_REQUIRE_NODE"))
+           (error who "JavaScript engine required by environment setting"))
-       (error who "JavaScript engine required by environment setting"))]))
+         (begin
+           (notice-line who "No JavaScript engine was found; using non-failing-javascript-stub.")
+           (notice-line who "Generated JavaScript test file: ~a" js-path)
+           (notice-line who "Generated tests were not executed by a JavaScript runtime.")
+           (notice-line who "Set JSMAKER_REQUIRE_ENGINE=1 to make a missing JavaScript engine fail.")))]))

testing/jsmaker-test-runner.rkt

@@ -6,7 +6,8 @@
 
 ;; Compatibility wrapper for the older single-engine test runner name.
 ;; It now delegates to the generic framework/executor layer.  When no engine
-;; is available, the framework generates the JavaScript test file and reports a
+;; is available, the framework generates the JavaScript test file and uses an
-;; skip/warning unless JSMAKER_REQUIRE_ENGINE or JSMAKER_REQUIRE_NODE is set.
+;; explicit non-failing-javascript-stub unless JSMAKER_REQUIRE_ENGINE or
+;; JSMAKER_REQUIRE_NODE is set.
 (define (run-jsmaker-node-regression who tests js-path)
   (run-jsmaker-regression who tests js-path))