Merge pull request #75 from pyscript/crawable-js-modules

Make polyscript.js_modules crawable

Author: WebReflection

docs/README.md

@@ -264,6 +264,26 @@ In *polyscript*, this is possible by defining one or more `[js_modules.X]` field
   * `[js_modules.main]` is a list of *source* -> *module name* pairs, similarly to how `[files]` field work, where the *module* name will then be reachable via `polyscript.js_modules.actual_name` in both *main* and *worker* world. As the *main* module lands on the main thread, where there is also likely some UI, it is also possible to define one or more related *CSS* to that module, as long as they target the very same name (see the example to better understand).
   * `[js_modules.worker]` is a list of *source* -> *module name* pairs that actually land only in `<script type="x" worker>` cases. These modules are still reachable through the very same `polyscript.js_modules.actual_name` convention and this feature is meant to be used for modules that only works best, or work regardless, outside the *main* world. As example, if your *JS* module implies that `document` or `window` references, among other *DOM* related APIs, are globally available, it means that that module should be part of the `[js_modules.main]` list instead ... however, if the module works out of the box in a *Worker* environment, it is best for performance reasons to explicitly define such module under this field. Please note that *CSS* files are not accepted within this list because there's no way *CSS* can be useful or land in any meaningful way within a *Worker* environment.
 
+All registeed modules can be then imported as such:
+
+```python
+# just import js_modules and reach names after
+from polyscript import js_modules
+js_modules.my_module.util()
+
+# import directly and reach names after
+from polyscript.js_modules import my_module
+my_module.util()
+
+# import deeply up to the module exports
+from polyscript.js_modules.my_module import util
+util()
+
+# import default or other fields with aliases
+from polyscript.js_modules.other import defalut as fn
+fn()
+```
+
 ### js_modules config example
 
 **TOML**

docs/core.js

@@ -1,7 +1,7 @@
 import '@ungap/with-resolvers';
 import { $$ } from 'basic-devtools';
 
-import { JSModules, assign, create, createOverload, createResolved, dedent, defineProperty, nodeInfo } from './utils.js';
+import { JSModules, assign, create, createOverload, createResolved, dedent, defineProperty, nodeInfo, registerJSModules } from './utils.js';
 import { getDetails } from './script-handler.js';
 import { registry as defaultRegistry, prefixes, configs } from './interpreters.js';
 import { getRuntimeID } from './loader.js';
@@ -99,7 +99,7 @@ export const handleCustomType = (node) => {
                         XWorker,
                     };
 
-                    module.registerJSModule(interpreter, 'polyscript.js_modules', JSModules);
+                    registerJSModules(runtime, module, interpreter, JSModules);
                     module.registerJSModule(interpreter, 'polyscript', {
                         js_modules: JSModules,
                         XWorker,

docs/core.js.map

@@ -4,7 +4,7 @@ import $xworker from './worker/class.js';
 import workerURL from './worker/url.js';
 import { getRuntime, getRuntimeID } from './loader.js';
 import { registry } from './interpreters.js';
-import { JSModules, all, dispatch, resolve, defineProperty, nodeInfo } from './utils.js';
+import { JSModules, all, dispatch, resolve, defineProperty, nodeInfo, registerJSModules } from './utils.js';
 import { getText } from './fetch-utils.js';
 
 const getRoot = (script) => {
@@ -60,7 +60,7 @@ const execute = async (script, source, XWorker, isAsync) => {
             configurable: true,
             get: () => script,
         });
-        module.registerJSModule(interpreter, 'polyscript.js_modules', JSModules);
+        registerJSModules(type, module, interpreter, JSModules);
         module.registerJSModule(interpreter, 'polyscript', {
             js_modules: JSModules,
             XWorker,

esm/custom.js

@@ -72,6 +72,24 @@ export const JSModules = new Proxy(jsModules, {
     ownKeys: map => [...map.keys()],
 });
 
+const has = (_, field) => !field.startsWith('_');
+
+const proxy = (modules, name) => new Proxy(
+    modules,
+    { has, get: (modules, field) => modules[name][field] }
+);
+
+export const registerJSModules = (type, module, interpreter, modules) => {
+    // Pyodide resolves JS modules magically
+    if (type === 'pyodide') return;
+
+    // other runtimes need this pretty ugly dance (it works though)
+    const jsModules = 'polyscript.js_modules';
+    for (const name of Reflect.ownKeys(modules))
+        module.registerJSModule(interpreter, `${jsModules}.${name}`, proxy(modules, name));
+    module.registerJSModule(interpreter, jsModules, modules);
+};
+
 export const importJS = (source, name) => import(source).then(esm => {
     jsModules.set(name, { ...esm });
 });

esm/script-handler.js

@@ -7,7 +7,7 @@
 import * as JSON from '@ungap/structured-clone/json';
 import coincident from 'coincident/window';
 
-import { assign, create, createFunction, createOverload, createResolved, dispatch } from '../utils.js';
+import { assign, create, createFunction, createOverload, createResolved, dispatch, registerJSModules } from '../utils.js';
 import createJSModules from './js_modules.js';
 import { configs, registry } from '../interpreters.js';
 import { getRuntime, getRuntimeID } from '../loader.js';
@@ -140,7 +140,7 @@ add('message', ({ data: { options, config: baseURL, code, hooks } }) => {
 
             let target = '';
 
-            details.registerJSModule(interpreter, 'polyscript.js_modules', JSModules);
+            registerJSModules(type, details, interpreter, JSModules);
             details.registerJSModule(interpreter, 'polyscript', {
                 xworker,
                 js_modules: JSModules,

esm/utils.js

@@ -1,43 +1,43 @@
-import { absoluteURL, defineProperties, defineProperty, entries, isCSS, js_modules } from '../utils.js';
+import { absoluteURL, entries, isArray, isCSS, js_modules } from '../utils.js';
 import { base } from '../interpreter/_utils.js';
 
+const has = (modules, name) => modules.has(name);
+
+const ownKeys = modules => [...modules.keys()];
+
+const proxy = (modules, window, sync, baseURL) => new Proxy(modules, {
+    has,
+    ownKeys,
+    get: (modules, name) => {
+        let value = modules.get(name);
+        if (isArray(value)) {
+            let sources = value;
+            value = null;
+            for (let source of sources) {
+                source = absoluteURL(source, baseURL);
+                if (isCSS(source)) sync.importCSS(source);
+                else {
+                    sync.importJS(source, name);
+                    value = window[js_modules].get(name);
+                }
+            }
+            modules.set(name, value);
+        }
+        return value;
+    },
+});
+
 export default (window, sync, mainModules) => {
-    const JSModules = {};
-    const descriptors = {};
-    const known = new Set;
-    const ops = new Map;
-    for (const [name, value] of globalThis[js_modules]) {
-        known.add(name);
-        descriptors[name] = { value };
-    }
-    // define lazy main modules resolution
+    let modules = globalThis[js_modules], baseURL = '';
     if (mainModules) {
+        baseURL = base.get(mainModules);
         for (let [source, module] of entries(mainModules)) {
-            // ignore modules already defined in worker
-            if (known.has(module)) continue;
-            let sources = ops.get(module);
-            if (!sources) ops.set(module, (sources = []));
-            sources.push(source);
-        }
-        for (const [name, sources] of ops) {
-            descriptors[name] = {
-                configurable: true,
-                get() {
-                    let value;
-                    for (let source of sources) {
-                        source = absoluteURL(source, base.get(mainModules));
-                        if (isCSS(source)) sync.importCSS(source);
-                        else {
-                            sync.importJS(source, name);
-                            value = window[js_modules].get(name);
-                        }
-                    }
-                    // override the getter and make it no more configurable
-                    defineProperty(JSModules, name, { configurable: false, get: () => value });
-                    return value;
-                }
-            };
+            let value = modules.get(module);
+            if (!value || isArray(value)) {
+                modules.set(module, value || (value = []));
+                value.push(source);
+            }
         }
     }
-    return defineProperties(JSModules, descriptors);
+    return proxy(modules, window, sync, baseURL);
 };

esm/worker/_template.js

@@ -83,6 +83,6 @@
         "sticky-module": "^0.1.1"
     },
     "worker": {
-        "blob": "sha256-gFg6QJJiDmvnxzhq0SoEdF6pDhzuRcvFEbZVZRxImpU="
+        "blob": "sha256-XS1OjXlDepikiPW23EqHwHDRYBPrT4+6/ILHLhu8E6c="
     }
 }

esm/worker/js_modules.js

@@ -0,0 +1,63 @@
+<!doctype html>
+<html lang="en">
+    <head>
+        <meta charset="UTF-8">
+        <meta name="viewport" content="width=device-width,initial-scale=1.0">
+        <title>Polyscript JS Modules</title>
+        <style>#main-map, #worker-map { height: 320px; } h3 { margin-bottom: 0; }</style>
+        <link rel="stylesheet" href="style.css">
+        <script type="module" src="../core.js"></script>
+    </head>
+    <body>
+        <!--
+            main only:      npx static-handler .
+            main/worker:    npx static-handler --coop --coep .
+
+            Note: --corp breaks the tiles server
+        -->
+
+        <h3>Main</h3>
+        <div id="main-map"></div>
+        <script type="pyodide" config="./modules.toml">
+            # needed to fix pyodide proxies
+            from pyodide.ffi import to_js
+
+            from polyscript import js_modules
+            from polyscript.js_modules import leaflet as L
+
+            print(js_modules.random_js.default)
+
+            center = to_js([51.505, -0.09])
+            mark = to_js([51.5, -0.09])
+            map = L.map('main-map').setView(center, 13)
+
+            L.tileLayer(
+                'https://tile.openstreetmap.org/{z}/{x}/{y}.png',
+                to_js({"attribution": '&copy; <a href="https://www.openstreetmap.org/copyright">OpenStreetMap</a> contributors'})
+            ).addTo(map)
+
+            L.marker(mark).addTo(map).bindPopup('Modules on Main are a no brainer 👍').openPopup()
+        </script>
+
+        <hr>
+
+        <h3>Worker</h3>
+        <div id="worker-map"></div>
+        <script type="pyodide" config="./modules.toml" worker>
+            from polyscript.js_modules import random_js, leaflet as L
+
+            print(random_js.default)
+
+            center = [51.505, -0.09]
+            mark = [51.5, -0.09]
+            map = L.map('worker-map').setView(center, 13)
+
+            L.tileLayer(
+                'https://tile.openstreetmap.org/{z}/{x}/{y}.png',
+                {"attribution": '&copy; <a href="https://www.openstreetmap.org/copyright">OpenStreetMap</a> contributors'}
+            ).addTo(map)
+
+            L.marker(mark).addTo(map).bindPopup('Modules on Worker are awesome 🥳').openPopup()
+        </script>
+    </body>
+</html>