external imports

README.md

@@ -98,6 +98,23 @@ Operators that are not supported by design (not pure functional):
   - increment, decrement
   - `delete`
 
+## Importing third-party libs
+
+Sometimes you want to import third party library that uses imperative language constructs. You may want to use it to perform side-effects or maybe it mutates data inside but still provides functional interface (does not mutate function arguments). Good example of such library is [bignumber.js](https://github.com/MikeMcl/bignumber.js/) - it makes a lot of mutating assignments inside, but `BigNumber` instances are immutable.
+
+To use `bignumber.js` you add an `external pragma` before the import:
+
+```
+/* external */
+import BigNumber from './path/to/bignumber.mjs';
+```
+
+`external pragma` is just a comment that contains only the literal string `external` (both styles for comments and extra whitespaces are allowed). Now the module is imported as a black box - you cannot debug `BigNumber` methods.
+
+![External import](docs/images/external_import.png)
+
+Currently every external is loaded once and cached until Leporello is restarted (TODO what happens if we load modules in iframe and then recreate iframe)
+
 ## Hotkeys
 
 See built-in Help

src/ast_utils.js

@@ -43,6 +43,7 @@ export const map_destructuring_identifiers = (node, mapper) => {
 export const collect_imports = module => {
   const imports = module.stmts
     .filter(n => n.type == 'import')
+    .filter(n => !n.is_external)
     .map(n => 
       n.imports.map(i => 
         ({name: i.value, module: n.full_import_path})

src/cmd.js

@@ -1,4 +1,5 @@
-import {map_object, pick_keys, collect_nodes_with_parents} from './utils.js'
+import {map_object, pick_keys, collect_nodes_with_parents, uniq} 
+  from './utils.js'
 import {
   is_eq, is_child, ancestry, ancestry_inc, map_tree,
   find_leaf, find_fn_by_location, find_node, find_error_origin_node
@@ -82,12 +83,97 @@ const run_code = (s, index, dirty_files) => {
     calltree_node_by_loc: null,
     // TODO keep selection_state?
     selection_state: null,
+    loading_external_imports_state: null,
   }
 
   if(!state.parse_result.ok) {
     return state
   } 
 
+  const external_import_nodes = 
+    Object
+      .entries(state.parse_result.modules)
+      .map(([module_name, node]) => 
+        node
+          .children
+          .filter(c => c.type == 'import' && c.is_external)
+          .map(node => ({node, module_name}))
+      )
+      .flat()
+
+  const external_imports = uniq( 
+    external_import_nodes.map(i => i.node.full_import_path)
+  )
+
+  if(
+    external_imports.length != 0
+    &&
+    (
+      state.external_imports_cache == null
+      ||
+      external_imports.some(i => state.external_imports_cache[i] == null)
+    )
+  ) {
+    return {...state, 
+      loading_external_imports_state: {
+        index,
+        external_imports,
+        external_import_nodes,
+      }
+    }
+  } else {
+    return external_imports_loaded(
+      state, 
+      state, 
+      state.external_imports_cache, 
+      index
+    )
+  }
+
+}
+
+const do_external_imports_loaded = (
+  state, 
+  prev_state, 
+  external_imports, 
+  index,
+) => {
+  if(
+    state.loading_external_imports_state 
+    != 
+    prev_state.loading_external_imports_state
+  ) {
+    // code was modified after loading started, discard
+    return state
+  }
+
+  if(external_imports != null) {
+    const errors = new Set(
+      Object
+        .entries(external_imports)
+        .filter(([url, result]) => !result.ok)
+        .map(([url, result]) => url)
+    )
+    if(errors.size != 0) {
+      const problems = state
+        .loading_external_imports_state
+        .external_import_nodes
+        .filter(({node}) => errors.has(node.full_import_path))
+        .map(({node, module_name}) => ({
+          index: node.index,
+          message: external_imports[node.full_import_path].error.message,
+          module: module_name,
+        }))
+      return {...state,
+        parse_result: {
+          ok: false,
+          cache: state.parse_result.cache,
+          problems,
+        }
+      }
+    }
+  }
+
   const node = find_call_node(state, index)
 
   if(
@@ -96,7 +182,11 @@ const run_code = (s, index, dirty_files) => {
     ||
     node.type == 'do' /* toplevel AST node */
   ) {
-    const result = eval_modules(state.parse_result.modules, state.parse_result.sorted)
+    const result = eval_modules(
+      state.parse_result.modules, 
+      state.parse_result.sorted,
+      external_imports
+    )
     const next = apply_eval_result(state, result)
 
     if(node == state.parse_result.modules[root_calltree_module(next)]) {
@@ -117,6 +207,7 @@ const run_code = (s, index, dirty_files) => {
   const result = eval_modules(
     state.parse_result.modules, 
     state.parse_result.sorted,
+    external_imports,
     {index: node.index, module: state.current_module},
   )
 
@@ -145,19 +236,44 @@ const run_code = (s, index, dirty_files) => {
   )
 }
 
+const external_imports_loaded = (
+  state, 
+  prev_state, 
+  external_imports, 
+  maybe_index,
+) => {
+  // index saved in loading_external_imports_state maybe stale, if cursor was
+  // moved after
+  // TODO refactor, make index controlled property saved in state?
+  const index = maybe_index ?? state.loading_external_imports_state.index
+
+  // TODO after edit we should fire embed_value_explorer, but we dont do it
+  // here because we dont have cursor position (index) here (see comment
+  // above). Currently it is fixed by having `external_imports_cache`, so code
+  // goes async path only on first external imports load
+  return {
+    ...do_external_imports_loaded(state, prev_state, external_imports, index), 
+    external_imports_cache: external_imports,
+    loading_external_imports_state: null
+  }
+}
+
 const input = (state, code, index) => {
   const files = {...state.files, [state.current_module]: code}
   const next = run_code({...state, files}, index, [state.current_module])
-  const effects1 = next.current_module == ''
+  const effect_save = next.current_module == ''
     ? {type: 'save_to_localstorage', args: ['code', code]}
     : {type: 'write', args: [
       next.current_module,
       next.files[next.current_module],
     ]}
+  if(next.loading_external_imports_state != null) {
+    return {state: next, effects: [effect_save]}
+  }
   const {state: next2, effects: effects2} = do_move_cursor(next, index)
   return {
     state: next2, 
-    effects: [effects1, effects2],
+    effects: [effect_save, effects2],
   }
 }
 
@@ -675,5 +791,6 @@ export const COMMANDS = {
   goto_problem,
   move_cursor,
   eval_selection,
+  external_imports_loaded,
   calltree: calltree_commands,
 }

src/effects.js

@@ -2,6 +2,30 @@ import {write_file} from './filesystem.js'
 import {color_file} from './color.js'
 import {root_calltree_node, calltree_node_loc} from './calltree.js'
 import {FLAGS} from './feature_flags.js'
+import {exec} from './index.js'
+
+const load_external_imports = async state => {
+  if(state.loading_external_imports_state == null) {
+    return
+  }
+  const urls = state.loading_external_imports_state.external_imports
+  const results = await Promise.allSettled(
+    urls.map(u => import(u))
+  )
+  const modules = Object.fromEntries(
+    results.map((r, i) => (
+      [
+        urls[i],
+        {
+          ok: r.status == 'fulfilled',
+          error: r.reason,
+          module: r.value,
+        }
+      ]
+    ))
+  )
+  exec('external_imports_loaded', state /* becomes prev_state */, modules)
+}
 
 const ensure_session = (ui, state, file = state.current_module) => {
   ui.editor.ensure_session(file, state.files[file])
@@ -84,6 +108,7 @@ export const render_initial_state = (ui, state) => {
     ui.render_debugger(state)
     render_coloring(ui, state)
   }
+  load_external_imports(state)
 }
 
 export const render_common_side_effects = (prev, next, command, ui) => {
@@ -111,11 +136,15 @@ export const render_common_side_effects = (prev, next, command, ui) => {
     ui.editor.switch_session(next.current_module)
   }
 
+  if(prev.loading_external_imports_state != next.loading_external_imports_state) {
+    load_external_imports(next)
+  }
+
   if(prev.parse_result != next.parse_result) {
     render_parse_result(ui, next)
   }
 
-  if(!next.parse_result.ok) {
+  if(!next.parse_result.ok || next.loading_external_imports_state != null) {
 
     ui.calltree.clear_calltree()
     ui.editor.for_each_session((file, session) => clear_coloring(ui, file))
@@ -124,6 +153,7 @@ export const render_common_side_effects = (prev, next, command, ui) => {
   } else {
 
     if(
+      // TODO refactor this condition
       prev.current_calltree_node == null 
       ||
       prev.calltree_changed_token != next.calltree_changed_token
@@ -232,5 +262,6 @@ export const EFFECTS = {
       ui.eval.clear_value_or_error()
     }
   },
+
 }
 

src/eval.js

@@ -235,9 +235,11 @@ const codegen = (node, cxt, parent) => {
   }
 }
 
-export const eval_modules = (modules, sorted, location) => {
+export const eval_modules = (modules, sorted, external_imports, location) => {
   // TODO gensym __modules, __exports
 
+  // TODO bug if module imported twice, once as external and as regular
+
   const codestring = 
     `
     let children, prev_children
@@ -435,7 +437,7 @@ export const eval_modules = (modules, sorted, location) => {
     }
 
     const run = entrypoint => {
-      const __modules = {}
+      const __modules = {...external_imports}
       let current_call
 
     `
@@ -483,7 +485,13 @@ export const eval_modules = (modules, sorted, location) => {
     }
     `
 
-  const actions = (new Function(codestring))()
+  const actions = new Function('external_imports', codestring)(
+    external_imports == null
+    ? null
+    : map_object(external_imports, (name, {module}) => 
+        ({exports: module, is_external: true})
+      )
+  )
 
   const calltree_actions =  {
     expand_calltree_node: (node) => {
@@ -524,8 +532,10 @@ export const eval_modules = (modules, sorted, location) => {
 const assign_code_calltree = (modules, calltree) =>
   map_object(
     calltree,
-    (module, {calls, exports}) => {
+    (module, {calls, exports, is_external}) => {
-      return {exports, calls: assign_code(modules, calls, modules[module])}
+      return is_external
+        ? {is_external, exports}
+        : {exports, calls: assign_code(modules, calls, modules[module])}
     }
   )
 

src/find_definitions.js

@@ -264,6 +264,7 @@ code analysis:
 - every assignment can only be to if identifier is earlier declared by let
 - assignment can only be inside if statement (after let) (relax it?)
 - cannot import names that are not exported from modules
+- module can be imported either as external or regular
 - cannot return from modules (even inside toplevel if statements)
 */
 export const analyze = (node, is_toplevel = true) => {

src/parse_js.js

@@ -78,6 +78,9 @@ const tokenize_js = (str) => {
   ]
   
   const TOKENS = [
+    {name: 'pragma_external'    , re: '//[\\s]*external[\\s]*[\r\n]'},
+    {name: 'pragma_external'    , re: '\\/\\*[\\s]*external[\\s]*\\*\\/'},
+
     {name: 'comment'            , re: '//[^\n]*'},
     {name: 'comment'            , re: '\\/\\*[\\s\\S]*?\\*\\/'},
     {name: 'newline'            , re: '[\r\n\]+'},
@@ -187,6 +190,8 @@ const current = cxt => cxt.current < cxt.tokens.length
 
 const literal = str => by_pred(token => token.string == str, 'expected ' + str)
 
+const insignificant_types = new Set(['newline', 'pragma_external'])
+
 const by_pred = (pred, error) => {
   const result = cxt => {
     const token = current(cxt)
@@ -203,7 +208,8 @@ const by_pred = (pred, error) => {
       }
     }
 
-    if(token.type == 'newline') {
+    // Skip non-significant tokens if they were not asked for explicitly
+    if(insignificant_types.has(token.type)) {
       return result({...cxt, current: cxt.current + 1})
     }
 
@@ -225,7 +231,7 @@ export const eof = cxt => {
   if(c == null) {
     return {ok: true, cxt}
   }
-  if(c.type == 'newline') {
+  if(insignificant_types.has(c.type)) {
     return eof({...cxt, current: cxt.current + 1})
   }
   return {ok: false, error: 'unexpected token, expected eof', cxt}
@@ -1169,23 +1175,30 @@ const import_statement =
   // TODO import *, import as
   if_ok(
     seq([
-      literal('import'),
+      optional(by_type('pragma_external')),
-      list(
+      seq([
-        ['{', '}'],
+        literal('import'),
-        identifier,
+        list(
-      ),
+          ['{', '}'],
-      literal('from'),
+          identifier,
-      string_literal
+        ),
+        literal('from'),
+        string_literal
+      ])
     ]),
-    ({value: [_0, identifiers, _2, module], ...node}) => ({
+    ({value: [external, imp]}) => {
-      ...node,
+      const {value: [_import, identifiers, _from, module], ...node} = imp
-      not_evaluatable: true,
+      return {
-      type: 'import',
+        ...node,
-      // TODO refactor hanlding of string literals. Drop quotes from value and
+        not_evaluatable: true,
-      // fix codegen for string_literal
+        type: 'import',
-      module: module.value.slice(1, module.value.length - 1),
+        is_external: external != null,
-      children: identifiers.value,
+        // TODO refactor hanlding of string literals. Drop quotes from value and
-    })
+        // fix codegen for string_literal
+        module: module.value.slice(1, module.value.length - 1),
+        children: identifiers.value,
+      }
+    }
   )
 
 const export_statement =

test/test.js

@@ -2,7 +2,8 @@ import {find_leaf, ancestry, find_node} from '../src/ast_utils.js'
 import {parse, print_debug_node} from '../src/parse_js.js'
 import {eval_tree, eval_frame, eval_modules} from '../src/eval.js'
 import {COMMANDS, get_initial_state} from '../src/cmd.js'
-import {root_calltree_node, active_frame, pp_calltree, do_pp_calltree} from '../src/calltree.js'
+import {root_calltree_node, active_frame, pp_calltree, do_pp_calltree} 
+  from '../src/calltree.js'
 import {color_file} from '../src/color.js'
 import {
   test, 
@@ -725,9 +726,15 @@ export const tests = [
       'b' : `import {c} from 'c'`,
       'c' : `export const c = 1`,
     })
+
+    // Break file c. If parse result is cached then the file will not be parsed
+    // and the code would not break
+    const spoil_file = {...s, files: {...s.files, 'c': ',,,'}}
+
     // change module ''
-    const s2 = COMMANDS.input(s, 'import {c} from "c"', 0)
+    const {state: s2} = COMMANDS.input(spoil_file, 'import {c} from "c"', 0)
-    // TODO assert that module 'c' was loaded from cache
+
+    assert_equal(s2.parse_result.ok, true)
   }),
 
   test('modules', () => {
@@ -770,6 +777,136 @@ export const tests = [
     assert_equal(mods.root.exports.is_eq, true)
   }),
 
+  test('bug parser pragma external', () => {
+    const result = parse(`
+      // external
+    `)
+    assert_equal(result.ok, true)
+  }),
+
+
+  test('module external', () => {
+    const code = `
+      // external
+      import {foo_var} from 'foo.js'
+      console.log(foo_var)
+    `
+    const s1 = test_initial_state(code)
+    assert_equal(s1.loading_external_imports_state.index, 0)
+    assert_equal(s1.loading_external_imports_state.external_imports, ['foo.js'])
+
+    const state = COMMANDS.external_imports_loaded(s1, s1, {
+      'foo.js': {
+        ok: true,
+        module: {
+          'foo_var': 'foo_value'
+        },
+      }
+    })
+    assert_equal(state.logs.logs[0].args, ['foo_value'])
+    assert_equal(state.loading_external_imports_state, null)
+  }),
+
+  test('module external input', () => {
+    const initial_code = ``
+    const initial = test_initial_state(initial_code)
+    const edited = `
+      // external
+      import {foo_var} from 'foo.js'
+      console.log(foo_var)
+    `
+
+    const index = edited.indexOf('foo_var')
+
+    const {state, effects} = COMMANDS.input(
+      initial, 
+      edited, 
+      index
+    )
+    // embed_value_explorer suspended until external imports resolved
+    assert_equal(effects.length, 1)
+    assert_equal(effects[0].type, 'save_to_localstorage')
+    assert_equal(state.loading_external_imports_state.index, index)
+    assert_equal(
+      state.loading_external_imports_state.external_imports,
+      ['foo.js'],
+    )
+
+    // TODO must have effect embed_value_explorer
+    const next = COMMANDS.external_imports_loaded(state, state, {
+      'foo.js': {
+        ok: true,
+        module: {
+          'foo_var': 'foo_value'
+        },
+      }
+    })
+    assert_equal(next.loading_external_imports_state, null)
+    assert_equal(next.logs.logs[0].args, ['foo_value'])
+  }),
+
+  test('module external load error', () => {
+    const code = `
+      // external
+      import {foo_var} from 'foo.js'
+      console.log(foo_var)
+    `
+    const initial = test_initial_state(code)
+
+    const next = COMMANDS.external_imports_loaded(initial, initial, {
+      'foo.js': {
+        ok: false,
+        error: new Error('Failed to resolve module'),
+      }
+    })
+
+    assert_equal(next.parse_result.ok, false)
+    assert_equal(
+      next.parse_result.problems, 
+      [
+        {
+          index: code.indexOf('import'),
+          message: 'Failed to resolve module',
+          module: '',
+        }
+      ]
+    )
+  }),
+
+  test('module external cache', () => {
+    const code = `
+      // external
+      import {foo_var} from 'foo.js'
+      console.log(foo_var)
+    `
+    const initial = test_initial_state(code)
+
+    const next = COMMANDS.external_imports_loaded(initial, initial, {
+      'foo.js': {
+        ok: true,
+        module: {
+          'foo_var': 'foo_value'
+        },
+      }
+    })
+
+    const edited = `
+      // external
+      import {foo_var} from 'foo.js'
+      foo_var
+    `
+
+    const {state, effects} = COMMANDS.input(
+      next, 
+      edited, 
+      edited.lastIndexOf('foo_var'),
+    )
+
+    // If cache was not used then effects will be `load_external_imports`
+    const embed = effects.find(e => e.type == 'embed_value_explorer')
+    assert_equal(embed.args[0].result.value, 'foo_value')
+  }),
+
   // Static analysis
 
   test('undeclared', () => {
@@ -1451,7 +1588,10 @@ const y = x()`
       const overflow = x => overflow(x + 1);
       overflow(0)
     `)
-    assert_equal(s.current_calltree_node.error.message, 'Maximum call stack size exceeded')
+    assert_equal(
+      s.current_calltree_node.error.message, 
+      'Maximum call stack size exceeded'
+    )
     assert_equal(s.current_calltree_node.toplevel, true)
     assert_equal(s.calltree_node_is_expanded[s.current_calltree_node.id], true)
   }),