diff --git a/README.md b/README.md index bc2df85..f845f99 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,23 @@ Operators that are not supported by design (not pure functional): - increment, decrement - `delete` +## Importing third-party libs + +Sometimes you want to import third party library that uses imperative language constructs. You may want to use it to perform side-effects or maybe it mutates data inside but still provides functional interface (does not mutate function arguments). Good example of such library is [bignumber.js](https://github.com/MikeMcl/bignumber.js/) - it makes a lot of mutating assignments inside, but `BigNumber` instances are immutable. + +To use `bignumber.js` you add an `external pragma` before the import: + +``` +/* external */ +import BigNumber from './path/to/bignumber.mjs'; +``` + +`external pragma` is just a comment that contains only the literal string `external` (both styles for comments and extra whitespaces are allowed). Now the module is imported as a black box - you cannot debug `BigNumber` methods. + +![External import](docs/images/external_import.png) + +Currently every external is loaded once and cached until Leporello is restarted (TODO what happens if we load modules in iframe and then recreate iframe) + ## Hotkeys See built-in Help diff --git a/docs/images/external_import.png b/docs/images/external_import.png new file mode 100644 index 0000000..5c74910 Binary files /dev/null and b/docs/images/external_import.png differ diff --git a/src/ast_utils.js b/src/ast_utils.js index bf417ec..c15ca97 100644 --- a/src/ast_utils.js +++ b/src/ast_utils.js @@ -43,6 +43,7 @@ export const map_destructuring_identifiers = (node, mapper) => { export const collect_imports = module => { const imports = module.stmts .filter(n => n.type == 'import') + .filter(n => !n.is_external) .map(n => n.imports.map(i => ({name: i.value, module: n.full_import_path}) diff --git a/src/cmd.js b/src/cmd.js index 073dc3e..8ab9cdf 100644 --- a/src/cmd.js +++ b/src/cmd.js @@ -1,4 +1,5 @@ -import {map_object, pick_keys, collect_nodes_with_parents} from './utils.js' +import {map_object, pick_keys, collect_nodes_with_parents, uniq} + from './utils.js' import { is_eq, is_child, ancestry, ancestry_inc, map_tree, find_leaf, find_fn_by_location, find_node, find_error_origin_node @@ -82,12 +83,97 @@ const run_code = (s, index, dirty_files) => { calltree_node_by_loc: null, // TODO keep selection_state? selection_state: null, + loading_external_imports_state: null, } if(!state.parse_result.ok) { return state } + const external_import_nodes = + Object + .entries(state.parse_result.modules) + .map(([module_name, node]) => + node + .children + .filter(c => c.type == 'import' && c.is_external) + .map(node => ({node, module_name})) + ) + .flat() + + const external_imports = uniq( + external_import_nodes.map(i => i.node.full_import_path) + ) + + if( + external_imports.length != 0 + && + ( + state.external_imports_cache == null + || + external_imports.some(i => state.external_imports_cache[i] == null) + ) + ) { + return {...state, + loading_external_imports_state: { + index, + external_imports, + external_import_nodes, + } + } + } else { + return external_imports_loaded( + state, + state, + state.external_imports_cache, + index + ) + } + +} + +const do_external_imports_loaded = ( + state, + prev_state, + external_imports, + index, +) => { + if( + state.loading_external_imports_state + != + prev_state.loading_external_imports_state + ) { + // code was modified after loading started, discard + return state + } + + if(external_imports != null) { + const errors = new Set( + Object + .entries(external_imports) + .filter(([url, result]) => !result.ok) + .map(([url, result]) => url) + ) + if(errors.size != 0) { + const problems = state + .loading_external_imports_state + .external_import_nodes + .filter(({node}) => errors.has(node.full_import_path)) + .map(({node, module_name}) => ({ + index: node.index, + message: external_imports[node.full_import_path].error.message, + module: module_name, + })) + return {...state, + parse_result: { + ok: false, + cache: state.parse_result.cache, + problems, + } + } + } + } + const node = find_call_node(state, index) if( @@ -96,7 +182,11 @@ const run_code = (s, index, dirty_files) => { || node.type == 'do' /* toplevel AST node */ ) { - const result = eval_modules(state.parse_result.modules, state.parse_result.sorted) + const result = eval_modules( + state.parse_result.modules, + state.parse_result.sorted, + external_imports + ) const next = apply_eval_result(state, result) if(node == state.parse_result.modules[root_calltree_module(next)]) { @@ -117,6 +207,7 @@ const run_code = (s, index, dirty_files) => { const result = eval_modules( state.parse_result.modules, state.parse_result.sorted, + external_imports, {index: node.index, module: state.current_module}, ) @@ -145,19 +236,44 @@ const run_code = (s, index, dirty_files) => { ) } +const external_imports_loaded = ( + state, + prev_state, + external_imports, + maybe_index, +) => { + // index saved in loading_external_imports_state maybe stale, if cursor was + // moved after + // TODO refactor, make index controlled property saved in state? + const index = maybe_index ?? state.loading_external_imports_state.index + + // TODO after edit we should fire embed_value_explorer, but we dont do it + // here because we dont have cursor position (index) here (see comment + // above). Currently it is fixed by having `external_imports_cache`, so code + // goes async path only on first external imports load + return { + ...do_external_imports_loaded(state, prev_state, external_imports, index), + external_imports_cache: external_imports, + loading_external_imports_state: null + } +} + const input = (state, code, index) => { const files = {...state.files, [state.current_module]: code} const next = run_code({...state, files}, index, [state.current_module]) - const effects1 = next.current_module == '' + const effect_save = next.current_module == '' ? {type: 'save_to_localstorage', args: ['code', code]} : {type: 'write', args: [ next.current_module, next.files[next.current_module], ]} + if(next.loading_external_imports_state != null) { + return {state: next, effects: [effect_save]} + } const {state: next2, effects: effects2} = do_move_cursor(next, index) return { state: next2, - effects: [effects1, effects2], + effects: [effect_save, effects2], } } @@ -675,5 +791,6 @@ export const COMMANDS = { goto_problem, move_cursor, eval_selection, + external_imports_loaded, calltree: calltree_commands, } diff --git a/src/effects.js b/src/effects.js index 13996d9..58b10d0 100644 --- a/src/effects.js +++ b/src/effects.js @@ -2,6 +2,30 @@ import {write_file} from './filesystem.js' import {color_file} from './color.js' import {root_calltree_node, calltree_node_loc} from './calltree.js' import {FLAGS} from './feature_flags.js' +import {exec} from './index.js' + +const load_external_imports = async state => { + if(state.loading_external_imports_state == null) { + return + } + const urls = state.loading_external_imports_state.external_imports + const results = await Promise.allSettled( + urls.map(u => import(u)) + ) + const modules = Object.fromEntries( + results.map((r, i) => ( + [ + urls[i], + { + ok: r.status == 'fulfilled', + error: r.reason, + module: r.value, + } + ] + )) + ) + exec('external_imports_loaded', state /* becomes prev_state */, modules) +} const ensure_session = (ui, state, file = state.current_module) => { ui.editor.ensure_session(file, state.files[file]) @@ -84,6 +108,7 @@ export const render_initial_state = (ui, state) => { ui.render_debugger(state) render_coloring(ui, state) } + load_external_imports(state) } export const render_common_side_effects = (prev, next, command, ui) => { @@ -111,11 +136,15 @@ export const render_common_side_effects = (prev, next, command, ui) => { ui.editor.switch_session(next.current_module) } + if(prev.loading_external_imports_state != next.loading_external_imports_state) { + load_external_imports(next) + } + if(prev.parse_result != next.parse_result) { render_parse_result(ui, next) } - if(!next.parse_result.ok) { + if(!next.parse_result.ok || next.loading_external_imports_state != null) { ui.calltree.clear_calltree() ui.editor.for_each_session((file, session) => clear_coloring(ui, file)) @@ -124,6 +153,7 @@ export const render_common_side_effects = (prev, next, command, ui) => { } else { if( + // TODO refactor this condition prev.current_calltree_node == null || prev.calltree_changed_token != next.calltree_changed_token @@ -232,5 +262,6 @@ export const EFFECTS = { ui.eval.clear_value_or_error() } }, + } diff --git a/src/eval.js b/src/eval.js index 78e9a39..29c37b0 100644 --- a/src/eval.js +++ b/src/eval.js @@ -235,9 +235,11 @@ const codegen = (node, cxt, parent) => { } } -export const eval_modules = (modules, sorted, location) => { +export const eval_modules = (modules, sorted, external_imports, location) => { // TODO gensym __modules, __exports + // TODO bug if module imported twice, once as external and as regular + const codestring = ` let children, prev_children @@ -435,7 +437,7 @@ export const eval_modules = (modules, sorted, location) => { } const run = entrypoint => { - const __modules = {} + const __modules = {...external_imports} let current_call ` @@ -483,7 +485,13 @@ export const eval_modules = (modules, sorted, location) => { } ` - const actions = (new Function(codestring))() + const actions = new Function('external_imports', codestring)( + external_imports == null + ? null + : map_object(external_imports, (name, {module}) => + ({exports: module, is_external: true}) + ) + ) const calltree_actions = { expand_calltree_node: (node) => { @@ -524,8 +532,10 @@ export const eval_modules = (modules, sorted, location) => { const assign_code_calltree = (modules, calltree) => map_object( calltree, - (module, {calls, exports}) => { - return {exports, calls: assign_code(modules, calls, modules[module])} + (module, {calls, exports, is_external}) => { + return is_external + ? {is_external, exports} + : {exports, calls: assign_code(modules, calls, modules[module])} } ) diff --git a/src/find_definitions.js b/src/find_definitions.js index 0e75a8e..06f7135 100644 --- a/src/find_definitions.js +++ b/src/find_definitions.js @@ -264,6 +264,7 @@ code analysis: - every assignment can only be to if identifier is earlier declared by let - assignment can only be inside if statement (after let) (relax it?) - cannot import names that are not exported from modules +- module can be imported either as external or regular - cannot return from modules (even inside toplevel if statements) */ export const analyze = (node, is_toplevel = true) => { diff --git a/src/parse_js.js b/src/parse_js.js index 3d8987d..48393db 100644 --- a/src/parse_js.js +++ b/src/parse_js.js @@ -78,6 +78,9 @@ const tokenize_js = (str) => { ] const TOKENS = [ + {name: 'pragma_external' , re: '//[\\s]*external[\\s]*[\r\n]'}, + {name: 'pragma_external' , re: '\\/\\*[\\s]*external[\\s]*\\*\\/'}, + {name: 'comment' , re: '//[^\n]*'}, {name: 'comment' , re: '\\/\\*[\\s\\S]*?\\*\\/'}, {name: 'newline' , re: '[\r\n\]+'}, @@ -187,6 +190,8 @@ const current = cxt => cxt.current < cxt.tokens.length const literal = str => by_pred(token => token.string == str, 'expected ' + str) +const insignificant_types = new Set(['newline', 'pragma_external']) + const by_pred = (pred, error) => { const result = cxt => { const token = current(cxt) @@ -203,7 +208,8 @@ const by_pred = (pred, error) => { } } - if(token.type == 'newline') { + // Skip non-significant tokens if they were not asked for explicitly + if(insignificant_types.has(token.type)) { return result({...cxt, current: cxt.current + 1}) } @@ -225,7 +231,7 @@ export const eof = cxt => { if(c == null) { return {ok: true, cxt} } - if(c.type == 'newline') { + if(insignificant_types.has(c.type)) { return eof({...cxt, current: cxt.current + 1}) } return {ok: false, error: 'unexpected token, expected eof', cxt} @@ -1169,23 +1175,30 @@ const import_statement = // TODO import *, import as if_ok( seq([ - literal('import'), - list( - ['{', '}'], - identifier, - ), - literal('from'), - string_literal + optional(by_type('pragma_external')), + seq([ + literal('import'), + list( + ['{', '}'], + identifier, + ), + literal('from'), + string_literal + ]) ]), - ({value: [_0, identifiers, _2, module], ...node}) => ({ - ...node, - not_evaluatable: true, - type: 'import', - // TODO refactor hanlding of string literals. Drop quotes from value and - // fix codegen for string_literal - module: module.value.slice(1, module.value.length - 1), - children: identifiers.value, - }) + ({value: [external, imp]}) => { + const {value: [_import, identifiers, _from, module], ...node} = imp + return { + ...node, + not_evaluatable: true, + type: 'import', + is_external: external != null, + // TODO refactor hanlding of string literals. Drop quotes from value and + // fix codegen for string_literal + module: module.value.slice(1, module.value.length - 1), + children: identifiers.value, + } + } ) const export_statement = diff --git a/test/test.js b/test/test.js index db44a18..c8bea5a 100644 --- a/test/test.js +++ b/test/test.js @@ -2,7 +2,8 @@ import {find_leaf, ancestry, find_node} from '../src/ast_utils.js' import {parse, print_debug_node} from '../src/parse_js.js' import {eval_tree, eval_frame, eval_modules} from '../src/eval.js' import {COMMANDS, get_initial_state} from '../src/cmd.js' -import {root_calltree_node, active_frame, pp_calltree, do_pp_calltree} from '../src/calltree.js' +import {root_calltree_node, active_frame, pp_calltree, do_pp_calltree} + from '../src/calltree.js' import {color_file} from '../src/color.js' import { test, @@ -725,9 +726,15 @@ export const tests = [ 'b' : `import {c} from 'c'`, 'c' : `export const c = 1`, }) + + // Break file c. If parse result is cached then the file will not be parsed + // and the code would not break + const spoil_file = {...s, files: {...s.files, 'c': ',,,'}} + // change module '' - const s2 = COMMANDS.input(s, 'import {c} from "c"', 0) - // TODO assert that module 'c' was loaded from cache + const {state: s2} = COMMANDS.input(spoil_file, 'import {c} from "c"', 0) + + assert_equal(s2.parse_result.ok, true) }), test('modules', () => { @@ -770,6 +777,136 @@ export const tests = [ assert_equal(mods.root.exports.is_eq, true) }), + test('bug parser pragma external', () => { + const result = parse(` + // external + `) + assert_equal(result.ok, true) + }), + + + test('module external', () => { + const code = ` + // external + import {foo_var} from 'foo.js' + console.log(foo_var) + ` + const s1 = test_initial_state(code) + assert_equal(s1.loading_external_imports_state.index, 0) + assert_equal(s1.loading_external_imports_state.external_imports, ['foo.js']) + + const state = COMMANDS.external_imports_loaded(s1, s1, { + 'foo.js': { + ok: true, + module: { + 'foo_var': 'foo_value' + }, + } + }) + assert_equal(state.logs.logs[0].args, ['foo_value']) + assert_equal(state.loading_external_imports_state, null) + }), + + test('module external input', () => { + const initial_code = `` + const initial = test_initial_state(initial_code) + const edited = ` + // external + import {foo_var} from 'foo.js' + console.log(foo_var) + ` + + const index = edited.indexOf('foo_var') + + const {state, effects} = COMMANDS.input( + initial, + edited, + index + ) + // embed_value_explorer suspended until external imports resolved + assert_equal(effects.length, 1) + assert_equal(effects[0].type, 'save_to_localstorage') + assert_equal(state.loading_external_imports_state.index, index) + assert_equal( + state.loading_external_imports_state.external_imports, + ['foo.js'], + ) + + // TODO must have effect embed_value_explorer + const next = COMMANDS.external_imports_loaded(state, state, { + 'foo.js': { + ok: true, + module: { + 'foo_var': 'foo_value' + }, + } + }) + assert_equal(next.loading_external_imports_state, null) + assert_equal(next.logs.logs[0].args, ['foo_value']) + }), + + test('module external load error', () => { + const code = ` + // external + import {foo_var} from 'foo.js' + console.log(foo_var) + ` + const initial = test_initial_state(code) + + const next = COMMANDS.external_imports_loaded(initial, initial, { + 'foo.js': { + ok: false, + error: new Error('Failed to resolve module'), + } + }) + + assert_equal(next.parse_result.ok, false) + assert_equal( + next.parse_result.problems, + [ + { + index: code.indexOf('import'), + message: 'Failed to resolve module', + module: '', + } + ] + ) + }), + + test('module external cache', () => { + const code = ` + // external + import {foo_var} from 'foo.js' + console.log(foo_var) + ` + const initial = test_initial_state(code) + + const next = COMMANDS.external_imports_loaded(initial, initial, { + 'foo.js': { + ok: true, + module: { + 'foo_var': 'foo_value' + }, + } + }) + + const edited = ` + // external + import {foo_var} from 'foo.js' + foo_var + ` + + const {state, effects} = COMMANDS.input( + next, + edited, + edited.lastIndexOf('foo_var'), + ) + + // If cache was not used then effects will be `load_external_imports` + const embed = effects.find(e => e.type == 'embed_value_explorer') + assert_equal(embed.args[0].result.value, 'foo_value') + }), + // Static analysis test('undeclared', () => { @@ -1451,7 +1588,10 @@ const y = x()` const overflow = x => overflow(x + 1); overflow(0) `) - assert_equal(s.current_calltree_node.error.message, 'Maximum call stack size exceeded') + assert_equal( + s.current_calltree_node.error.message, + 'Maximum call stack size exceeded' + ) assert_equal(s.current_calltree_node.toplevel, true) assert_equal(s.calltree_node_is_expanded[s.current_calltree_node.id], true) }),