From 0cf6d782994591a635850129bba5294fb38cea3b Mon Sep 17 00:00:00 2001 From: Dmitry Vasilev Date: Sun, 5 Feb 2023 02:08:53 +0800 Subject: [PATCH] external runtime --- src/eval.js | 481 +++------------------------------------ src/runtime.js | 430 ++++++++++++++++++++++++++++++++++ test/self_hosted_test.js | 6 - 3 files changed, 465 insertions(+), 452 deletions(-) diff --git a/src/eval.js b/src/eval.js index 5fee925..e5535f9 100644 --- a/src/eval.js +++ b/src/eval.js @@ -16,8 +16,7 @@ import { import {has_toplevel_await} from './find_definitions.js' // external -// TODO -// import {} from './runtime.js' +import {run, do_eval_expand_calltree_node, do_eval_find_call} from './runtime.js' // TODO: fix error messages. For example, "__fn is not a function" @@ -283,53 +282,13 @@ ${JSON.stringify(errormessage)}, true)` } } -// TODO remove -/* -const SyncPromise = value => ({ - then: function(cb) { - const result = cb(this.value) - if(result.value - }, - value, - is_sync_promise: true, -}) -*/ - -/* -Converts generator-returning function to promise-returning function. Allows to -have the same code both for sync and async. If we have only sync modules (no -toplevel awaits), then code executes synchronously, and if there are async -modules, then code executes asynchronoulsy, but we have syntactic niceties of -'yield', 'try', 'catch' -*/ -const gen_to_promise = gen_fn => { - return (...args) => { - const gen = gen_fn(...args) - const next = result => { - if(result.done){ - return result.value - } else { - if(result.value instanceof run_window.Promise) { - return result.value.then( - value => next(gen.next(value)), - error => next(gen.throw(error)), - ) - } else { - return next(gen.next(result.value)) - } - } - } - return next(gen.next()) - } -} - -export const eval_modules = gen_to_promise(function*( +export const eval_modules = ( parse_result, external_imports, on_deferred_call, calltree_changed_token, location -){ +) => { // TODO gensym __cxt, __trace, __trace_call // TODO bug if module imported twice, once as external and as regular @@ -379,218 +338,47 @@ export const eval_modules = gen_to_promise(function*( ) }, - calltree_changed_token + calltree_changed_token, } const Function = is_async ? globalThis.run_window.eval('(async function(){})').constructor : globalThis.run_window.Function - let calltree - - apply_promise_patch(cxt) - - for(let current_module of parse_result.sorted) { - cxt.found_call = null - cxt.children = null - calltree = { - toplevel: true, - module: current_module, - id: cxt.call_counter++ - } - - const module_fn = new Function( - '__cxt', - '__trace', - '__trace_call', - '__do_await', - codegen(parse_result.modules[current_module], {module: current_module}) - ) - - try { - cxt.modules[current_module] = {} - yield module_fn( - cxt, - __trace, - __trace_call, - __do_await, - ) - calltree.ok = true - } catch(error) { - calltree.ok = false - calltree.error = error - } - calltree.children = cxt.children - if(!calltree.ok) { - break - } - } - - cxt.is_recording_deferred_calls = true - const _logs = cxt.logs - cxt.logs = [] - cxt.children = null - - remove_promise_patch(cxt) - - cxt.searched_location = null - const call = cxt.found_call - cxt.found_call = null - - return { - modules: cxt.modules, - calltree: assign_code(parse_result.modules, calltree), - // TODO assign_code to 'call' - call, - logs: _logs, - eval_cxt: cxt, - } -}) - -const apply_promise_patch = cxt => { - - cxt.promise_then = Promise.prototype.then - - Promise.prototype.then = function then(on_resolve, on_reject) { - - if(cxt.children == null) { - cxt.children = [] + const module_fns = parse_result.sorted.map(module => ( + { + module, + fn: new Function( + '__cxt', + '__trace', + '__trace_call', + '__do_await', + codegen(parse_result.modules[module], {module}) + ) } - let children_copy = cxt.children + )) - const make_callback = (cb, ok) => typeof(cb) != 'function' - ? cb - : value => { - if(this.status == null) { - this.status = ok ? {ok, value} : {ok, error: value} - } - const current = cxt.children - cxt.children = children_copy - try { - return cb(value) - } finally { - cxt.children = current - } - } + const result = run(module_fns, cxt) - return cxt.promise_then.call( - this, - make_callback(on_resolve, true), - make_callback(on_reject, false), - ) - } -} + const make_result = result => ({ + modules: result.modules, + // TODO assign_code to 'call' and refactor call site + call: result.call, + logs: result.logs, + eval_cxt: result.eval_cxt, + calltree: assign_code(parse_result.modules, result.calltree), + }) -const remove_promise_patch = cxt => { - Promise.prototype.then = cxt.promise_then -} - -const set_record_call = cxt => { - for(let i = 0; i < cxt.stack.length; i++) { - cxt.stack[i] = true - } -} - -const do_expand_calltree_node = (cxt, node) => { - if(node.fn.__location != null) { - // fn is hosted, it created call, this time with children - const result = cxt.children[0] - result.id = node.id - result.children = cxt.prev_children - result.has_more_children = false - return result + if(result.then != null) { + return result.then(make_result) } else { - // fn is native, it did not created call, only its child did - return {...node, - children: cxt.children, - has_more_children: false, - } + return make_result(result) } + } -export const eval_expand_calltree_node = (cxt, parse_result, node) => { - cxt.is_recording_deferred_calls = false - cxt.children = null - try { - if(node.is_new) { - new node.fn(...node.args) - } else { - node.fn.apply(node.context, node.args) - } - } catch(e) { - // do nothing. Exception was caught and recorded inside '__trace' - } - cxt.is_recording_deferred_calls = true - return assign_code(parse_result.modules, do_expand_calltree_node(cxt, node)) -} - - -/* - Try to find call of function with given 'location' - - Function is synchronous, because we recorded calltree nodes for all async - function calls. Here we walk over calltree, find leaves that have - 'has_more_children' set to true, and rerunning fns in these leaves with - 'searched_location' being set, until we find find call or no children - left. - - We dont rerun entire execution because we want find_call to be - synchronous for simplicity -*/ export const eval_find_call = (cxt, parse_result, calltree, location) => { - // TODO remove - if(cxt.children != null) { - throw new Error('illegal state') - } - - const do_find = node => { - if(node.children != null) { - for(let c of node.children) { - const result = do_find(c) - if(result != null) { - return result - } - } - // call was not find in children, return null - return null - } - - - if(node.has_more_children) { - try { - if(node.is_new) { - new node.fn(...node.args) - } else { - node.fn.apply(node.context, node.args) - } - } catch(e) { - // do nothing. Exception was caught and recorded inside '__trace' - } - - if(cxt.found_call != null) { - return { - node: do_expand_calltree_node(cxt, node), - call: cxt.found_call, - } - } else { - cxt.children = null - } - } - - // node has no children, return null - return null - } - - cxt.is_recording_deferred_calls = false - cxt.searched_location = location - - const result = do_find(calltree) - - cxt.children = null - cxt.searched_location = null - cxt.found_call = null - cxt.is_recording_deferred_calls = true - + const result = do_eval_find_call(cxt, calltree, location) if(result == null) { return null } @@ -603,211 +391,11 @@ export const eval_find_call = (cxt, parse_result, calltree, location) => { } } -const __do_await = async (cxt, value) => { - // children is an array of child calls for current function call. But it - // can be null to save one empty array allocation in case it has no child - // calls. Allocate array now, so we can have a reference to this array - // which will be used after await - if(cxt.children == null) { - cxt.children = [] - } - const children_copy = cxt.children - if(value instanceof Promise) { - cxt.promise_then.call(value, - v => { - value.status = {ok: true, value: v} - }, - e => { - value.status = {ok: false, error: e} - } - ) - } - try { - return await value - } finally { - cxt.children = children_copy - } -} - -const __trace = (cxt, fn, name, argscount, __location, get_closure) => { - const result = (...args) => { - if(result.__closure == null) { - result.__closure = get_closure() - } - - const children_copy = cxt.children - cxt.children = null - cxt.stack.push(false) - - const is_found_call = - (cxt.searched_location != null && cxt.found_call == null) - && - ( - __location.index == cxt.searched_location.index - && - __location.module == cxt.searched_location.module - ) - - if(is_found_call) { - // Assign temporary value to prevent nested calls from populating - // found_call - cxt.found_call = {} - } - - let ok, value, error - - const is_toplevel_call_copy = cxt.is_toplevel_call - cxt.is_toplevel_call = false - - try { - value = fn(...args) - ok = true - if(value instanceof Promise) { - set_record_call(cxt) - } - return value - } catch(_error) { - ok = false - error = _error - set_record_call(cxt) - throw error - } finally { - - cxt.prev_children = cxt.children - - const call = { - id: cxt.call_counter++, - ok, - value, - error, - fn: result, - args: argscount == null - ? args - // Do not capture unused args - : args.slice(0, argscount), - } - - if(is_found_call) { - cxt.found_call = call - set_record_call(cxt) - } - - const should_record_call = cxt.stack.pop() - - if(should_record_call) { - call.children = cxt.children - } else { - call.has_more_children = cxt.children != null && cxt.children.length != 0 - } - cxt.children = children_copy - if(cxt.children == null) { - cxt.children = [] - } - cxt.children.push(call) - - cxt.is_toplevel_call = is_toplevel_call_copy - - if(cxt.is_recording_deferred_calls && cxt.is_toplevel_call) { - if(cxt.children.length != 1) { - throw new Error('illegal state') - } - const call = cxt.children[0] - cxt.children = null - const _logs = cxt.logs - cxt.logs = [] - cxt.on_deferred_call(call, cxt.calltree_changed_token, _logs) - } - } - } - - Object.defineProperty(result, 'name', {value: name}) - result.__location = __location - return result -} - -const __trace_call = (cxt, fn, context, args, errormessage, is_new = false) => { - if(fn != null && fn.__location != null && !is_new) { - // Call will be traced, because tracing code is already embedded inside - // fn - return fn(...args) - } - - if(typeof(fn) != 'function') { - throw new TypeError( - errormessage - + ' is not a ' - + (is_new ? 'constructor' : 'function') - ) - } - - const children_copy = cxt.children - cxt.children = null - cxt.stack.push(false) - - // TODO: other console fns - const is_log = fn == console.log || fn == console.error - - if(is_log) { - set_record_call(cxt) - } - - let ok, value, error - - try { - if(!is_log) { - if(is_new) { - value = new fn(...args) - } else { - value = fn.apply(context, args) - } - } else { - value = undefined - } - ok = true - if(value instanceof Promise) { - set_record_call(cxt) - } - return value - } catch(_error) { - ok = false - error = _error - set_record_call(cxt) - throw error - } finally { - - cxt.prev_children = cxt.children - - const call = { - id: cxt.call_counter++, - ok, - value, - error, - fn, - args, - context, - is_log, - is_new, - } - - if(is_log) { - // TODO do not collect logs on find_call? - cxt.logs.push(call) - } - - const should_record_call = cxt.stack.pop() - - if(should_record_call) { - call.children = cxt.children - } else { - call.has_more_children = cxt.children != null && cxt.children.length != 0 - } - - cxt.children = children_copy - if(cxt.children == null) { - cxt.children = [] - } - cxt.children.push(call) - } +export const eval_expand_calltree_node = (cxt, parse_result, node) => { + return assign_code( + parse_result.modules, + do_eval_expand_calltree_node(cxt, node) + ) } // TODO: assign_code: benchmark and use imperative version for perf? @@ -828,6 +416,7 @@ const assign_code = (modules, call) => { } } + export const eval_tree = node => { return eval_modules( { diff --git a/src/runtime.js b/src/runtime.js index e69de29..c08a794 100644 --- a/src/runtime.js +++ b/src/runtime.js @@ -0,0 +1,430 @@ +/* +Converts generator-returning function to promise-returning function. Allows to +have the same code both for sync and async. If we have only sync modules (no +toplevel awaits), then code executes synchronously, and if there are async +modules, then code executes asynchronoulsy, but we have syntactic niceties of +'yield', 'try', 'catch' +*/ +const gen_to_promise = gen_fn => { + return (...args) => { + const gen = gen_fn(...args) + const next = result => { + if(result.done){ + return result.value + } else { + if(result.value instanceof run_window.Promise) { + return result.value.then( + value => next(gen.next(value)), + error => next(gen.throw(error)), + ) + } else { + return next(gen.next(result.value)) + } + } + } + return next(gen.next()) + } +} + +export const run = gen_to_promise(function*(module_fns, cxt){ + let calltree + + apply_promise_patch(cxt) + + for(let {module, fn} of module_fns) { + cxt.found_call = null + cxt.children = null + calltree = { + toplevel: true, + module, + id: cxt.call_counter++ + } + + try { + cxt.modules[module] = {} + yield fn(cxt, __trace, __trace_call, __do_await) + calltree.ok = true + } catch(error) { + calltree.ok = false + calltree.error = error + } + calltree.children = cxt.children + if(!calltree.ok) { + break + } + } + + cxt.is_recording_deferred_calls = true + const _logs = cxt.logs + cxt.logs = [] + cxt.children = null + + remove_promise_patch(cxt) + + cxt.searched_location = null + const call = cxt.found_call + cxt.found_call = null + + return { + modules: cxt.modules, + calltree, + call, + logs: _logs, + eval_cxt: cxt, + } +}) + +const apply_promise_patch = cxt => { + + cxt.promise_then = Promise.prototype.then + + Promise.prototype.then = function then(on_resolve, on_reject) { + + if(cxt.children == null) { + cxt.children = [] + } + let children_copy = cxt.children + + const make_callback = (cb, ok) => typeof(cb) != 'function' + ? cb + : value => { + if(this.status == null) { + this.status = ok ? {ok, value} : {ok, error: value} + } + const current = cxt.children + cxt.children = children_copy + try { + return cb(value) + } finally { + cxt.children = current + } + } + + return cxt.promise_then.call( + this, + make_callback(on_resolve, true), + make_callback(on_reject, false), + ) + } +} + +const remove_promise_patch = cxt => { + Promise.prototype.then = cxt.promise_then +} + +const set_record_call = cxt => { + for(let i = 0; i < cxt.stack.length; i++) { + cxt.stack[i] = true + } +} + +const do_expand_calltree_node = (cxt, node) => { + if(node.fn.__location != null) { + // fn is hosted, it created call, this time with children + const result = cxt.children[0] + result.id = node.id + result.children = cxt.prev_children + result.has_more_children = false + return result + } else { + // fn is native, it did not created call, only its child did + return {...node, + children: cxt.children, + has_more_children: false, + } + } +} + +export const do_eval_expand_calltree_node = (cxt, node) => { + cxt.is_recording_deferred_calls = false + cxt.children = null + try { + if(node.is_new) { + new node.fn(...node.args) + } else { + node.fn.apply(node.context, node.args) + } + } catch(e) { + // do nothing. Exception was caught and recorded inside '__trace' + } + cxt.is_recording_deferred_calls = true + return do_expand_calltree_node(cxt, node) +} + + +/* + Try to find call of function with given 'location' + + Function is synchronous, because we recorded calltree nodes for all async + function calls. Here we walk over calltree, find leaves that have + 'has_more_children' set to true, and rerunning fns in these leaves with + 'searched_location' being set, until we find find call or no children + left. + + We dont rerun entire execution because we want find_call to be + synchronous for simplicity +*/ +export const do_eval_find_call = (cxt, calltree, location) => { + // TODO remove + if(cxt.children != null) { + throw new Error('illegal state') + } + + const do_find = node => { + if(node.children != null) { + for(let c of node.children) { + const result = do_find(c) + if(result != null) { + return result + } + } + // call was not find in children, return null + return null + } + + + if(node.has_more_children) { + try { + if(node.is_new) { + new node.fn(...node.args) + } else { + node.fn.apply(node.context, node.args) + } + } catch(e) { + // do nothing. Exception was caught and recorded inside '__trace' + } + + if(cxt.found_call != null) { + return { + node: do_expand_calltree_node(cxt, node), + call: cxt.found_call, + } + } else { + cxt.children = null + } + } + + // node has no children, return null + return null + } + + cxt.is_recording_deferred_calls = false + cxt.searched_location = location + + const result = do_find(calltree) + + cxt.children = null + cxt.searched_location = null + cxt.found_call = null + cxt.is_recording_deferred_calls = true + + return result +} + +const __do_await = async (cxt, value) => { + // children is an array of child calls for current function call. But it + // can be null to save one empty array allocation in case it has no child + // calls. Allocate array now, so we can have a reference to this array + // which will be used after await + if(cxt.children == null) { + cxt.children = [] + } + const children_copy = cxt.children + if(value instanceof Promise) { + cxt.promise_then.call(value, + v => { + value.status = {ok: true, value: v} + }, + e => { + value.status = {ok: false, error: e} + } + ) + } + try { + return await value + } finally { + cxt.children = children_copy + } +} + +const __trace = (cxt, fn, name, argscount, __location, get_closure) => { + const result = (...args) => { + if(result.__closure == null) { + result.__closure = get_closure() + } + + const children_copy = cxt.children + cxt.children = null + cxt.stack.push(false) + + const is_found_call = + (cxt.searched_location != null && cxt.found_call == null) + && + ( + __location.index == cxt.searched_location.index + && + __location.module == cxt.searched_location.module + ) + + if(is_found_call) { + // Assign temporary value to prevent nested calls from populating + // found_call + cxt.found_call = {} + } + + let ok, value, error + + const is_toplevel_call_copy = cxt.is_toplevel_call + cxt.is_toplevel_call = false + + try { + value = fn(...args) + ok = true + if(value instanceof Promise) { + set_record_call(cxt) + } + return value + } catch(_error) { + ok = false + error = _error + set_record_call(cxt) + throw error + } finally { + + cxt.prev_children = cxt.children + + const call = { + id: cxt.call_counter++, + ok, + value, + error, + fn: result, + args: argscount == null + ? args + // Do not capture unused args + : args.slice(0, argscount), + } + + if(is_found_call) { + cxt.found_call = call + set_record_call(cxt) + } + + const should_record_call = cxt.stack.pop() + + if(should_record_call) { + call.children = cxt.children + } else { + call.has_more_children = cxt.children != null && cxt.children.length != 0 + } + cxt.children = children_copy + if(cxt.children == null) { + cxt.children = [] + } + cxt.children.push(call) + + cxt.is_toplevel_call = is_toplevel_call_copy + + if(cxt.is_recording_deferred_calls && cxt.is_toplevel_call) { + if(cxt.children.length != 1) { + throw new Error('illegal state') + } + const call = cxt.children[0] + cxt.children = null + const _logs = cxt.logs + cxt.logs = [] + cxt.on_deferred_call(call, cxt.calltree_changed_token, _logs) + } + } + } + + Object.defineProperty(result, 'name', {value: name}) + result.__location = __location + return result +} + +const __trace_call = (cxt, fn, context, args, errormessage, is_new = false) => { + if(fn != null && fn.__location != null && !is_new) { + // Call will be traced, because tracing code is already embedded inside + // fn + return fn(...args) + } + + if(typeof(fn) != 'function') { + throw new TypeError( + errormessage + + ' is not a ' + + (is_new ? 'constructor' : 'function') + ) + } + + const children_copy = cxt.children + cxt.children = null + cxt.stack.push(false) + + // TODO: other console fns + const is_log = fn == console.log || fn == console.error + + if(is_log) { + set_record_call(cxt) + } + + let ok, value, error + + try { + if(!is_log) { + if(is_new) { + value = new fn(...args) + } else { + value = fn.apply(context, args) + } + } else { + value = undefined + } + ok = true + if(value instanceof Promise) { + set_record_call(cxt) + } + return value + } catch(_error) { + ok = false + error = _error + set_record_call(cxt) + throw error + } finally { + + cxt.prev_children = cxt.children + + const call = { + id: cxt.call_counter++, + ok, + value, + error, + fn, + args, + context, + is_log, + is_new, + } + + if(is_log) { + // TODO do not collect logs on find_call? + cxt.logs.push(call) + } + + const should_record_call = cxt.stack.pop() + + if(should_record_call) { + call.children = cxt.children + } else { + call.has_more_children = cxt.children != null && cxt.children.length != 0 + } + + cxt.children = children_copy + if(cxt.children == null) { + cxt.children = [] + } + cxt.children.push(call) + } +} + diff --git a/test/self_hosted_test.js b/test/self_hosted_test.js index 2b38b69..c46e546 100644 --- a/test/self_hosted_test.js +++ b/test/self_hosted_test.js @@ -45,7 +45,6 @@ const adjust_path = path => { ) } -/* const load_external_modules = async state => { const urls = state.loading_external_imports_state.external_imports const results = await Promise.all( @@ -63,7 +62,6 @@ const load_external_modules = async state => { )) ) } -*/ const dir = load_dir('.') @@ -74,13 +72,9 @@ const i = test_initial_state( {project_dir: dir, entrypoint: 'test/run.js'} ) -/* assert_equal(i.loading_external_imports_state != null, true) const external_imports = await load_external_modules(i) const loaded = COMMANDS.external_imports_loaded(i, i, external_imports) -*/ - -const loaded = i assert_equal(loaded.eval_modules_state != null, true) const s = loaded.eval_modules_state