refactor

src/runtime/record_io.js

@@ -0,0 +1,301 @@
+import {set_record_call} from './runtime.js'
+
+// Current context for current execution of code
+let cxt
+
+const label = Symbol('io_patches_applied')
+
+export const set_current_context = _cxt => {
+  cxt = _cxt
+  // When we develop leporello.js inside itself, patches may be applied twice,
+  // once for host leporello, and another for leporello under development. This
+  // module would be loaded twice, once from host window, another time from
+  // app_window. Every module will have its own 'label', so we can apply
+  // patches twice
+  if(cxt.window.__io_patched_by == null) {
+    cxt.window.__io_patched_by = new Set()
+  }
+  if(!cxt.window.__io_patched_by.has(label)) {
+    apply_io_patches()
+    cxt.window.__io_patched_by.add(label)
+  }
+}
+
+const io_patch = (path, use_context = false) => {
+  let obj = cxt.window
+  for(let i = 0; i < path.length - 1; i++) {
+    obj = obj[path[i]]
+  }
+  const method = path.at(-1)
+  if(obj == null || obj[method] == null) {
+    // Method is absent in current env, skip patching
+    return
+  }
+  const name = path.join('.')
+
+  const original = obj[method]
+
+  obj[method] = make_patched_method(original, name, use_context)
+
+  obj[method].__original = original
+}
+
+const make_patched_method = (original, name, use_context) => {
+  const method = function(...args) {
+    if(cxt.io_trace_is_replay_aborted) {
+      // Try to finish fast
+      const error = new Error('io replay was aborted')
+      error.__ignore = true
+      throw error
+    }
+
+    const has_new_target = new.target != null
+
+    if(cxt.is_recording_deferred_calls) {
+      // TODO record trace on deferred calls?
+      return has_new_target 
+        ? new original(...args)
+        : original.apply(this, args)
+    }
+
+    const cxt_copy = cxt
+
+    if(cxt.io_trace_is_recording) {
+      let ok, value, error
+      try {
+        // save call, so on expand_call and find_call IO functions would not be
+        // called. 
+        // TODO: we have a problem when IO function is called from third-party
+        // lib and async context is lost
+        set_record_call(cxt)
+
+        const index = cxt.io_trace.length
+
+        if(name == 'setTimeout') {
+          args = args.slice()
+          // Patch callback
+          const cb = args[0]
+          args[0] = Object.defineProperty(function() {
+            if(cxt_copy != cxt) {
+              // If code execution was cancelled, then never call callback
+              return
+            }
+            if(cxt.io_trace_is_replay_aborted) {
+              // Non necessary
+              return
+            }
+            cxt.io_trace.push({type: 'resolution', index})
+            cb()
+          }, 'name', {value: cb.name})
+        }
+
+        value = has_new_target 
+          ? new original(...args)
+          : original.apply(this, args)
+
+        if(value instanceof cxt.window.Promise) {
+          // TODO use __original_then, not finally which calls
+          // patched 'then'?
+          value = value.finally(() => {
+            if(cxt_copy != cxt) {
+              return
+            }
+            if(cxt.io_trace_is_replay_aborted) {
+              // Non necessary
+              return
+            }
+            cxt.io_trace.push({type: 'resolution', index})
+          })
+        }
+
+        ok = true
+        return value
+      } catch(e) {
+        error = e
+        ok = false
+        throw e
+      } finally {
+        cxt.io_trace.push({
+          type: 'call',
+          name,
+          ok, 
+          value, 
+          error, 
+          args, 
+          // To discern calls with and without 'new' keyword, primary for
+          // Date that can be called with and without new
+          has_new_target,
+          use_context,
+          context: use_context ? this : undefined,
+        })
+      }
+    } else {
+      // IO trace replay
+
+      const call = cxt.io_trace[cxt.io_trace_index]
+
+      // TODO if call == null or call.type == 'resolution', then do not discard
+      // trace, instead switch to record mode and append new calls to the
+      // trace?
+      if(
+        call == null
+        || call.type != 'call'
+        || call.has_new_target != has_new_target
+        || call.use_context && (call.context != this)
+        || call.name != name
+        || (
+            (name == 'setTimeout' && (args[1] != call.args[1])) /* compares timeout*/
+            ||
+            (
+              name != 'setTimeout' 
+              && 
+              JSON.stringify(call.args) != JSON.stringify(args)
+            )
+           )
+      ){
+        cxt.io_trace_is_replay_aborted = true
+        cxt.io_trace_abort_replay()
+        // throw error to prevent further code execution. It
+        // is not necesseary, becuase execution would not have
+        // any effects anyway
+        const error = new Error('io replay aborted')
+        error.__ignore = true
+        throw error
+      } else {
+
+        const next_resolution = cxt.io_trace.find((e, i) => 
+          e.type == 'resolution' && i > cxt.io_trace_index
+        )
+
+        if(next_resolution != null && !cxt.io_trace_resolver_is_set) {
+          const original_setTimeout = cxt.window.setTimeout.__original
+          cxt.io_trace_resolver_is_set = true
+
+          original_setTimeout(() => {
+            if(cxt_copy != cxt) {
+              return
+            }
+
+            if(cxt.io_trace_is_replay_aborted) {
+              return
+            }
+
+            cxt.io_trace_resolver_is_set = false
+
+            // Sanity check
+            if(cxt.io_trace_index >= cxt.io_trace.length) {
+              throw new Error('illegal state')
+            }  
+
+            const next_event = cxt.io_trace[cxt.io_trace_index]
+            if(next_event.type == 'call') {
+              cxt.io_trace_is_replay_aborted = true
+              cxt.io_trace_abort_replay()
+            } else {
+              while(
+                cxt.io_trace_index < cxt.io_trace.length 
+                && 
+                cxt.io_trace[cxt.io_trace_index].type == 'resolution'
+              ) {
+                const resolution = cxt.io_trace[cxt.io_trace_index]
+                const resolver = cxt.io_trace_resolvers.get(resolution.index)
+
+                cxt.io_trace_index++
+
+                if(cxt.io_trace[resolution.index].name == 'setTimeout') {
+                  resolver()
+                } else {
+                  resolver(cxt.io_trace[resolution.index].value)
+                }
+              }
+            }
+
+            }, 0)
+        }
+
+        cxt.io_trace_index++
+
+        if(call.ok) {
+          // Use Symbol.toStringTag for comparison because Promise may
+          // originate from another window (if window was reopened after record
+          // trace) and instanceof would not work
+          if(call.value?.[Symbol.toStringTag] == 'Promise') {
+            // Always make promise originate from app_window
+            return new cxt.window.Promise(resolve => {
+              cxt.io_trace_resolvers.set(cxt.io_trace_index - 1, resolve)
+            })
+          } else if(name == 'setTimeout') {
+            const timeout_cb = args[0]
+            cxt.io_trace_resolvers.set(cxt.io_trace_index - 1, timeout_cb)
+            return call.value
+          } else {
+            return call.value
+          }
+        } else {
+          throw call.error
+        }
+      }
+    }
+  }
+
+  Object.defineProperty(method, 'name', {value: original.name})
+
+  return method
+}
+
+const patch_Date = () => {
+  const Date = cxt.window.Date
+  const Date_patched = make_patched_method(Date, 'Date', false)
+  cxt.window.Date = function(...args) {
+    if(args.length == 0) {
+      // return current Date, IO operation
+      if(new.target != null) {
+        return new Date_patched(...args)
+      } else {
+        return Date_patched(...args)
+      }
+    } else {
+      // pure function
+      if(new.target != null) {
+        return new Date(...args)
+      } else {
+        return Date(...args)
+      }
+    }
+  }
+  cxt.window.Date.__original = Date
+
+  cxt.window.Date.parse = Date.parse
+  cxt.window.Date.now =   Date.now
+  cxt.window.Date.UTC =   Date.UTC
+  io_patch(['Date', 'now'])
+}
+
+export const apply_io_patches = () => {
+  io_patch(['Math', 'random'])
+
+  io_patch(['setTimeout'])
+  // TODO if call setTimeout and then clearTimeout, trace it and remove call of
+  // clearTimeout, and make only setTimeout, then it would never be called when
+  // replaying from trace
+  io_patch(['clearTimeout'])
+
+  // TODO patch setInterval to only cleanup all intervals on finish
+
+  patch_Date()
+
+  io_patch(['fetch'])
+  // Check if Response is defined, for node.js
+  if(cxt.window.Response != null) {
+    const Response_methods = [
+      'arrayBuffer',
+      'blob',
+      'formData',
+      'json',
+      'text',
+    ]
+    for(let key of Response_methods) {
+      io_patch(['Response', 'prototype', key], true)
+    }
+  }
+}

src/runtime/runtime.js

@@ -0,0 +1,581 @@
+import {set_current_context} from './record_io.js'
+
+/*
+Converts generator-returning function to promise-returning function. Allows to
+have the same code both for sync and async. If we have only sync modules (no
+toplevel awaits), then code executes synchronously, and if there are async
+modules, then code executes asynchronoulsy, but we have syntactic niceties of
+'yield', 'try', 'catch'
+*/
+const gen_to_promise = gen_fn => {
+  return (...args) => {
+    const gen = gen_fn(...args)
+    const next = result => {
+      if(result.done){
+        return result.value
+      } else {
+        // If promise
+        if(result.value?.then != null) {
+          return result.value.__original_then(
+            value => next(gen.next(value)),
+            error => next(gen.throw(error)),
+          )
+        } else {
+          return next(gen.next(result.value))
+        }
+      }
+    }
+    return next(gen.next())
+  }
+}
+
+const make_promise_with_rejector = cxt => {
+  let rejector
+  const p = new cxt.window.Promise(r => rejector = r)
+  return [p, rejector]
+}
+
+const do_run = function*(module_fns, cxt, io_trace){
+  let calltree
+
+  const calltree_node_by_loc = new Map(
+    module_fns.map(({module}) => [module, new Map()])
+  )
+
+  const [replay_aborted_promise, io_trace_abort_replay] = 
+    make_promise_with_rejector(cxt)
+
+  cxt = (io_trace == null || io_trace.length == 0)
+    // TODO group all io_trace_ properties to single object?
+    ? {...cxt,
+      calltree_node_by_loc,
+      logs: [],
+      io_trace_is_recording: true,
+      io_trace: [],
+    }
+    : {...cxt,
+      calltree_node_by_loc,
+      logs: [],
+      io_trace_is_recording: false,
+      io_trace,
+      io_trace_is_replay_aborted: false,
+      io_trace_resolver_is_set: false,
+      // Map of (index in io_trace) -> resolve
+      io_trace_resolvers: new Map(),
+      io_trace_index: 0,
+      io_trace_abort_replay,
+    }
+
+  apply_promise_patch(cxt)
+  set_current_context(cxt)
+
+  for(let i = 0; i < module_fns.length; i++) {
+    const {module, fn} = module_fns[i]
+
+    cxt.is_entrypoint = i == module_fns.length - 1
+
+    cxt.children = null
+    calltree = {
+      toplevel: true, 
+      module,
+      id: ++cxt.call_counter,
+      let_vars: {},
+    }
+
+    try {
+      cxt.modules[module] = {}
+      const result = fn(
+        cxt, 
+        calltree.let_vars,
+        calltree_node_by_loc.get(module),
+        __trace, 
+        __trace_call, 
+        __do_await, 
+        __save_ct_node_for_path,
+        Multiversion,
+      )
+      if(result instanceof cxt.window.Promise) {
+        yield cxt.window.Promise.race([replay_aborted_promise, result])
+      } else {
+        yield result
+      }
+      calltree.ok = true
+    } catch(error) {
+      calltree.ok = false
+      calltree.error = error
+    }
+    calltree.children = cxt.children
+    calltree.next_id = cxt.call_counter + 1
+    if(!calltree.ok) {
+      break
+    }
+  }
+
+  cxt.is_recording_deferred_calls = true
+  const _logs = cxt.logs
+  cxt.logs = []
+  cxt.children = null
+
+  remove_promise_patch(cxt)
+
+  return {
+    modules: cxt.modules,
+    calltree,
+    logs: _logs,
+    rt_cxt: cxt,
+    calltree_node_by_loc,
+  }
+}
+
+export const run = gen_to_promise(function*(module_fns, cxt, io_trace) {
+  const result = yield* do_run(module_fns, cxt, io_trace)
+
+  if(result.rt_cxt.io_trace_is_replay_aborted) {
+    // TODO test next line
+    result.rt_cxt.is_recording_deferred_calls = false
+
+    // run again without io trace
+    return yield* do_run(module_fns, cxt, null)
+  } else {
+    return result
+  }
+})
+
+
+const apply_promise_patch = cxt => {
+  const original_then = cxt.window.Promise.prototype.then
+  cxt.window.Promise.prototype.__original_then = cxt.window.Promise.prototype.then
+
+  cxt.window.Promise.prototype.then = function then(on_resolve, on_reject) {
+
+    if(cxt.children == null) {
+      cxt.children = []
+    }
+    let children_copy = cxt.children
+
+    const make_callback = (cb, ok) => typeof(cb) != 'function'
+      ? cb
+      : value => {
+          if(this.status == null) {
+            this.status = ok ? {ok, value} : {ok, error: value}
+          }
+          const current = cxt.children
+          cxt.children = children_copy
+          try {
+            return cb(value)
+          } finally {
+            cxt.children = current
+          }
+        }
+
+    return original_then.call(
+      this,
+      make_callback(on_resolve, true),
+      make_callback(on_reject, false),
+    )
+  }
+}
+
+const remove_promise_patch = cxt => {
+  cxt.window.Promise.prototype.then = cxt.window.Promise.prototype.__original_then
+}
+
+export const set_record_call = cxt => {
+  for(let i = 0; i < cxt.stack.length; i++) {
+    cxt.stack[i] = true
+  }
+}
+
+export const do_eval_expand_calltree_node = (cxt, node) => {
+  cxt.is_recording_deferred_calls = false
+  cxt.is_expanding_calltree_node = true
+  cxt.touched_multiversions = new Set()
+
+  // Save call counter and set it to the value it had when executed 'fn' for
+  // the first time
+  const call_counter = cxt.call_counter
+  cxt.call_counter = node.fn.__location == null
+    // Function is native, set call_counter to node.id
+    ? node.id 
+    // call_counter will be incremented inside __trace and produce the same id
+    // as node.id
+    : node.id - 1 
+
+  cxt.children = null
+  try {
+    if(node.is_new) {
+      new node.fn(...node.args)
+    } else {
+      node.fn.apply(node.context, node.args)
+    }
+  } catch(e) {
+    // do nothing. Exception was caught and recorded inside '__trace'
+  }
+
+  // Restore call counter
+  cxt.call_counter = call_counter
+
+  // Recover multiversions affected by expand_calltree_node
+  for(let m of cxt.touched_multiversions) {
+    if(m.is_expanding_calltree_node) {
+      delete m.is_expanding_calltree_node
+    }
+    if(m.latest_copy != null) {
+      m.latest = m.latest_copy.value
+    }
+  }
+  delete cxt.touched_multiversions
+
+  cxt.is_expanding_calltree_node = false
+  cxt.is_recording_deferred_calls = true
+  const children = cxt.children
+  cxt.children = null
+
+  if(node.fn.__location != null) {
+    // fn is hosted, it created call, this time with children
+    const result = children[0]
+    if(result.id != node.id) {
+      throw new Error('illegal state')
+    }
+    result.children = cxt.prev_children
+    result.has_more_children = false
+    return result
+  } else {
+    // fn is native, it did not created call, only its child did
+    return {...node, 
+      children: children,
+      has_more_children: false,
+    }
+  }
+}
+
+
+
+const __do_await = async (cxt, value) => {
+  // children is an array of child calls for current function call. But it
+  // can be null to save one empty array allocation in case it has no child
+  // calls. Allocate array now, so we can have a reference to this array
+  // which will be used after await
+  if(cxt.children == null) {
+    cxt.children = []
+  }
+  const children_copy = cxt.children
+  if(value instanceof cxt.window.Promise) {
+    value.__original_then(
+      v => {
+        value.status = {ok: true, value: v}
+      },
+      e => {
+        value.status = {ok: false, error: e}
+      }
+    )
+  }
+  try {
+    return await value
+  } finally {
+    cxt.children = children_copy
+  }
+}
+
+const __trace = (cxt, fn, name, argscount, __location, get_closure, has_versioned_let_vars) => {
+  const result = (...args) => {
+    if(result.__closure == null) {
+      result.__closure = get_closure()
+    }
+
+    const children_copy = cxt.children
+    cxt.children = null
+    cxt.stack.push(false)
+
+    const call_id = ++cxt.call_counter
+
+    // populate calltree_node_by_loc only for entrypoint module
+    if(cxt.is_entrypoint && !cxt.skip_save_ct_node_for_path) {
+      let nodes_of_module = cxt.calltree_node_by_loc.get(__location.module)
+      if(nodes_of_module == null) {
+        nodes_of_module = new Map()
+        cxt.calltree_node_by_loc.set(__location.module, nodes_of_module)
+      }
+      if(nodes_of_module.get(__location.index) == null) {
+        set_record_call(cxt)
+        nodes_of_module.set(__location.index, call_id)
+      }
+    }
+
+    let let_vars
+    if(has_versioned_let_vars) {
+      let_vars = cxt.let_vars = {}
+    }
+
+    let ok, value, error
+
+    const is_toplevel_call_copy = cxt.is_toplevel_call
+    cxt.is_toplevel_call = false
+
+    try {
+      value = fn(...args)
+      ok = true
+      if(value instanceof cxt.window.Promise) {
+        set_record_call(cxt)
+      }
+      return value
+    } catch(_error) {
+      ok = false
+      error = _error
+      set_record_call(cxt)
+      if(cxt.is_recording_deferred_calls && is_toplevel_call_copy) {
+        if(error instanceof cxt.window.Error) {
+          error.__ignore = true
+        }
+      }
+      throw error
+    } finally {
+
+      cxt.prev_children = cxt.children
+
+      const call = {
+        id: call_id,
+        next_id: cxt.call_counter + 1,
+        let_vars,
+        ok,
+        value,
+        error,
+        fn: result,
+        args: argscount == null 
+          ? args
+          // Do not capture unused args
+          : args.slice(0, argscount),
+      }
+
+      const should_record_call = cxt.stack.pop()
+
+      if(should_record_call) {
+        call.children = cxt.children
+      } else {
+        call.has_more_children = cxt.children != null && cxt.children.length != 0
+      }
+      cxt.children = children_copy
+      if(cxt.children == null) {
+        cxt.children = []
+      }
+      cxt.children.push(call)
+
+      cxt.is_toplevel_call = is_toplevel_call_copy
+
+      if(cxt.is_recording_deferred_calls && cxt.is_toplevel_call) {
+        if(cxt.children.length != 1) {
+          throw new Error('illegal state')
+        }
+        const call = cxt.children[0]
+        cxt.children = null
+        const _logs = cxt.logs
+        cxt.logs = []
+        cxt.on_deferred_call(call, cxt.calltree_changed_token, _logs)
+      }
+    }
+  }
+
+  Object.defineProperty(result, 'name', {value: name})
+  result.__location = __location
+  return result
+}
+
+const __trace_call = (cxt, fn, context, args, errormessage, is_new = false) => {
+  if(fn != null && fn.__location != null && !is_new) {
+    // Call will be traced, because tracing code is already embedded inside
+    // fn
+    return fn(...args)
+  }
+
+  if(typeof(fn) != 'function') {
+    throw new TypeError(
+      errormessage 
+      + ' is not a ' 
+      + (is_new ? 'constructor' : 'function')
+    )
+  }
+
+  const children_copy = cxt.children
+  cxt.children = null
+  cxt.stack.push(false)
+
+  const call_id = ++cxt.call_counter
+
+  // TODO: other console fns
+  const is_log = fn == cxt.window.console.log || fn == cxt.window.console.error
+
+  if(is_log) {
+    set_record_call(cxt)
+  }
+
+  let ok, value, error
+
+  try {
+    if(!is_log) {
+      if(is_new) {
+        value = new fn(...args)
+      } else {
+        value = fn.apply(context, args)
+      }
+    } else {
+      value = undefined
+    }
+    ok = true
+    if(value instanceof cxt.window.Promise) {
+      set_record_call(cxt)
+    }
+    return value
+  } catch(_error) {
+    ok = false
+    error = _error
+    set_record_call(cxt)
+    throw error
+  } finally {
+
+    cxt.prev_children = cxt.children
+
+    const call = {
+      id: call_id,
+      next_id: cxt.call_counter + 1,
+      ok,
+      value,
+      error,
+      fn,
+      args,
+      context,
+      is_log,
+      is_new,
+    }
+    
+    if(is_log) {
+      cxt.logs.push(call)
+    }
+
+    const should_record_call = cxt.stack.pop()
+
+    if(should_record_call) {
+      call.children = cxt.children
+    } else {
+      call.has_more_children = cxt.children != null && cxt.children.length != 0
+    }
+
+    cxt.children = children_copy
+    if(cxt.children == null) {
+      cxt.children = []
+    }
+    cxt.children.push(call)
+  }
+}
+
+const __save_ct_node_for_path = (cxt, __calltree_node_by_loc, index, __call_id) => {
+  if(!cxt.is_entrypoint) {
+    return
+  }
+
+  if(cxt.skip_save_ct_node_for_path) {
+    return
+  }
+  if(__calltree_node_by_loc.get(index) == null) {
+    __calltree_node_by_loc.set(index, __call_id)
+    set_record_call(cxt)
+  }
+}
+
+// https://stackoverflow.com/a/29018745
+function binarySearch(arr, el, compare_fn) {
+    let m = 0;
+    let n = arr.length - 1;
+    while (m <= n) {
+        let k = (n + m) >> 1;
+        let cmp = compare_fn(el, arr[k]);
+        if (cmp > 0) {
+            m = k + 1;
+        } else if(cmp < 0) {
+            n = k - 1;
+        } else {
+            return k;
+        }
+    }
+    return ~m;
+}
+
+// 'let' variable recording the history of its values
+export class Multiversion {
+  constructor(cxt, initial) {
+    this.cxt = cxt
+    this.is_expanding_calltree_node = cxt.is_expanding_calltree_node
+    this.latest = initial
+    this.versions = [{call_id: cxt.call_counter, value: initial}]
+  }
+
+  get() {
+    const call_id = this.cxt.call_counter
+
+    if(!this.cxt.is_expanding_calltree_node) {
+      return this.latest
+    } else {
+      if(this.is_expanding_calltree_node) {
+        // var was created during current expansion, use its latest value
+        return this.latest
+      } else {
+        if(this.latest_copy != null) {
+          // value was set during expand_calltree_node, use this value
+          return this.latest
+        }
+        // TODO on first read, set latest and latest_copy?
+        return this.get_version(call_id)
+      }
+    }
+  }
+
+  get_version(call_id) {
+    const idx = binarySearch(this.versions, call_id, (id, el) => id - el.call_id)
+    if(idx == 0) {
+      // This branch is unreachable. get_version will be never called for a
+      // call_id where let variable was declared.
+      throw new Error('illegal state')
+    } else if(idx > 0) {
+      return this.versions[idx - 1].value
+    } else if(idx == -1) {
+      throw new Error('illegal state')
+    } else {
+      return this.versions[-idx - 2].value
+    }
+  }
+
+  set(value) {
+    const call_id = this.cxt.call_counter
+    if(this.cxt.is_expanding_calltree_node) {
+      if(this.is_expanding_calltree_node) {
+        this.latest = value
+        this.set_version(call_id, value)
+        this.cxt.touched_multiversions.add(this)
+      } else {
+        if(this.latest_copy == null) {
+          this.latest_copy = {value: this.latest}
+        }
+        this.cxt.touched_multiversions.add(this)
+        this.latest = value
+      }
+    } else {
+      this.latest = value
+      this.set_version(call_id, value)
+    }
+  }
+
+  last_version_number() {
+    return this.versions.at(-1).call_id
+  }
+
+  set_version(call_id, value) {
+    const last_version = this.versions.at(-1)
+    if(last_version.call_id > call_id) {
+      throw new Error('illegal state')
+    }
+    if(last_version.call_id == call_id) {
+      last_version.value = value
+      return
+    }
+    this.versions.push({call_id, value})
+  }
+}