manage generated sources

apps/remix-ide-e2e/src/tests/debugger.test.ts

@@ -147,7 +147,27 @@ module.exports = {
     .waitForElementPresent('*[data-id="treeViewLoadMore"]')
     .click('*[data-id="treeViewLoadMore"]')
     .assert.containsText('*[data-id="solidityLocals"]', '149: 0 uint256')
-    .notContainsText('*[data-id="solidityLocals"]', '150: 0 uint256')
+    .notContainsText('*[data-id="solidityLocals"]', '150: 0 uint256')    
+  },
+
+  'Should debug using generated sources': function (browser: NightwatchBrowser) {
+    browser
+    .clickLaunchIcon('solidity')
+    .setSolidityCompilerVersion('soljson-v0.7.2+commit.51b20bc0.js')
+    .clickLaunchIcon('udapp')    
+    .testContracts('withGeneratedSources.sol', sources[4]['browser/withGeneratedSources.sol'], ['A'])
+    .createContract('')
+    .clickInstance(4)
+    .clickFunction('f - transact (not payable)', {types: 'uint256[] ', values: '[]'})
+    .debugTransaction(8)
+    .pause(2000)
+    .click('*[data-id="debuggerTransactionStartButton"]') // stop debugging
+    .click('*[data-id="debugGeneratedSourcesLabel"]') // select debug with generated sources
+    .click('*[data-id="debuggerTransactionStartButton"]') // start debugging
+    .pause(2000)
+    .getEditorValue((content) => {
+      browser.assert.ok(content.indexOf('if slt(sub(dataEnd, headStart), 32) { revert(0, 0) }') != -1, 'current displayed content is not a generated source')
+    })
     .end()
   },
 
@@ -227,6 +247,17 @@ const sources = [
     }
         `
     }
+  },
+  {
+    'browser/withGeneratedSources.sol': {
+      content: `
+      // SPDX-License-Identifier: GPL-3.0
+      pragma experimental ABIEncoderV2; 
+      contract A { 
+        function f(uint[] memory) public returns (uint256) { } 
+      }
+      `
+    }
   }
 ]
 
@@ -293,4 +324,4 @@ const localVariable_step717_ABIEncoder = {
       "error": "<decoding failed - no decoder for calldata>",
       "type": "bytes"
   }
-}
+}
\ No newline at end of file

apps/remix-ide/src/app/tabs/debugger/debuggerUI.js

@@ -22,6 +22,26 @@ var css = csjs`
   .statusMessage {
     margin-left: 15px;
   }
+
+  .debuggerConfig {
+    display: flex;
+    align-items: center;
+  }
+
+  .debuggerConfig label {
+    margin: 0;
+  }
+
+  .debuggerSection {
+    padding: 12px 24px 16px;
+  }
+
+  .debuggerLabel {
+    margin-bottom: 2px;
+    font-size: 11px;
+    line-height: 12px;
+    text-transform: uppercase;
+  }
 `
 
 class DebuggerUI {
@@ -32,7 +52,9 @@ class DebuggerUI {
     this.event = new EventManager()
 
     this.isActive = false
-
+    this.opt = {
+      debugWithGeneratedSources: false
+    }
     this.sourceHighlighter = new SourceHighlighter()
 
     this.startTxBrowser()
@@ -72,12 +94,29 @@ class DebuggerUI {
       this.isActive = isActive
     })
 
-    this.debugger.event.register('newSourceLocation', async (lineColumnPos, rawLocation) => {
+    this.debugger.event.register('newSourceLocation', async (lineColumnPos, rawLocation, generatedSources) => {
+      if (!lineColumnPos) return
       const contracts = await this.fetchContractAndCompile(
         this.currentReceipt.contractAddress || this.currentReceipt.to,
         this.currentReceipt)
       if (contracts) {
-        const path = contracts.getSourceName(rawLocation.file)
+        let path = contracts.getSourceName(rawLocation.file)
+        if (!path) {
+          // check in generated sources
+          for (const source of generatedSources) {
+            if (source.id === rawLocation.file) {
+              path = `.debugger/generated-sources/${source.name}`
+              let content
+              try {
+                content = await this.debuggerModule.call('fileManager', 'getFile', path, source.contents)
+              } catch (e) {}
+              if (content !== source.contents) {
+                await this.debuggerModule.call('fileManager', 'setFile', path, source.contents)
+              }
+              break
+            }
+          }
+        }
         if (path) {
           await this.debuggerModule.call('editor', 'discardHighlight')
           await this.debuggerModule.call('editor', 'highlight', lineColumnPos, path)
@@ -137,7 +176,8 @@ class DebuggerUI {
           console.error(e)
         }
         return null
-      }
+      },
+      debugWithGeneratedSources: this.opt.debugWithGeneratedSources
     })
 
     this.listenToEvents()
@@ -167,7 +207,8 @@ class DebuggerUI {
             console.error(e)
           }
           return null
-        }
+        },
+        debugWithGeneratedSources: false
       })
       debug.debugger.traceManager.traceRetriever.getTrace(hash, (error, trace) => {
         if (error) return reject(error)
@@ -186,12 +227,18 @@ class DebuggerUI {
     this.stepManagerView = yo`<div class="px-2"></div>`
 
     var view = yo`
-      <div>
+      <div>        
         <div class="px-2">
+          <div class="mt-3">
+            <p class="mt-2 ${css.debuggerLabel}">Debugger Configuration</p>
+              <div class="mt-2 ${css.debuggerConfig} custom-control custom-checkbox">
+              <input class="custom-control-input" id="debugGeneratedSourcesInput" onchange=${(event) => { this.opt.debugWithGeneratedSources = event.target.checked }} type="checkbox" title="Debug with generated sources">
+              <label data-id="debugGeneratedSourcesLabel" class="form-check-label custom-control-label" for="debugGeneratedSourcesInput">Debug Generated sources if available (>= Solidity 0.7.2)</label>
+            </div>            
+          </div>       
           ${this.txBrowser.render()}
           ${this.stepManagerView}
-          ${this.debuggerHeadPanelsView}
-        </div>
+          ${this.debuggerHeadPanelsView}          
         <div class="${css.statusMessage}">${this.statusMessage}</div>
         ${this.debuggerPanelsView}
       </div>

apps/remix-ide/src/remixAppManager.js

@@ -10,7 +10,7 @@ const requiredModules = [ // services + layout views + system views
   'terminal', 'settings', 'pluginManager']
 
 export function isNative (name) {
-  const nativePlugins = ['vyper', 'workshops']
+  const nativePlugins = ['vyper', 'workshops', 'debugger']
   return nativePlugins.includes(name) || requiredModules.includes(name)
 }
 

libs/remix-astwalker/src/astWalker.ts

@@ -12,7 +12,7 @@ const isObject = function(obj: any): boolean {
 export function isAstNode(node: Record<string, unknown>): boolean {
   return (
     isObject(node) &&
-    'id' in node &&
+    // 'id' in node &&
     'nodeType' in node &&
     'src' in node
   )

libs/remix-debug/src/Ethdebugger.js

@@ -26,6 +26,7 @@ const {SolidityProxy, stateDecoder, localDecoder, InternalCallTree} = require('.
 function Ethdebugger (opts) {
   this.compilationResult = opts.compilationResult || function (contractAddress) { return null }
   this.web3 = opts.web3
+  this.opts = opts
 
   this.event = new EventManager()
 
@@ -36,7 +37,12 @@ function Ethdebugger (opts) {
   this.solidityProxy = new SolidityProxy({getCurrentCalledAddressAt: this.traceManager.getCurrentCalledAddressAt.bind(this.traceManager), getCode: this.codeManager.getCode.bind(this.codeManager)})
   this.storageResolver = null
 
-  this.callTree = new InternalCallTree(this.event, this.traceManager, this.solidityProxy, this.codeManager, { includeLocalVariables: true })
+  const includeLocalVariables = true
+  this.callTree = new InternalCallTree(this.event, 
+      this.traceManager,
+      this.solidityProxy,
+      this.codeManager, 
+      { ...opts, includeLocalVariables})
 }
 
 Ethdebugger.prototype.setManagers = function () {
@@ -44,8 +50,13 @@ Ethdebugger.prototype.setManagers = function () {
   this.codeManager = new CodeManager(this.traceManager)
   this.solidityProxy = new SolidityProxy({getCurrentCalledAddressAt: this.traceManager.getCurrentCalledAddressAt.bind(this.traceManager), getCode: this.codeManager.getCode.bind(this.codeManager)})
   this.storageResolver = null
+  const includeLocalVariables = true
 
-  this.callTree = new InternalCallTree(this.event, this.traceManager, this.solidityProxy, this.codeManager, { includeLocalVariables: true })
+  this.callTree = new InternalCallTree(this.event, 
+      this.traceManager, 
+      this.solidityProxy, 
+      this.codeManager, 
+      { ...this.opts, includeLocalVariables})
   this.event.trigger('managersChanged')
 }
 

libs/remix-debug/src/cmdline/index.js

@@ -79,6 +79,7 @@ class CmdLine {
     this.txHash = txNumber
     this.debugger.debug(null, txNumber, null, () => {
       this.debugger.event.register('newSourceLocation', (lineColumnPos, rawLocation) => {
+        if (!lineColumnPos) return
         this.lineColumnPos = lineColumnPos
         this.rawLocation = rawLocation
         this.events.emit('source', [lineColumnPos, rawLocation])

libs/remix-debug/src/debugger/debugger.js

@@ -18,7 +18,8 @@ function Debugger (options) {
 
   this.debugger = new Ethdebugger({
     web3: options.web3,
-    compilationResult: this.compilationResult
+    debugWithGeneratedSources: options.debugWithGeneratedSources,
+    compilationResult: this.compilationResult,
   })
 
   const {traceManager, callTree, solidityProxy} = this.debugger
@@ -59,8 +60,17 @@ Debugger.prototype.registerAndHighlightCodeItem = async function (index) {
 
     this.debugger.callTree.sourceLocationTracker.getValidSourceLocationFromVMTraceIndex(address, index, compilationResultForAddress.data.contracts).then((rawLocation) => {
       if (compilationResultForAddress && compilationResultForAddress.data) {
-        var lineColumnPos = this.offsetToLineColumnConverter.offsetToLineColumn(rawLocation, rawLocation.file, compilationResultForAddress.source.sources, compilationResultForAddress.data.sources)
-        this.event.trigger('newSourceLocation', [lineColumnPos, rawLocation])
+        const generatedSources = this.debugger.callTree.sourceLocationTracker.getGeneratedSourcesFromAddress(address)
+        const astSources = Object.assign({}, compilationResultForAddress.data.sources)
+        const sources = Object.assign({}, compilationResultForAddress.source.sources)
+        if (generatedSources) {
+          for (const genSource of generatedSources) {
+            astSources[genSource.name] = { id: genSource.id, ast: genSource.ast }
+            sources[genSource.name] = { content: genSource.contents }
+          }
+        }
+        var lineColumnPos = this.offsetToLineColumnConverter.offsetToLineColumn(rawLocation, rawLocation.file, sources, astSources)
+        this.event.trigger('newSourceLocation', [lineColumnPos, rawLocation, generatedSources])
       } else {
         this.event.trigger('newSourceLocation', [null])
       }

libs/remix-debug/src/solidity-decoder/solidityProxy.js

@@ -39,15 +39,15 @@ class SolidityProxy {
     * @param {Int} vmTraceIndex  - index in the vm trave where to resolve the executed contract name
     * @param {Function} cb  - callback returns (error, contractName)
     */
-  async contractNameAt (vmTraceIndex) {
+  async contractObjectAt (vmTraceIndex) {
     const address = this.getCurrentCalledAddressAt(vmTraceIndex)
-    if (this.cache.contractNameByAddress[address]) {
-      return this.cache.contractNameByAddress[address]
+    if (this.cache.contractObjectByAddress[address]) {
+      return this.cache.contractObjectByAddress[address]
     }
     const code = await this.getCode(address)
-    const contractName = contractNameFromCode(this.contracts, code.bytecode, address)
-    this.cache.contractNameByAddress[address] = contractName
-    return contractName
+    const contract = contractObjectFromCode(this.contracts, code.bytecode, address)
+    this.cache.contractObjectByAddress[address] = contract
+    return contract
   }
 
   /**
@@ -86,8 +86,8 @@ class SolidityProxy {
     * @return {Object} - returns state variables of @args vmTraceIndex
     */
   async extractStateVariablesAt (vmtraceIndex) {
-    const contractName = await this.contractNameAt(vmtraceIndex)
-    return this.extractStateVariables(contractName)
+    const contract = await this.contractObjectAt(vmtraceIndex)
+    return this.extractStateVariables(contract.name)
   }
 
   /**
@@ -96,9 +96,13 @@ class SolidityProxy {
     * @param {Object} sourceLocation  - source location containing the 'file' to retrieve the AST from
     * @return {Object} - AST of the current file
     */
-  ast (sourceLocation) {
+  ast (sourceLocation, generatedSources) {
     const file = this.fileNameFromIndex(sourceLocation.file)
-    if (this.sources[file]) {
+    if (!file && generatedSources && generatedSources.length) {
+      for (const source of generatedSources) {
+        if (source.id === sourceLocation.file) return source.ast
+      }
+    } else if (this.sources[file]) {
       return this.sources[file].ast
     }
     return null
@@ -115,13 +119,13 @@ class SolidityProxy {
   }
 }
 
-function contractNameFromCode (contracts, code, address) {
+function contractObjectFromCode (contracts, code, address) {
   const isCreation = traceHelper.isContractCreation(address)
   for (let file in contracts) {
     for (let contract in contracts[file]) {
       const bytecode = isCreation ? contracts[file][contract].evm.bytecode.object : contracts[file][contract].evm.deployedBytecode.object
       if (util.compareByteCode(code, '0x' + bytecode)) {
-        return contract
+        return { name: contract, contract: contracts[file][contract] }
       }
     }
   }
@@ -133,7 +137,7 @@ class Cache {
     this.reset()
   }
   reset () {
-    this.contractNameByAddress = {}
+    this.contractObjectByAddress = {}
     this.stateVariablesByContractName = {}
     this.contractDeclarations = null
     this.statesDefinitions = null

libs/remix-debug/src/source/astWalker.js

+'use strict'
+/**
+ * Crawl the given AST through the function walk(ast, callback)
+ */
+function AstWalker () {} // eslint-disable-line
+
+/**
+ * visit all the AST nodes
+ *
+ * @param {Object} ast  - AST node
+ * @param {Object or Function} callback  - if (Function) the function will be called for every node.
+ *                                       - if (Object) callback[<Node Type>] will be called for
+ *                                         every node of type <Node Type>. callback["*"] will be called for all other nodes.
+ *                                         in each case, if the callback returns false it does not descend into children.
+ *                                         If no callback for the current type, children are visited.
+ */
+AstWalker.prototype.walk = function (ast, callback) {
+  if (callback instanceof Function) {
+    callback = { '*': callback }
+  }
+  if (!('*' in callback)) {
+    callback['*'] = function () { return true }
+  }
+  const nodes = ast.nodes || (ast.body && ast.body.statements) || ast.declarations || ast.statements
+  if (nodes && ast.initializationExpression) { // 'for' loop handling
+    nodes.push(ast.initializationExpression)
+  }
+  if (manageCallBack(ast, callback) && nodes && nodes.length > 0) {
+    for (let k in nodes) {
+      const child = nodes[k]
+      this.walk(child, callback)
+    }
+  }
+}
+
+/**
+ * walk the given @astList
+ *
+ * @param {Object} sourcesList - sources list (containing root AST node)
+ * @param {Function} - callback used by AstWalker to compute response
+ */
+AstWalker.prototype.walkAstList = function (sourcesList, callback) {
+  const walker = new AstWalker()
+  for (let k in sourcesList) {
+    walker.walk(sourcesList[k].ast, callback)
+  }
+}
+
+function manageCallBack (node, callback) {
+  if (node.nodeType in callback) {
+    return callback[node.nodeType](node)
+  } else {
+    return callback['*'](node)
+  }
+}
+
+module.exports = AstWalker

libs/remix-debug/src/source/sourceLocationTracker.js

@@ -9,7 +9,10 @@ const util = remixLib.util
 /**
  * Process the source code location for the current executing bytecode
  */
-function SourceLocationTracker (_codeManager) {
+function SourceLocationTracker (_codeManager, { debugWithGeneratedSources }) {
+  this.opts = {
+    debugWithGeneratedSources: debugWithGeneratedSources || false
+  }
   this.codeManager = _codeManager
   this.event = new EventManager()
   this.sourceMappingDecoder = new SourceMappingDecoder()
@@ -25,7 +28,7 @@ function SourceLocationTracker (_codeManager) {
  */
 SourceLocationTracker.prototype.getSourceLocationFromInstructionIndex = async function (address, index, contracts) {
   const sourceMap = await extractSourceMap(this, this.codeManager, address, contracts)
-  return this.sourceMappingDecoder.atIndex(index, sourceMap)
+  return this.sourceMappingDecoder.atIndex(index, sourceMap.map)
 }
 
 /**
@@ -38,7 +41,19 @@ SourceLocationTracker.prototype.getSourceLocationFromInstructionIndex = async fu
 SourceLocationTracker.prototype.getSourceLocationFromVMTraceIndex = async function (address, vmtraceStepIndex, contracts) {  
   const sourceMap = await extractSourceMap(this, this.codeManager, address, contracts)
   const index = this.codeManager.getInstructionIndex(address, vmtraceStepIndex)
-  return this.sourceMappingDecoder.atIndex(index, sourceMap)
+  return this.sourceMappingDecoder.atIndex(index, sourceMap.map)
+}
+
+/**
+ * Returns the generated sources from a specific @arg address
+ *
+ * @param {String} address - contract address from which has generated sources
+ * @param {Object} generatedSources - Object containing the sourceid, ast and the source code.
+ */
+SourceLocationTracker.prototype.getGeneratedSourcesFromAddress = function (address) {
+  if (!this.opts.debugWithGeneratedSources) return null
+  if (this.sourceMapByAddress[address]) return this.sourceMapByAddress[address].generatedSources
+  return null
 }
 
 /**
@@ -73,7 +88,9 @@ function getSourceMap (address, code, contracts) {
 
       bytes = isCreation ? bytecode.object : deployedBytecode.object
       if (util.compareByteCode(code, '0x' + bytes)) {
-        return isCreation ? bytecode.sourceMap : deployedBytecode.sourceMap
+        const generatedSources = isCreation ? bytecode.generatedSources : deployedBytecode.generatedSources
+        const map = isCreation ? bytecode.sourceMap : deployedBytecode.sourceMap
+        return { generatedSources, map }
       }
     }
   }

libs/remix-debug/test/debugger.js

 var tape = require('tape')
+var deepequal = require('deep-equal')
 var remixLib = require('@remix-project/remix-lib')
 var compilerInput = require('./helpers/compilerHelper').compilerInput
 var SourceMappingDecoder = require('../src/source/sourceMappingDecoder')
@@ -263,7 +264,8 @@ function testDebugging (debugManager) {
       const location = await debugManager.sourceLocationFromVMTraceIndex(address, 330)
       debugManager.decodeLocalsAt(330, location, (error, decodedlocals) => {
         if (error) return t.end(error)
-        t.equal(JSON.stringify(decodedlocals), JSON.stringify(tested))
+        
+        t.ok(deepequal(decodedlocals, tested), `locals does not match. expected: ${JSON.stringify(tested)} - current: ${decodedlocals}`)
       })
     } catch (error) {
       return t.end(error)