Merge pull request #374 from ethereum/manage_generatedSources

apps/remix-ide-e2e/src/tests/debugger.test.ts

@@ -147,7 +147,27 @@ module.exports = {
     .waitForElementPresent('*[data-id="treeViewLoadMore"]')
     .click('*[data-id="treeViewLoadMore"]')
     .assert.containsText('*[data-id="solidityLocals"]', '149: 0 uint256')
-    .notContainsText('*[data-id="solidityLocals"]', '150: 0 uint256')
+    .notContainsText('*[data-id="solidityLocals"]', '150: 0 uint256')    
+  },
+
+  'Should debug using generated sources': function (browser: NightwatchBrowser) {
+    browser
+    .clickLaunchIcon('solidity')
+    .setSolidityCompilerVersion('soljson-v0.7.2+commit.51b20bc0.js')
+    .clickLaunchIcon('udapp')    
+    .testContracts('withGeneratedSources.sol', sources[4]['browser/withGeneratedSources.sol'], ['A'])
+    .createContract('')
+    .clickInstance(4)
+    .clickFunction('f - transact (not payable)', {types: 'uint256[] ', values: '[]'})
+    .debugTransaction(8)
+    .pause(2000)
+    .click('*[data-id="debuggerTransactionStartButton"]') // stop debugging
+    .click('*[data-id="debugGeneratedSourcesLabel"]') // select debug with generated sources
+    .click('*[data-id="debuggerTransactionStartButton"]') // start debugging
+    .pause(2000)
+    .getEditorValue((content) => {
+      browser.assert.ok(content.indexOf('if slt(sub(dataEnd, headStart), 32) { revert(0, 0) }') != -1, 'current displayed content is not a generated source')
+    })
     .end()
   },
 
@@ -227,6 +247,17 @@ const sources = [
     }
         `
     }
+  },
+  {
+    'browser/withGeneratedSources.sol': {
+      content: `
+      // SPDX-License-Identifier: GPL-3.0
+      pragma experimental ABIEncoderV2; 
+      contract A { 
+        function f(uint[] memory) public returns (uint256) { } 
+      }
+      `
+    }
   }
 ]
 
@@ -293,4 +324,4 @@ const localVariable_step717_ABIEncoder = {
       "error": "<decoding failed - no decoder for calldata>",
       "type": "bytes"
   }
-}
+}
\ No newline at end of file

apps/remix-ide/src/app/tabs/debugger/debuggerUI.js

@@ -22,6 +22,26 @@ var css = csjs`
   .statusMessage {
     margin-left: 15px;
   }
+
+  .debuggerConfig {
+    display: flex;
+    align-items: center;
+  }
+
+  .debuggerConfig label {
+    margin: 0;
+  }
+
+  .debuggerSection {
+    padding: 12px 24px 16px;
+  }
+
+  .debuggerLabel {
+    margin-bottom: 2px;
+    font-size: 11px;
+    line-height: 12px;
+    text-transform: uppercase;
+  }
 `
 
 class DebuggerUI {
@@ -32,7 +52,9 @@ class DebuggerUI {
     this.event = new EventManager()
 
     this.isActive = false
-
+    this.opt = {
+      debugWithGeneratedSources: false
+    }
     this.sourceHighlighter = new SourceHighlighter()
 
     this.startTxBrowser()
@@ -72,12 +94,31 @@ class DebuggerUI {
       this.isActive = isActive
     })
 
-    this.debugger.event.register('newSourceLocation', async (lineColumnPos, rawLocation) => {
+    this.debugger.event.register('newSourceLocation', async (lineColumnPos, rawLocation, generatedSources) => {
+      if (!lineColumnPos) return
       const contracts = await this.fetchContractAndCompile(
         this.currentReceipt.contractAddress || this.currentReceipt.to,
         this.currentReceipt)
       if (contracts) {
-        const path = contracts.getSourceName(rawLocation.file)
+        let path = contracts.getSourceName(rawLocation.file)
+        if (!path) {
+          // check in generated sources
+          for (const source of generatedSources) {
+            if (source.id === rawLocation.file) {
+              path = `browser/.debugger/generated-sources/${source.name}`
+              let content
+              try {
+                content = await this.debuggerModule.call('fileManager', 'getFile', path, source.contents)
+              } catch (e) {
+                console.log('unable to fetch generated sources, the file probably doesn\'t exist yet', e)
+              }
+              if (content !== source.contents) {
+                await this.debuggerModule.call('fileManager', 'setFile', path, source.contents)
+              }
+              break
+            }
+          }
+        }
         if (path) {
           await this.debuggerModule.call('editor', 'discardHighlight')
           await this.debuggerModule.call('editor', 'highlight', lineColumnPos, path)
@@ -137,7 +178,8 @@ class DebuggerUI {
           console.error(e)
         }
         return null
-      }
+      },
+      debugWithGeneratedSources: this.opt.debugWithGeneratedSources
     })
 
     this.listenToEvents()
@@ -167,7 +209,8 @@ class DebuggerUI {
             console.error(e)
           }
           return null
-        }
+        },
+        debugWithGeneratedSources: false
       })
       debug.debugger.traceManager.traceRetriever.getTrace(hash, (error, trace) => {
         if (error) return reject(error)
@@ -188,10 +231,16 @@ class DebuggerUI {
     var view = yo`
       <div>
         <div class="px-2">
+          <div class="mt-3">
+            <p class="mt-2 ${css.debuggerLabel}">Debugger Configuration</p>
+              <div class="mt-2 ${css.debuggerConfig} custom-control custom-checkbox">
+              <input class="custom-control-input" id="debugGeneratedSourcesInput" onchange=${(event) => { this.opt.debugWithGeneratedSources = event.target.checked }} type="checkbox" title="Debug with generated sources">
+              <label data-id="debugGeneratedSourcesLabel" class="form-check-label custom-control-label" for="debugGeneratedSourcesInput">Debug generated sources if available (from Solidity v0.7.2)</label>
+            </div>            
+          </div>       
           ${this.txBrowser.render()}
           ${this.stepManagerView}
-          ${this.debuggerHeadPanelsView}
-        </div>
+          ${this.debuggerHeadPanelsView}          
         <div class="${css.statusMessage}">${this.statusMessage}</div>
         ${this.debuggerPanelsView}
       </div>

apps/remix-ide/src/remixAppManager.js

@@ -10,7 +10,7 @@ const requiredModules = [ // services + layout views + system views
   'terminal', 'settings', 'pluginManager']
 
 export function isNative (name) {
-  const nativePlugins = ['vyper', 'workshops']
+  const nativePlugins = ['vyper', 'workshops', 'debugger']
   return nativePlugins.includes(name) || requiredModules.includes(name)
 }
 

libs/remix-astwalker/src/astWalker.ts

@@ -18,6 +18,14 @@ export function isAstNode(node: Record<string, unknown>): boolean {
   )
 }
 
+export function isYulAstNode(node: Record<string, unknown>): boolean {
+  return (
+    isObject(node) &&
+    'nodeType' in node &&
+    'src' in node
+  )
+}
+
 
 /**
  * Crawl the given AST through the function walk(ast, callback)
@@ -200,7 +208,7 @@ export class AstWalker extends EventEmitter {
   }
   // eslint-disable-next-line @typescript-eslint/ban-types, @typescript-eslint/explicit-module-boundary-types
   walkFullInternal(ast: AstNode, callback: Function) {
-    if (isAstNode(ast)) {
+    if (isAstNode(ast) || isYulAstNode(ast)) {
       // console.log(`XXX id ${ast.id}, nodeType: ${ast.nodeType}, src: ${ast.src}`);
       callback(ast);
       for (const k of Object.keys(ast)) {
@@ -223,7 +231,7 @@ export class AstWalker extends EventEmitter {
   // Normalizes parameter callback and calls walkFullInternal
   // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
   walkFull(ast: AstNode, callback: any) {
-    if (isAstNode(ast)) return this.walkFullInternal(ast, callback);
+    if (isAstNode(ast) || isYulAstNode(ast)) return this.walkFullInternal(ast, callback);
   }
 
   // eslint-disable-next-line @typescript-eslint/ban-types, @typescript-eslint/explicit-module-boundary-types

libs/remix-astwalker/src/sourceMappings.ts

-import { isAstNode, AstWalker } from './astWalker';
+import { isAstNode, isYulAstNode, AstWalker } from './astWalker';
 import { AstNode, LineColPosition, LineColRange, Location } from "./types";
 import { util } from "@remix-project/remix-lib";
 
@@ -31,7 +31,7 @@ export function lineColPositionFromOffset(offset: number, lineBreaks: Array<numb
  * @param astNode  The object to convert.
  */
 export function sourceLocationFromAstNode(astNode: AstNode): Location | null {
-  if (isAstNode(astNode) && astNode.src) {
+  if (isAstNode(astNode) && isYulAstNode(astNode) && astNode.src) {
     return sourceLocationFromSrc(astNode.src)
   }
   return null;

libs/remix-debug/src/Ethdebugger.js

@@ -26,6 +26,7 @@ const {SolidityProxy, stateDecoder, localDecoder, InternalCallTree} = require('.
 function Ethdebugger (opts) {
   this.compilationResult = opts.compilationResult || function (contractAddress) { return null }
   this.web3 = opts.web3
+  this.opts = opts
 
   this.event = new EventManager()
 
@@ -36,7 +37,12 @@ function Ethdebugger (opts) {
   this.solidityProxy = new SolidityProxy({getCurrentCalledAddressAt: this.traceManager.getCurrentCalledAddressAt.bind(this.traceManager), getCode: this.codeManager.getCode.bind(this.codeManager)})
   this.storageResolver = null
 
-  this.callTree = new InternalCallTree(this.event, this.traceManager, this.solidityProxy, this.codeManager, { includeLocalVariables: true })
+  const includeLocalVariables = true
+  this.callTree = new InternalCallTree(this.event, 
+      this.traceManager,
+      this.solidityProxy,
+      this.codeManager, 
+      { ...opts, includeLocalVariables})
 }
 
 Ethdebugger.prototype.setManagers = function () {
@@ -44,8 +50,13 @@ Ethdebugger.prototype.setManagers = function () {
   this.codeManager = new CodeManager(this.traceManager)
   this.solidityProxy = new SolidityProxy({getCurrentCalledAddressAt: this.traceManager.getCurrentCalledAddressAt.bind(this.traceManager), getCode: this.codeManager.getCode.bind(this.codeManager)})
   this.storageResolver = null
+  const includeLocalVariables = true
 
-  this.callTree = new InternalCallTree(this.event, this.traceManager, this.solidityProxy, this.codeManager, { includeLocalVariables: true })
+  this.callTree = new InternalCallTree(this.event, 
+      this.traceManager, 
+      this.solidityProxy, 
+      this.codeManager, 
+      { ...this.opts, includeLocalVariables})
   this.event.trigger('managersChanged')
 }
 

libs/remix-debug/src/cmdline/index.js

@@ -79,6 +79,7 @@ class CmdLine {
     this.txHash = txNumber
     this.debugger.debug(null, txNumber, null, () => {
       this.debugger.event.register('newSourceLocation', (lineColumnPos, rawLocation) => {
+        if (!lineColumnPos) return
         this.lineColumnPos = lineColumnPos
         this.rawLocation = rawLocation
         this.events.emit('source', [lineColumnPos, rawLocation])

libs/remix-debug/src/debugger/debugger.js

@@ -18,7 +18,8 @@ function Debugger (options) {
 
   this.debugger = new Ethdebugger({
     web3: options.web3,
-    compilationResult: this.compilationResult
+    debugWithGeneratedSources: options.debugWithGeneratedSources,
+    compilationResult: this.compilationResult,
   })
 
   const {traceManager, callTree, solidityProxy} = this.debugger
@@ -59,8 +60,17 @@ Debugger.prototype.registerAndHighlightCodeItem = async function (index) {
 
     this.debugger.callTree.sourceLocationTracker.getValidSourceLocationFromVMTraceIndex(address, index, compilationResultForAddress.data.contracts).then((rawLocation) => {
       if (compilationResultForAddress && compilationResultForAddress.data) {
-        var lineColumnPos = this.offsetToLineColumnConverter.offsetToLineColumn(rawLocation, rawLocation.file, compilationResultForAddress.source.sources, compilationResultForAddress.data.sources)
-        this.event.trigger('newSourceLocation', [lineColumnPos, rawLocation])
+        const generatedSources = this.debugger.callTree.sourceLocationTracker.getGeneratedSourcesFromAddress(address)
+        const astSources = Object.assign({}, compilationResultForAddress.data.sources)
+        const sources = Object.assign({}, compilationResultForAddress.source.sources)
+        if (generatedSources) {
+          for (const genSource of generatedSources) {
+            astSources[genSource.name] = { id: genSource.id, ast: genSource.ast }
+            sources[genSource.name] = { content: genSource.contents }
+          }
+        }
+        var lineColumnPos = this.offsetToLineColumnConverter.offsetToLineColumn(rawLocation, rawLocation.file, sources, astSources)
+        this.event.trigger('newSourceLocation', [lineColumnPos, rawLocation, generatedSources])
       } else {
         this.event.trigger('newSourceLocation', [null])
       }

libs/remix-debug/src/solidity-decoder/solidityProxy.js

@@ -39,15 +39,15 @@ class SolidityProxy {
     * @param {Int} vmTraceIndex  - index in the vm trave where to resolve the executed contract name
     * @param {Function} cb  - callback returns (error, contractName)
     */
-  async contractNameAt (vmTraceIndex) {
+  async contractObjectAt (vmTraceIndex) {
     const address = this.getCurrentCalledAddressAt(vmTraceIndex)
-    if (this.cache.contractNameByAddress[address]) {
-      return this.cache.contractNameByAddress[address]
+    if (this.cache.contractObjectByAddress[address]) {
+      return this.cache.contractObjectByAddress[address]
     }
     const code = await this.getCode(address)
-    const contractName = contractNameFromCode(this.contracts, code.bytecode, address)
-    this.cache.contractNameByAddress[address] = contractName
-    return contractName
+    const contract = contractObjectFromCode(this.contracts, code.bytecode, address)
+    this.cache.contractObjectByAddress[address] = contract
+    return contract
   }
 
   /**
@@ -86,8 +86,8 @@ class SolidityProxy {
     * @return {Object} - returns state variables of @args vmTraceIndex
     */
   async extractStateVariablesAt (vmtraceIndex) {
-    const contractName = await this.contractNameAt(vmtraceIndex)
-    return this.extractStateVariables(contractName)
+    const contract = await this.contractObjectAt(vmtraceIndex)
+    return this.extractStateVariables(contract.name)
   }
 
   /**
@@ -96,9 +96,13 @@ class SolidityProxy {
     * @param {Object} sourceLocation  - source location containing the 'file' to retrieve the AST from
     * @return {Object} - AST of the current file
     */
-  ast (sourceLocation) {
+  ast (sourceLocation, generatedSources) {
     const file = this.fileNameFromIndex(sourceLocation.file)
-    if (this.sources[file]) {
+    if (!file && generatedSources && generatedSources.length) {
+      for (const source of generatedSources) {
+        if (source.id === sourceLocation.file) return source.ast
+      }
+    } else if (this.sources[file]) {
       return this.sources[file].ast
     }
     return null
@@ -115,13 +119,13 @@ class SolidityProxy {
   }
 }
 
-function contractNameFromCode (contracts, code, address) {
+function contractObjectFromCode (contracts, code, address) {
   const isCreation = traceHelper.isContractCreation(address)
   for (let file in contracts) {
     for (let contract in contracts[file]) {
       const bytecode = isCreation ? contracts[file][contract].evm.bytecode.object : contracts[file][contract].evm.deployedBytecode.object
       if (util.compareByteCode(code, '0x' + bytecode)) {
-        return contract
+        return { name: contract, contract: contracts[file][contract] }
       }
     }
   }
@@ -133,7 +137,7 @@ class Cache {
     this.reset()
   }
   reset () {
-    this.contractNameByAddress = {}
+    this.contractObjectByAddress = {}
     this.stateVariablesByContractName = {}
     this.contractDeclarations = null
     this.statesDefinitions = null

libs/remix-debug/src/source/sourceLocationTracker.js

@@ -9,7 +9,10 @@ const util = remixLib.util
 /**
  * Process the source code location for the current executing bytecode
  */
-function SourceLocationTracker (_codeManager) {
+function SourceLocationTracker (_codeManager, { debugWithGeneratedSources }) {
+  this.opts = {
+    debugWithGeneratedSources: debugWithGeneratedSources || false
+  }
   this.codeManager = _codeManager
   this.event = new EventManager()
   this.sourceMappingDecoder = new SourceMappingDecoder()
@@ -25,7 +28,7 @@ function SourceLocationTracker (_codeManager) {
  */
 SourceLocationTracker.prototype.getSourceLocationFromInstructionIndex = async function (address, index, contracts) {
   const sourceMap = await extractSourceMap(this, this.codeManager, address, contracts)
-  return this.sourceMappingDecoder.atIndex(index, sourceMap)
+  return this.sourceMappingDecoder.atIndex(index, sourceMap.map)
 }
 
 /**
@@ -38,7 +41,19 @@ SourceLocationTracker.prototype.getSourceLocationFromInstructionIndex = async fu
 SourceLocationTracker.prototype.getSourceLocationFromVMTraceIndex = async function (address, vmtraceStepIndex, contracts) {  
   const sourceMap = await extractSourceMap(this, this.codeManager, address, contracts)
   const index = this.codeManager.getInstructionIndex(address, vmtraceStepIndex)
-  return this.sourceMappingDecoder.atIndex(index, sourceMap)
+  return this.sourceMappingDecoder.atIndex(index, sourceMap.map)
+}
+
+/**
+ * Returns the generated sources from a specific @arg address
+ *
+ * @param {String} address - contract address from which has generated sources
+ * @param {Object} generatedSources - Object containing the sourceid, ast and the source code.
+ */
+SourceLocationTracker.prototype.getGeneratedSourcesFromAddress = function (address) {
+  if (!this.opts.debugWithGeneratedSources) return null
+  if (this.sourceMapByAddress[address]) return this.sourceMapByAddress[address].generatedSources
+  return null
 }
 
 /**
@@ -73,7 +88,9 @@ function getSourceMap (address, code, contracts) {
 
       bytes = isCreation ? bytecode.object : deployedBytecode.object
       if (util.compareByteCode(code, '0x' + bytes)) {
-        return isCreation ? bytecode.sourceMap : deployedBytecode.sourceMap
+        const generatedSources = isCreation ? bytecode.generatedSources : deployedBytecode.generatedSources
+        const map = isCreation ? bytecode.sourceMap : deployedBytecode.sourceMap
+        return { generatedSources, map }
       }
     }
   }

libs/remix-debug/test/astwalker.js

-'use strict'
-const tape = require('tape')
-const AstWalker = require('../src/source/astWalker')
-const node = require('./resources/ast')
-
-tape('ASTWalker', function (t) {
-  t.test('ASTWalker.walk', function (st) {
-    st.plan(24)
-    const astwalker = new AstWalker()
-
-    astwalker.walk(node.ast.ast, function (node) {
-      if (node.nodeType === 'ContractDefinition') {
-        checkContract(st, node)
-      }
-      if (node.nodeType === 'FunctionDefinition') {
-        checkSetFunction(st, node)
-      }
-      return true
-    })
-
-    const callback = {}
-    callback.FunctionDefinition = function (node) {
-      st.equal(node.nodeType, 'FunctionDefinition')
-      st.equal(node.name === 'set' || node.name === 'get', true)
-      return true
-    }
-    astwalker.walk(node.ast.ast, callback)
-  })
-})
-
-function checkContract (st, node) {
-  st.equal(node.name, 'test')
-  st.equal(node.nodes[0].name, 'x')
-  st.equal(node.nodes[0].typeDescriptions.typeString, 'int256')
-  st.equal(node.nodes[1].name, 'y')
-  st.equal(node.nodes[1].typeDescriptions.typeString, 'int256')
-  st.equal(node.nodes[2].nodeType, 'FunctionDefinition')
-  st.equal(node.nodes[2].stateMutability, 'nonpayable')
-  st.equal(node.nodes[2].name, 'set')
-  st.equal(node.nodes[2].visibility, 'public')
-}
-
-function checkSetFunction (st, node) {
-  if (node.name === 'set') {
-    st.equal(node.parameters.nodeType, 'ParameterList')
-    st.equal(node.returnParameters.nodeType, 'ParameterList')
-    st.equal(node.body.nodeType, 'Block')
-    st.equal(node.body.statements[0].nodeType, 'ExpressionStatement')
-    checkExpressionStatement(st, node.body.statements[0])
-  }
-}
-
-function checkExpressionStatement (st, node) {
-  st.equal(node.expression.nodeType, 'Assignment')
-  st.equal(node.expression.operator, '=')
-  st.equal(node.expression.typeDescriptions.typeString, 'int256')
-  st.equal(node.expression.leftHandSide.nodeType, 'Identifier')
-  st.equal(node.expression.leftHandSide.name, 'x')
-  st.equal(node.expression.rightHandSide.nodeType, 'Identifier')
-  st.equal(node.expression.rightHandSide.name, '_x')
-}

libs/remix-debug/test/debugger.js

 var tape = require('tape')
+var deepequal = require('deep-equal')
 var remixLib = require('@remix-project/remix-lib')
 var compilerInput = require('./helpers/compilerHelper').compilerInput
 var SourceMappingDecoder = require('../src/source/sourceMappingDecoder')
@@ -263,7 +264,8 @@ function testDebugging (debugManager) {
       const location = await debugManager.sourceLocationFromVMTraceIndex(address, 330)
       debugManager.decodeLocalsAt(330, location, (error, decodedlocals) => {
         if (error) return t.end(error)
-        t.equal(JSON.stringify(decodedlocals), JSON.stringify(tested))
+        
+        t.ok(deepequal(decodedlocals, tested), `locals does not match. expected: ${JSON.stringify(tested)} - current: ${decodedlocals}`)
       })
     } catch (error) {
       return t.end(error)

libs/remix-debug/test/sourceLocationTracker.js

@@ -23,7 +23,7 @@ tape('SourceLocationTracker', function (t) {
 
     traceManager.resolveTrace(tx).then(async () => {     
 
-      const sourceLocationTracker = new SourceLocationTracker(codeManager)
+      const sourceLocationTracker = new SourceLocationTracker(codeManager, {debugWithGeneratedSources: false})
       
       try {
         const map = await sourceLocationTracker.getSourceLocationFromVMTraceIndex('0x0d3a18d64dfe4f927832ab58d6451cecc4e517c5', 0, output.contracts)
@@ -54,7 +54,7 @@ tape('SourceLocationTracker', function (t) {
 
     traceManager.resolveTrace(tx).then(async () => {     
 
-      const sourceLocationTracker = new SourceLocationTracker(codeManager)
+      const sourceLocationTracker = new SourceLocationTracker(codeManager, { debugWithGeneratedSources: false })
       
       try {
         let map = await sourceLocationTracker.getSourceLocationFromVMTraceIndex('0x0d3a18d64dfe4f927832ab58d6451cecc4e517c5', 0, output.contracts)