Merge pull request #924 from ethereum/remix_tests

.gitignore

@@ -9,3 +9,5 @@ docs/_build
 package-lock.json
 .DS_Store
 .tern-port
+TODO
+soljson.js

.travis.yml

@@ -5,6 +5,7 @@ env:
   - TEST_DIR=remix-lib
   - TEST_DIR=remix-solidity
   - TEST_DIR=remix-debug
+  - TEST_DIR=remix-tests
 script: 
   - cd $TEST_DIR && npm install && npm test
 deploy:
@@ -15,4 +16,3 @@ deploy:
         branch: master
         condition: $TEST_DIR = remix-debugger
 cache: false
-

remix-tests/LICENSE.md

+MIT License
+
+Copyright (c) 2018 Remix Team
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

remix-tests/README.md

+[![Build Status](https://travis-ci.org/ethereum/remix-tests.svg?branch=master)](https://travis-ci.org/ethereum/remix-tests)
+
+Remix-Tests
+---
+
+> Tests for the Ethereum tool suite [Remix](https://github.com/ethereum/remix)
+
+### Installation
+
+`npm -g install remix-tests`
+
+### Test structure
+
+Example test file:
+```Javascript
+pragma solidity ^0.4.7;
+import "remix_tests.sol"; // injected by remix-tests
+import "./simple_storage.sol";
+
+contract MyTest {
+  SimpleStorage foo;
+  uint i = 0;
+
+  function beforeAll() {
+    foo = new SimpleStorage();
+  }
+
+  function beforeEach() {
+    if (i == 1) {
+      foo.set(200);
+    }
+    i += 1;
+  }
+
+  function initialValueShouldBe100() public {
+    Assert.equal(foo.get(), 100, "initial value is not correct");
+  }
+
+  function initialValueShouldBe200() public constant returns {
+    return Assert.equal(foo.get(), 200, "initial value is not correct");
+  }
+
+}
+```
+
+Available special functions:
+* `beforeEach` - runs before each test
+* `beforeAll` - runs before all tests
+
+#### Assert library
+
+Available functions:
+`Assert.ok(value, message)`
+`Assert.equal(value1, value2, message)`
+`Assert.notEqual(value1, value2, message)`
+
+supported values currently are: `bool` `uint` `int` `address` `bytes32`
+
+### Command Line
+
+Remix-Tests will assume the tests will files whose name end with "_test.sol". e.g `simple_storage_test.sol`
+
+Usage:
+
+* A directory with tests files `remix-tests examples/`
+* A test file `remix-tests examples/simple_storage_test.sol`
+
+### Library
+
+Importing the library:
+```Javascript
+const RemixTests = require('remix-tests');
+```
+
+Running a single test object:
+```Javascript
+remixTests.runTest(contractName, contractObj, testCallback, resultsCallback)
+```
+params:
+`testName` - `string` name of the test
+`testObj` -  web3.js 1.0 contract instance of the test
+`testCallback(object)` -  called each time there is a test event. 3 possible type of objects:
+* `{ type: 'contract', value: '<TestName>', filename: '<test_filename.sol>' }`
+* `{ type: 'testPass', value: '<name of testing function>', time: <time taken>, context: '<TestName>'}`
+* `{ type: 'testFailure', value: '<name of testing function>', time: <time taken>, context: '<TestName>', errMsg: '<message in the Assert>' }`
+
+`resultsCallback(object)`
+* `passingNum` - number of passing tests
+* `failureNum` - number of failing tests
+* `timePassed` - time it took for all the tests to run (in seconds)
+
+
+Running a set of tests given the sourcecode:
+```Javascript
+remixTests.runTestSources(contractSources, testCallback, resultCallback, finalCallback, importFileCb);
+```
+params:
+`contractSources` - `object` -> `filename => { content: source }`
+`testCallback(object)` - called each time there is a test event. 3 possible type of objects:
+* `{ type: 'contract', value: '<TestName>', filename: '<test_filename.sol>' }`
+* `{ type: 'testPass', value: '<name of testing function>', time: <time taken>, context: '<TestName>'}`
+* `{ type: 'testFailure', value: '<name of testing function>', time: <time taken>, context: '<TestName>', errMsg: '<message in the Assert>' }`
+
+`resultCallback(object)`
+* `passingNum` - number of passing tests
+* `failureNum` - number of failing tests
+* `timePassed` - time it took for all the tests to run (in seconds)
+
+`finalCallback(err)` - called when all tests finish running.
+`importCb(url, cb)`
+
+## Contribute
+
+Please feel free! Open an issue or a pull request. Please conform to [`standard`](https://standardjs.com/) for code styles, and make sure that you add any relevant tests.
+
+## License
+
+[MIT](LICENSE.md) © 2018 Remix Team

remix-tests/bin/remix-tests

+#!/usr/bin/env node
+
+require('../src/run.js');
+

remix-tests/examples/simple_storage.sol

+pragma solidity ^0.4.7;
+contract SimpleStorage {
+  uint public storedData;
+
+  function SimpleStorage() public {
+    storedData = 100;
+  }
+
+  function set(uint x) public {
+    storedData = x;
+  }
+
+  function get() public view returns (uint retVal) {
+    return storedData;
+  }
+
+}

remix-tests/examples/simple_storage2_test.sol

+pragma solidity ^0.4.7;
+import "./tests.sol";
+import "./simple_storage.sol";
+
+contract MyTest2 {
+  SimpleStorage foo;
+  uint i = 0;
+
+  function beforeAll() {
+    foo = new SimpleStorage();
+  }
+
+  function beforeEach() {
+    if (i == 1) {
+      foo.set(200);
+    }
+    i += 1;
+  }
+
+  function initialValueShouldBe100() public constant returns (bool) {
+    return Assert.equal(foo.get(), 100, "initial value is not correct");
+  }
+
+  function initialValueShouldBe200() public constant returns (bool) {
+    return Assert.equal(foo.get(), 200, "initial value is not correct");
+  }
+
+}
+

remix-tests/examples/simple_storage_test.sol

+pragma solidity ^0.4.7;
+import "./tests.sol";
+import "./simple_storage.sol";
+
+contract MyTest {
+  SimpleStorage foo;
+  uint i = 0;
+
+  function beforeAll() {
+    foo = new SimpleStorage();
+  }
+
+  function beforeEach() {
+    if (i == 1) {
+      foo.set(200);
+    }
+    i += 1;
+  }
+
+  function initialValueShouldBe100() public {
+    Assert.equal(foo.get(), 100, "initial value is not correct");
+  }
+
+  function initialValueShouldBe200() public {
+    Assert.equal(foo.get(), 200, "initial value is not correct");
+  }
+
+}
+

remix-tests/package.json

+{
+  "name": "remix-tests",
+  "version": "0.0.6",
+  "description": "Tests for the Ethereum tool suite Remix",
+  "main": "./src/index.js",
+  "contributors": [
+    {
+      "name": "Iuri Matias",
+      "email": "iuri@ethereum.org"
+    },
+    {
+      "name": "Yann Levreau",
+      "email": "yann@ethdev.com"
+    }
+  ],
+  "bin": {
+    "remix-tests": "./bin/remix-tests"
+  },
+  "scripts": {
+    "lint": "standard",
+    "test": "standard && mocha tests/ -t 300000"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/ethereum/remix-tests.git"
+  },
+  "author": "Remix Team",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/ethereum/remix-tests/issues"
+  },
+  "standard": {
+    "ignore": [
+      "tests/"
+    ]
+  },
+  "homepage": "https://github.com/ethereum/remix-tests#readme",
+  "dependencies": {
+    "async": "^2.6.0",
+    "babel-preset-es2017": "^6.24.1",
+    "change-case": "^3.0.1",
+    "colors": "^1.1.2",
+    "commander": "^2.13.0",
+    "remix-simulator": "latest",
+    "remix-solidity": "^0.2.2",
+    "solc": "^0.4.24",
+    "standard": "^10.0.3",
+    "web3": "1.0.0-beta.27"
+  },
+  "devDependencies": {
+    "mocha": "^5.1.0"
+  }
+}

remix-tests/sol/tests.sol.js

+module.exports = `
+pragma solidity ^0.4.7;
+
+library Assert {
+
+  event AssertionEvent(
+    bool passed,
+    string message
+  );
+
+  function ok(bool a, string message) public returns (bool result) {
+    result = a;
+    emit AssertionEvent(result, message);
+  }
+
+  function equal(uint a, uint b, string message) public returns (bool result) {
+    result = (a == b);
+    emit AssertionEvent(result, message);
+  }
+
+  function equal(int a, int b, string message) public returns (bool result) {
+    result = (a == b);
+    emit AssertionEvent(result, message);
+  }
+
+  function equal(bool a, bool b, string message) public returns (bool result) {
+    result = (a == b);
+    emit AssertionEvent(result, message);
+  }
+
+  // TODO: only for certain versions of solc
+  //function equal(fixed a, fixed b, string message) public returns (bool result) {
+  //  result = (a == b);
+  //  emit AssertionEvent(result, message);
+  //}
+
+  // TODO: only for certain versions of solc
+  //function equal(ufixed a, ufixed b, string message) public returns (bool result) {
+  //  result = (a == b);
+  //  emit AssertionEvent(result, message);
+  //}
+
+  function equal(address a, address b, string message) public returns (bool result) {
+    result = (a == b);
+    emit AssertionEvent(result, message);
+  }
+
+  function equal(bytes32 a, bytes32 b, string message) public returns (bool result) {
+    result = (a == b);
+    emit AssertionEvent(result, message);
+  }
+
+  function equal(string a, string b, string message) public returns (bool result) {
+     result = (keccak256(a) == keccak256(b));
+     AssertionEvent(result, message);
+  }
+
+  function notEqual(uint a, uint b, string message) public returns (bool result) {
+    result = (a != b);
+    emit AssertionEvent(result, message);
+  }
+
+  function notEqual(int a, int b, string message) public returns (bool result) {
+    result = (a != b);
+    emit AssertionEvent(result, message);
+  }
+
+  function notEqual(bool a, bool b, string message) public returns (bool result) {
+    result = (a != b);
+    emit AssertionEvent(result, message);
+  }
+
+  // TODO: only for certain versions of solc
+  //function notEqual(fixed a, fixed b, string message) public returns (bool result) {
+  //  result = (a != b);
+  //  emit AssertionEvent(result, message);
+  //}
+
+  // TODO: only for certain versions of solc
+  //function notEqual(ufixed a, ufixed b, string message) public returns (bool result) {
+  //  result = (a != b);
+  //  emit AssertionEvent(result, message);
+  //}
+
+  function notEqual(address a, address b, string message) public returns (bool result) {
+    result = (a != b);
+    emit AssertionEvent(result, message);
+  }
+
+  function notEqual(bytes32 a, bytes32 b, string message) public returns (bool result) {
+    result = (a != b);
+    emit AssertionEvent(result, message);
+  }
+
+  function notEqual(string a, string b, string message) public returns (bool result) {
+    result = (keccak256(a) != keccak256(b));
+    AssertionEvent(result, message);
+  }
+
+}
+`

remix-tests/src/compiler.js

+let fs = require('fs')
+var async = require('async')
+var path = require('path')
+
+let RemixCompiler = require('remix-solidity').Compiler
+
+// TODO: replace this with remix's own compiler code
+
+function compileFileOrFiles (filename, isDirectory, cb) {
+  let compiler, filepath
+
+  const sources = {
+    'tests.sol': { content: require('../sol/tests.sol.js') },
+    'remix_tests.sol': { content: require('../sol/tests.sol.js') }
+  }
+
+  // TODO: for now assumes filepath dir contains all tests, later all this
+  // should be replaced with remix's & browser solidity compiler code
+  filepath = (isDirectory ? filename : path.dirname(filename))
+
+  fs.readdirSync(filepath).forEach(file => {
+    sources[file] = {content: fs.readFileSync(path.join(filepath, file)).toString()}
+  })
+
+  async.waterfall([
+    function loadCompiler (next) {
+      compiler = new RemixCompiler()
+      compiler.onInternalCompilerLoaded()
+      // compiler.event.register('compilerLoaded', this, function (version) {
+      next()
+      // });
+    },
+    function doCompilation (next) {
+      compiler.event.register('compilationFinished', this, function (success, data, source) {
+        next(null, data)
+      })
+      compiler.compile(sources, filepath)
+    }
+  ], function (err, result) {
+    let errors = (result.errors || []).filter((e) => e.type === 'Error' || e.severity === 'error')
+    if (errors.length > 0) {
+      console.dir(errors)
+      return cb(new Error('errors compiling'))
+    }
+    cb(err, result.contracts)
+  })
+}
+
+function compileContractSources (sources, importFileCb, cb) {
+  let compiler, filepath
+
+  if (!sources['remix_tests.sol']) {
+    sources['remix_tests.sol'] = {content: require('../sol/tests.sol.js')}
+  }
+
+  async.waterfall([
+    function loadCompiler (next) {
+      compiler = new RemixCompiler(importFileCb)
+      compiler.onInternalCompilerLoaded()
+      // compiler.event.register('compilerLoaded', this, function (version) {
+      next()
+      // });
+    },
+    function doCompilation (next) {
+      compiler.event.register('compilationFinished', this, function (success, data, source) {
+        next(null, data)
+      })
+      compiler.compile(sources, filepath)
+    }
+  ], function (err, result) {
+    let errors = (result.errors || []).filter((e) => e.type === 'Error' || e.severity === 'error')
+    if (errors.length > 0) {
+      console.dir(errors)
+      return cb(new Error('errors compiling'))
+    }
+    cb(err, result.contracts)
+  })
+}
+
+module.exports = {
+  compileFileOrFiles: compileFileOrFiles,
+  compileContractSources: compileContractSources
+}

remix-tests/src/deployer.js

+var async = require('async')
+var remixLib = require('remix-lib')
+
+function deployAll (compileResult, web3, callback) {
+  let compiledObject = {}
+  let contracts = {}
+  let accounts = []
+
+  async.waterfall([
+    function getAccountList (next) {
+      web3.eth.getAccounts((_err, _accounts) => {
+        accounts = _accounts
+        next()
+      })
+    },
+    function getContractData (next) {
+      for (let contractFile in compileResult) {
+        for (let contractName in compileResult[contractFile]) {
+          let contract = compileResult[contractFile][contractName]
+
+          const className = contractName
+          const filename = contractFile
+
+          let abi = contract.abi
+          let code = contract.evm.bytecode.object
+
+          compiledObject[className] = {}
+          compiledObject[className].abi = abi
+          compiledObject[className].code = code
+          compiledObject[className].filename = filename
+          compiledObject[className].className = className
+          compiledObject[className].raw = contract
+
+          if (contractFile.indexOf('_test.sol') >= 0) {
+            compiledObject[className].isTest = true
+          }
+        }
+      }
+      next()
+    },
+    function determineContractsToDeploy (next) {
+      let contractsToDeploy = ['Assert']
+      let allContracts = Object.keys(compiledObject)
+
+      for (let contractName of allContracts) {
+        if (contractName === 'Assert') {
+          continue
+        }
+        if (compiledObject[contractName].isTest) {
+          contractsToDeploy.push(contractName)
+        }
+      }
+      next(null, contractsToDeploy)
+    },
+    function deployContracts (contractsToDeploy, next) {
+      var deployRunner = (deployObject, contractObject, contractName, filename, callback) => {
+        deployObject.estimateGas().then((gasValue) => {
+          deployObject.send({
+            from: accounts[0],
+            gas: Math.ceil(gasValue * 1.2)
+          }).on('receipt', function (receipt) {
+            contractObject.options.address = receipt.contractAddress
+            contractObject.options.from = accounts[0]
+            contractObject.options.gas = 5000 * 1000
+            compiledObject[contractName].deployedAddress = receipt.contractAddress
+
+            contracts[contractName] = contractObject
+            contracts[contractName].filename = filename
+
+            callback(null, { result: { createdAddress: receipt.contractAddress } }) // TODO this will only work with JavaScriptV VM
+          }).on('error', function (err) {
+            console.dir(err)
+            callback(err)
+          })
+        })
+      }
+
+      async.eachOfLimit(contractsToDeploy, 1, function (contractName, index, nextEach) {
+        let contract = compiledObject[contractName]
+        let encodeDataFinalCallback = (error, contractDeployData) => {
+          if (error) return nextEach(error)
+          let contractObject = new web3.eth.Contract(contract.abi)
+          let deployObject = contractObject.deploy({arguments: [], data: '0x' + contractDeployData.dataHex})
+          deployRunner(deployObject, contractObject, contractName, contract.filename, (error) => { nextEach(error) })
+        }
+
+        let encodeDataStepCallback = (msg) => { console.dir(msg) }
+
+        let encodeDataDeployLibraryCallback = (libData, callback) => {
+          let abi = compiledObject[libData.data.contractName].abi
+          let code = compiledObject[libData.data.contractName].code
+          let libraryObject = new web3.eth.Contract(abi)
+          let deployObject = libraryObject.deploy({arguments: [], data: '0x' + code})
+          deployRunner(deployObject, libraryObject, libData.data.contractName, contract.filename, callback)
+        }
+
+        let funAbi = null // no need to set the abi for encoding the constructor
+        let params = '' // we suppose that the test contract does not have any param in the constructor
+        remixLib.execution.txFormat.encodeConstructorCallAndDeployLibraries(contractName, contract.raw, compileResult, params, funAbi, encodeDataFinalCallback, encodeDataStepCallback, encodeDataDeployLibraryCallback)
+      }, function () {
+        next(null, contracts)
+      })
+    }
+  ], callback)
+}
+
+module.exports = {
+  deployAll: deployAll
+}

remix-tests/src/index.js

+const async = require('async')
+const path = require('path')
+const fs = require('fs')
+require('colors')
+
+let Compiler = require('./compiler.js')
+let Deployer = require('./deployer.js')
+let TestRunner = require('./testRunner.js')
+
+const Web3 = require('web3')
+const Provider = require('remix-simulator').Provider
+
+var createWeb3Provider = function () {
+  let web3 = new Web3()
+  web3.setProvider(new Provider())
+  return web3
+}
+
+var runTestSources = function (contractSources, testCallback, resultCallback, finalCallback, importFileCb) {
+  async.waterfall([
+    function compile (next) {
+      Compiler.compileContractSources(contractSources, importFileCb, next)
+    },
+    function deployAllContracts (compilationResult, next) {
+      let web3 = createWeb3Provider()
+      Deployer.deployAll(compilationResult, web3, function (err, contracts) {
+        if (err) {
+          next(err)
+        }
+
+        next(null, compilationResult, contracts)
+      })
+    },
+    function determineTestContractsToRun (compilationResult, contracts, next) {
+      let contractsToTest = []
+
+      for (let filename in compilationResult) {
+        if (filename.indexOf('_test.sol') < 0) {
+          continue
+        }
+        Object.keys(compilationResult[filename]).forEach(contractName => {
+          contractsToTest.push(contractName)
+        })
+      }
+
+      next(null, contractsToTest, contracts)
+    },
+    function runTests (contractsToTest, contracts, next) {
+      let totalPassing = 0
+      let totalFailing = 0
+      let totalTime = 0
+      let errors = []
+
+      var _testCallback = function (result) {
+        if (result.type === 'testFailure') {
+          errors.push(result)
+        }
+        testCallback(result)
+      }
+
+      var _resultsCallback = function (_err, result, cb) {
+        resultCallback(_err, result, () => {})
+        totalPassing += result.passingNum
+        totalFailing += result.failureNum
+        totalTime += result.timePassed
+        cb()
+      }
+
+      async.eachOfLimit(contractsToTest, 1, (contractName, index, cb) => {
+        TestRunner.runTest(contractName, contracts[contractName], _testCallback, (err, result) => {
+          if (err) {
+            return cb(err)
+          }
+          _resultsCallback(null, result, cb)
+        })
+      }, function (err, _results) {
+        if (err) {
+          return next(err)
+        }
+
+        let finalResults = {}
+
+        finalResults.totalPassing = totalPassing || 0
+        finalResults.totalFailing = totalFailing || 0
+        finalResults.totalTime = totalTime || 0
+        finalResults.errors = []
+
+        errors.forEach((error, _index) => {
+          finalResults.errors.push({context: error.context, value: error.value, message: error.errMsg})
+        })
+
+        next(null, finalResults)
+      })
+    }
+  ], finalCallback)
+}
+
+var runTestFiles = function (filepath, isDirectory, web3) {
+  async.waterfall([
+    function compile (next) {
+      Compiler.compileFileOrFiles(filepath, isDirectory, next)
+    },
+    function deployAllContracts (compilationResult, next) {
+      Deployer.deployAll(compilationResult, web3, function (err, contracts) {
+        if (err) {
+          next(err)
+        }
+
+        next(null, compilationResult, contracts)
+      })
+    },
+    function determineTestContractsToRun (compilationResult, contracts, next) {
+      let contractsToTest = []
+      if (isDirectory) {
+        fs.readdirSync(filepath).forEach(filename => {
+          if (filename.indexOf('_test.sol') < 0) {
+            return
+          }
+          Object.keys(compilationResult[path.basename(filename)]).forEach(contractName => {
+            contractsToTest.push(contractName)
+          })
+        })
+      } else {
+        contractsToTest = Object.keys(compilationResult[path.basename(filepath)])
+      }
+
+      next(null, contractsToTest, contracts)
+    },
+    function runTests (contractsToTest, contracts, next) {
+      let totalPassing = 0
+      let totalFailing = 0
+      let totalTime = 0
+      let errors = []
+
+      var testCallback = function (result) {
+        if (result.type === 'contract') {
+          console.log('\n  ' + result.value)
+        } else if (result.type === 'testPass') {
+          console.log('\t✓ '.green.bold + result.value.grey)
+        } else if (result.type === 'testFailure') {
+          console.log('\t✘ '.bold.red + result.value.red)
+          errors.push(result)
+        }
+      }
+      var resultsCallback = function (_err, result, cb) {
+        totalPassing += result.passingNum
+        totalFailing += result.failureNum
+        totalTime += result.timePassed
+        cb()
+      }
+
+      async.eachOfLimit(contractsToTest, 1, (contractName, index, cb) => {
+        TestRunner.runTest(contractName, contracts[contractName], testCallback, (err, result) => {
+          if (err) {
+            return cb(err)
+          }
+          resultsCallback(null, result, cb)
+        })
+      }, function (err, _results) {
+        if (err) {
+          return next(err)
+        }
+
+        console.log('\n')
+        if (totalPassing > 0) {
+          console.log(('  ' + totalPassing + ' passing ').green + ('(' + totalTime + 's)').grey)
+        }
+        if (totalFailing > 0) {
+          console.log(('  ' + totalFailing + ' failing').red)
+        }
+        console.log('')
+
+        errors.forEach((error, index) => {
+          console.log('  ' + (index + 1) + ') ' + error.context + ' ' + error.value)
+          console.log('')
+          console.log(('\t error: ' + error.errMsg).red)
+        })
+        console.log('')
+
+        next()
+      })
+    }
+  ], function () {
+  })
+}
+
+module.exports = {
+  runTestFiles: runTestFiles,
+  runTestSources: runTestSources,
+  runTest: TestRunner.runTest,
+  assertLibCode: require('../sol/tests.sol.js')
+}

remix-tests/src/run.js

+const commander = require('commander')
+const Web3 = require('web3')
+const RemixTests = require('./index.js')
+const fs = require('fs')
+
+const Provider = require('remix-simulator').Provider
+
+commander.action(function (filename) {
+  let web3 = new Web3()
+  // web3.setProvider(new web3.providers.HttpProvider('http://localhost:8545'))
+  web3.setProvider(new Provider())
+  // web3.setProvider(new web3.providers.WebsocketProvider('ws://localhost:8546'))
+
+  let isDirectory = fs.lstatSync(filename).isDirectory()
+  RemixTests.runTestFiles(filename, isDirectory, web3)
+})
+
+if (!process.argv.slice(2).length) {
+  console.log('please specify filename')
+}
+
+commander.parse(process.argv)

remix-tests/src/testRunner.js

+var async = require('async')
+var changeCase = require('change-case')
+var Web3 = require('web3')
+
+function getAvailableFunctions (jsonInterface) {
+  return jsonInterface.reverse().filter((x) => x.type === 'function').map((x) => x.name)
+}
+
+function getTestFunctions (jsonInterface) {
+  let specialFunctions = ['beforeAll', 'beforeEach', 'afterAll', 'afterEach']
+  return jsonInterface.filter((x) => specialFunctions.indexOf(x.name) < 0 && x.type === 'function')
+}
+
+function createRunList (jsonInterface) {
+  let availableFunctions = getAvailableFunctions(jsonInterface)
+  let testFunctions = getTestFunctions(jsonInterface)
+  let runList = []
+
+  if (availableFunctions.indexOf('beforeAll') >= 0) {
+    runList.push({name: 'beforeAll', type: 'internal', constant: false})
+  }
+
+  for (let func of testFunctions) {
+    if (availableFunctions.indexOf('beforeEach') >= 0) {
+      runList.push({name: 'beforeEach', type: 'internal', constant: false})
+    }
+    runList.push({name: func.name, type: 'test', constant: func.constant})
+    if (availableFunctions.indexOf('afterEach') >= 0) {
+      runList.push({name: 'afterEach', type: 'internal', constant: false})
+    }
+  }
+
+  if (availableFunctions.indexOf('afterAll') >= 0) {
+    runList.push({name: 'afterAll', type: 'internal', constant: false})
+  }
+
+  return runList
+}
+
+function runTest (testName, testObject, testCallback, resultsCallback) {
+  let runList = createRunList(testObject._jsonInterface)
+
+  let passingNum = 0
+  let failureNum = 0
+  let timePassed = 0
+  let web3 = new Web3()
+
+  testCallback({type: 'contract', value: testName, filename: testObject.filename})
+  async.eachOfLimit(runList, 1, function (func, index, next) {
+    let method = testObject.methods[func.name].apply(testObject.methods[func.name], [])
+    let startTime = Date.now()
+    if (func.constant) {
+      method.call().then((result) => {
+        let time = Math.ceil((Date.now() - startTime) / 1000.0)
+        if (result) {
+          testCallback({type: 'testPass', value: changeCase.sentenceCase(func.name), time: time, context: testName})
+          passingNum += 1
+          timePassed += time
+        } else {
+          testCallback({type: 'testFailure', value: changeCase.sentenceCase(func.name), time: time, errMsg: 'function returned false', context: testName})
+          failureNum += 1
+        }
+        next()
+      })
+    } else {
+      method.send().on('receipt', function (receipt) {
+        try {
+          let time = Math.ceil((Date.now() - startTime) / 1000.0)
+          let topic = Web3.utils.sha3('AssertionEvent(bool,string)')
+
+          let testPassed = false
+
+          for (let i in receipt.events) {
+            let event = receipt.events[i]
+            if (event.raw.topics.indexOf(topic) >= 0) {
+              var testEvent = web3.eth.abi.decodeParameters(['bool', 'string'], event.raw.data)
+              if (!testEvent[0]) {
+                testCallback({type: 'testFailure', value: changeCase.sentenceCase(func.name), time: time, errMsg: testEvent[1], context: testName})
+                failureNum += 1
+                return next()
+              }
+              testPassed = true
+            }
+          }
+
+          if (testPassed) {
+            testCallback({type: 'testPass', value: changeCase.sentenceCase(func.name), time: time, context: testName})
+            passingNum += 1
+          }
+
+          return next()
+        } catch (err) {
+          console.log('error!')
+          console.dir(err)
+          return next(err)
+        }
+      }).on('error', function (err) {
+        next(err)
+      })
+    }
+  }, function () {
+    resultsCallback(null, {
+      passingNum: passingNum,
+      failureNum: failureNum,
+      timePassed: timePassed
+    })
+  })
+}
+
+module.exports = {
+  runTest: runTest
+}

remix-tests/tests/examples_1/simple_storage.sol

+pragma solidity ^0.4.7;
+contract SimpleStorage {
+  uint public storedData;
+
+  function SimpleStorage() public {
+    storedData = 100;
+  }
+
+  function set(uint x) public {
+    storedData = x;
+  }
+
+  function get() public view returns (uint retVal) {
+    return storedData;
+  }
+
+}

remix-tests/tests/examples_1/simple_storage_test.sol

+pragma solidity ^0.4.7;
+import "remix_tests.sol";
+import "./simple_storage.sol";
+
+contract MyTest {
+  SimpleStorage foo;
+
+  function beforeAll() {
+    foo = new SimpleStorage();
+  }
+
+  function initialValueShouldBe100() public constant returns (bool) {
+    //return Assert.equal(foo.get(), 100, "initial value is not correct");
+    return foo.get() == 100;
+  }
+
+  function initialValueShouldBe200() public constant returns (bool) {
+    //return Assert.equal(foo.get(), 200, "initial value is not correct");
+    return foo.get() == 200;
+  }
+
+  function shouldTriggerOneFail() public {
+    Assert.equal(uint(1), uint(2), "the test 1 fails");
+    Assert.equal(uint(1), uint(2), "the test 2 fails");
+  }
+
+  function shouldTriggerOnePass() public {
+    Assert.equal(uint(1), uint(1), "the test 3 fails");
+  }
+
+}
+

remix-tests/tests/examples_2/simple_storage.sol

+pragma solidity ^0.4.7;
+contract SimpleStorage {
+  uint public storedData;
+
+  function SimpleStorage() public {
+    storedData = 100;
+  }
+
+  function set(uint x) public {
+    storedData = x;
+  }
+
+  function get() public view returns (uint retVal) {
+    return storedData;
+  }
+
+}

remix-tests/tests/examples_2/simple_storage_test.sol

+pragma solidity ^0.4.7;
+import "./tests.sol";
+import "./simple_storage.sol";
+
+contract MyTest {
+  SimpleStorage foo;
+  uint i = 0;
+
+  function beforeEach() {
+    foo = new SimpleStorage();
+    if (i == 1) {
+      foo.set(200);
+    }
+    i += 1;
+  }
+
+  function initialValueShouldBe100() public constant returns (bool) {
+    //return Assert.equal(foo.get(), 100, "initial value is not correct");
+    return foo.get() == 100;
+  }
+
+  function initialValueShouldBe200() public constant returns (bool) {
+    //return Assert.equal(foo.get(), 200, "initial value is not correct");
+    return foo.get() == 200;
+  }
+
+}
+

remix-tests/tests/examples_3/simple_string.sol

+pragma solidity ^0.4.7;
+contract SimpleString {
+  string public storedData;
+
+  function SimpleString() public {
+    storedData = "Hello world!";
+  }
+
+  function get() public view returns (string retVal) {
+    return storedData;
+  }
+}

remix-tests/tests/examples_3/simple_string_test.sol

+pragma solidity ^0.4.7;
+import "./tests.sol";
+import "./simple_string.sol";
+
+contract StringTest {
+  SimpleString foo;
+
+  function beforeAll() {
+    foo = new SimpleString();
+  }
+
+  function initialValueShouldBeHello() public constant returns (bool) {
+    return Assert.equal(foo.get(), "Hello world!", "initial value is not correct");
+  }
+
+  function valueShouldNotBeHelloWorld() public constant returns (bool) {
+    return Assert.notEqual(foo.get(), "Hello wordl!", "initial value is not correct");
+  }
+
+  function valueShouldBeHelloWorld() public constant returns (bool) {
+    return Assert.equal(foo.get(), "Hello wordl!", "initial value is not correct");
+  }
+}

remix-tests/tests/testRunner.js

+const async = require('async')
+const Web3 = require('web3')
+const assert = require('assert')
+
+let Compiler = require('../src/compiler.js')
+let Deployer = require('../src/deployer.js')
+let TestRunner = require('../src/testRunner.js')
+const Provider = require('remix-simulator').Provider
+
+function compileAndDeploy (filename, callback) {
+  let web3 = new Web3()
+  web3.setProvider(new Provider())
+
+  async.waterfall([
+    function compile (next) {
+      Compiler.compileFileOrFiles(filename, false, next)
+    },
+    function deployAllContracts (compilationResult, next) {
+      Deployer.deployAll(compilationResult, web3, next)
+    }
+  ], function (_err, contracts) {
+    callback(null, contracts)
+  })
+}
+
+describe('testRunner', function () {
+  describe('#runTest', function() {
+    describe('test with beforeAll', function () {
+      let filename = 'tests/examples_1/simple_storage_test.sol'
+      let tests = [], results = {}
+
+      before(function (done) {
+        compileAndDeploy(filename, function (_err, contracts) {
+          var testCallback = function (test) {
+            tests.push(test)
+          }
+          var resultsCallback = function (_err, _results) {
+            results = _results
+            done()
+          }
+          TestRunner.runTest('MyTest', contracts.MyTest, testCallback, resultsCallback)
+        })
+      })
+
+      it('should 1 passing test', function () {
+        assert.equal(results.passingNum, 2)
+      })
+
+      it('should 1 failing test', function () {
+        assert.equal(results.failureNum, 2)
+      })
+
+      it('should returns 5 messages', function () {
+        assert.deepEqual(tests, [
+          { type: 'contract',    value: 'MyTest', filename: 'simple_storage_test.sol' },
+          { type: 'testFailure', value: 'Should trigger one fail', time: 1, context: 'MyTest', errMsg: 'the test 1 fails' },
+          { type: 'testPass',    value: 'Should trigger one pass', time: 1, context: 'MyTest'},
+          { type: 'testPass',    value: 'Initial value should be100', time: 1, context: 'MyTest' },
+          { type: 'testFailure', value: 'Initial value should be200', time: 1, context: 'MyTest', errMsg: 'function returned false' }
+        ])
+      })
+    })
+
+    describe('test with beforeEach', function () {
+      let filename = 'tests/examples_2/simple_storage_test.sol'
+      let tests = [], results = {}
+
+      before(function (done) {
+        compileAndDeploy(filename, function (_err, contracts) {
+          var testCallback = function (test) {
+            tests.push(test)
+          }
+          var resultsCallback = function (_err, _results) {
+            results = _results
+            done()
+          }
+          TestRunner.runTest('MyTest', contracts.MyTest, testCallback, resultsCallback)
+        })
+      })
+
+      it('should 2 passing tests', function () {
+        assert.equal(results.passingNum, 2)
+      })
+
+      it('should 0 failing tests', function () {
+        assert.equal(results.failureNum, 0)
+      })
+
+      it('should returns 3 messages', function () {
+        assert.deepEqual(tests, [
+          { type: 'contract', value: 'MyTest', filename: 'simple_storage_test.sol' },
+          { type: 'testPass', value: 'Initial value should be100', time: 1, context: 'MyTest' },
+          { type: 'testPass', value: 'Initial value should be200', time: 1, context: 'MyTest' }
+        ])
+      })
+    })
+
+    // Test string comparision
+    describe('test with beforeAll', function () {
+      let filename = 'tests/examples_3/simple_string_test.sol'
+      let tests = [], results = {}
+
+      before(function (done) {
+        compileAndDeploy(filename, function (_err, contracts) {
+          var testCallback = function (test) {
+            tests.push(test)
+          }
+          var resultsCallback = function (_err, _results) {
+            results = _results
+            done()
+          }
+          TestRunner.runTest('StringTest', contracts.StringTest, testCallback, resultsCallback)
+          TestRunner.runTest('StringTest2', contracts.StringTest2, testCallback, resultsCallback)
+        })
+      })
+
+      it('should 2 passing tests', function () {
+        assert.equal(results.passingNum, 2)
+      })
+
+      it('should 1 failing tests', function () {
+        assert.equal(results.failureNum, 1)
+      })
+
+      it('should returns 3 messages', function () {
+        assert.deepEqual(tests, [
+          { type: 'contract', value: 'StringTest', filename: 'simple_string_test.sol' },
+          { type: 'testFailure', value: 'Value should be hello world', time: 1, context: 'StringTest', "errMsg": "function returned false" },
+          { type: 'testPass', value: 'Value should not be hello world', time: 1, context: 'StringTest' },
+          { type: 'testPass', value: 'Initial value should be hello', time: 1, context: 'StringTest' },
+        ])
+      })
+    })
+  })
+})