Merge pull request #1277 from andremedeiros/fix-test-reporter

remix-tests/package.json

@@ -45,7 +45,6 @@
     "remix-lib": "0.4.12",
     "remix-simulator": "0.1.9-alpha.5",
     "remix-solidity": "0.3.15",
-    "signale": "^1.2.1",
     "web3": "1.0.0-beta.36",
     "winston": "^3.0.0"
   },

remix-tests/src/testRunner.ts

@@ -62,15 +62,13 @@ export function runTest (testName, testObject: any, contractDetails: any, opts:
     let timePassed: number = 0
     let web3 = new Web3()
 
-    const userAgent = (typeof (navigator) !== 'undefined') && navigator.userAgent ? navigator.userAgent.toLowerCase() : '-'
-    const isBrowser = !(typeof (window) === 'undefined' || userAgent.indexOf(' electron/') > -1)
-    if (!isBrowser) {
-        let signale = require('signale')
-        signale.warn('DO NOT TRY TO ACCESS (IN YOUR SOLIDITY TEST) AN ACCOUNT GREATER THAN THE LENGTH OF THE FOLLOWING ARRAY (' + opts.accounts.length + ') :')
-        signale.warn(opts.accounts)
-        signale.warn('e.g: the following code won\'t work in the current context:')
-        signale.warn('TestsAccounts.getAccount(' + opts.accounts.length + ')')
+    const accts: TestResultInterface = {
+      type: 'accountList',
+      value: opts.accounts
     }
+
+    testCallback(undefined, accts);
+
     const resp: TestResultInterface = {
       type: 'contract',
       value: testName,
@@ -93,7 +91,7 @@ export function runTest (testName, testObject: any, contractDetails: any, opts:
         let startTime = Date.now()
         if (func.constant) {
             method.call(sendParams).then((result) => {
-                let time = Math.ceil((Date.now() - startTime) / 1000.0)
+                let time = (Date.now() - startTime) / 1000.0
                 if (result) {
                     const resp: TestResultInterface = {
                       type: 'testPass',
@@ -120,7 +118,7 @@ export function runTest (testName, testObject: any, contractDetails: any, opts:
         } else {
             method.send(sendParams).on('receipt', (receipt) => {
                 try {
-                    let time: number = Math.ceil((Date.now() - startTime) / 1000.0)
+                    let time: number = (Date.now() - startTime) / 1000.0
                     let topic = Web3.utils.sha3('AssertionEvent(bool,string)')
                     let testPassed: boolean = false
 

remix-tests/tests/testRunner.ts

@@ -9,14 +9,46 @@ import { deployAll } from '../dist/deployer'
 import { runTest } from '../dist/index'
 import { ResultsInterface, TestCbInterface, ResultCbInterface } from '../dist/index'
 
-var provider = new Provider()
+// deepEqualExcluding allows us to exclude specific keys whose values vary.
+// In this specific test, we'll use this helper to exclude `time` keys.
+// Assertions for the existance of these will be made at the correct places.
+function deepEqualExcluding(a: any, b: any, excludedKeys: string[]) {
+  function removeKeysFromObject(obj: any, excludedKeys: string[]) {
+    if (obj !== Object(obj)) {
+      return obj
+    }
+
+    if(Object.prototype.toString.call(obj) !== '[object Array]') {
+      obj = Object.assign({}, obj)
+      for (const key of excludedKeys) {
+        delete obj[key]
+      }
+
+      return obj
+    }
+
+    let newObj = []
+    for (const idx in obj) {
+      newObj[idx] = removeKeysFromObject(obj[idx], excludedKeys);
+    }
+
+    return newObj
+  }
+
+  let aStripped: any = removeKeysFromObject(a, excludedKeys);
+  let bStripped: any = removeKeysFromObject(b, excludedKeys);
+
+  assert.deepEqual(aStripped, bStripped)
+}
+
+let accounts: string[]
+let provider = new Provider()
 
 async function compileAndDeploy(filename: string, callback: Function) {
   let web3: Web3 = new Web3()
   await provider.init()
   web3.setProvider(provider)
   let compilationData: object
-  let accounts: string[]
   async.waterfall([
     function getAccountList(next: Function): void {
       web3.eth.getAccounts((_err: Error | null | undefined, _accounts: string[]) => {
@@ -43,26 +75,39 @@ async function compileAndDeploy(filename: string, callback: Function) {
 
 
 describe('testRunner', () => {
-  describe('#runTest', () => {
+    let tests: any[] = [], results: ResultsInterface;
+
+    const testCallback: TestCbInterface = (err, test) => {
+      if (err) { throw err }
+
+      if (test.type === 'testPass' || test.type === 'testFailure') {
+        assert.ok(test.time, 'test time not reported')
+        assert.ok(!Number.isInteger(test.time || 0), 'test time should not be an integer')
+      }
+
+      tests.push(test)
+    }
+
+    const resultsCallback: Function = (done) => {
+      return (err, _results) => {
+        if (err) { throw err }
+        results = _results
+        done()
+      }
+    }
+
+    describe('#runTest', () => {
     describe('test with beforeAll', () => {
       let filename: string = 'tests/examples_1/simple_storage_test.sol'
-      let tests: any[] = [], results: ResultsInterface;
 
       before((done) => {
-        compileAndDeploy(filename, (_err: Error | null | undefined, compilationData: object, contracts: any, accounts: object) => {
-          const testCallback: TestCbInterface = (err, test) => {
-            if (err) { throw err }
-            tests.push(test)
-          }
-          const resultsCallback: ResultCbInterface = (err, _results) => {
-            if (err) { throw err }
-            results = _results
-            done()
-          }
-          runTest('MyTest', contracts.MyTest, compilationData[filename]['MyTest'], { accounts }, testCallback, resultsCallback)
+        compileAndDeploy(filename, (_err: Error | null | undefined, compilationData: object, contracts: any, accounts: string[]) => {
+          runTest('MyTest', contracts.MyTest, compilationData[filename]['MyTest'], { accounts }, testCallback, resultsCallback(done))
         })
       })
 
+      after(() => { tests = [] })
+
       it('should 1 passing test', function () {
         assert.equal(results.passingNum, 2)
       })
@@ -71,37 +116,29 @@ describe('testRunner', () => {
         assert.equal(results.failureNum, 2)
       })
 
-      it('should returns 5 messages', function () {
-        assert.deepEqual(tests, [
+      it('should return 6 messages', function () {
+        deepEqualExcluding(tests, [
+          { type: 'accountList', value: accounts },
           { type: 'contract', value: 'MyTest', filename: 'tests/examples_1/simple_storage_test.sol' },
-          { type: 'testFailure', value: 'Should trigger one fail', time: 1, context: 'MyTest', errMsg: 'the test 1 fails' },
-          { type: 'testPass', value: 'Should trigger one pass', time: 1, context: 'MyTest' },
-          { type: 'testPass', value: 'Initial value should be100', time: 1, context: 'MyTest' },
-          { type: 'testFailure', value: 'Initial value should be200', time: 1, context: 'MyTest', errMsg: 'function returned false' }
-        ])
+          { type: 'testFailure', value: 'Should trigger one fail', context: 'MyTest', errMsg: 'the test 1 fails' },
+          { type: 'testPass', value: 'Should trigger one pass', context: 'MyTest' },
+          { type: 'testPass', value: 'Initial value should be100', context: 'MyTest' },
+          { type: 'testFailure', value: 'Initial value should be200', context: 'MyTest', errMsg: 'function returned false' }
+        ], ['time'])
       })
     })
 
     describe('test with beforeEach', function () {
       let filename = 'tests/examples_2/simple_storage_test.sol'
-      let tests: any[] = [], results: ResultsInterface;
 
       before(function (done) {
-        compileAndDeploy(filename, function (_err: Error | null | undefined, compilationData: object, contracts: any, accounts: object) {
-          const testCallback: TestCbInterface = (err, test) => {
-            if (err) { throw err }
-            tests.push(test)
-          }
-          const resultsCallback: ResultCbInterface = (err, _results) => {
-            if (err) { throw err }
-            results = _results
-            done()
-          }
-
-          runTest('MyTest', contracts.MyTest, compilationData[filename]['MyTest'], { accounts }, testCallback, resultsCallback)
+        compileAndDeploy(filename, function (_err: Error | null | undefined, compilationData: object, contracts: any, accounts: string[]) {
+          runTest('MyTest', contracts.MyTest, compilationData[filename]['MyTest'], { accounts }, testCallback, resultsCallback(done))
         })
       })
 
+      after(() => { tests = [] })
+
       it('should 2 passing tests', function () {
         assert.equal(results.passingNum, 2)
       })
@@ -110,35 +147,28 @@ describe('testRunner', () => {
         assert.equal(results.failureNum, 0)
       })
 
-      it('should returns 3 messages', function () {
-        assert.deepEqual(tests, [
+      it('should return 4 messages', function () {
+        deepEqualExcluding(tests, [
+          { type: 'accountList', value: accounts },
           { type: 'contract', value: 'MyTest', filename: 'tests/examples_2/simple_storage_test.sol' },
-          { type: 'testPass', value: 'Initial value should be100', time: 1, context: 'MyTest' },
-          { type: 'testPass', value: 'Initial value should be200', time: 1, context: 'MyTest' }
-        ])
+          { type: 'testPass', value: 'Initial value should be100', context: 'MyTest' },
+          { type: 'testPass', value: 'Initial value should be200', context: 'MyTest' }
+        ], ['time'])
       })
     })
 
     // Test string equality
     describe('test string equality', function () {
       let filename = 'tests/examples_3/simple_string_test.sol'
-      let tests: any[] = [], results: ResultsInterface;
 
       before(function (done) {
         compileAndDeploy(filename, (_err, compilationData, contracts, accounts) => {
-          const testCallback: TestCbInterface = (err, test) => {
-            if (err) { throw err }
-            tests.push(test)
-          }
-          const resultsCallback: ResultCbInterface = (err, _results) => {
-            if (err) { throw err }
-            results = _results
-            done()
-          }
-          runTest('StringTest', contracts.StringTest, compilationData[filename]['StringTest'], { accounts }, testCallback, resultsCallback)
+          runTest('StringTest', contracts.StringTest, compilationData[filename]['StringTest'], { accounts }, testCallback, resultsCallback(done))
         })
       })
 
+      after(() => { tests = [] })
+
       it('should 2 passing tests', function () {
         assert.equal(results.passingNum, 2)
       })
@@ -147,36 +177,29 @@ describe('testRunner', () => {
         assert.equal(results.failureNum, 1)
       })
 
-      it('should returns 3 messages', function () {
-        assert.deepEqual(tests, [
+      it('should return 4 messages', function () {
+        deepEqualExcluding(tests, [
+          { type: 'accountList', value: accounts },
           { type: 'contract', value: 'StringTest', filename: 'tests/examples_3/simple_string_test.sol' },
-          { type: 'testFailure', value: 'Value should be hello world', time: 1, context: 'StringTest', "errMsg": "initial value is not correct" },
-          { type: 'testPass', value: 'Value should not be hello wordl', time: 1, context: 'StringTest' },
-          { type: 'testPass', value: 'Initial value should be hello', time: 1, context: 'StringTest' },
-        ])
+          { type: 'testFailure', value: 'Value should be hello world', context: 'StringTest', "errMsg": "initial value is not correct" },
+          { type: 'testPass', value: 'Value should not be hello wordl', context: 'StringTest' },
+          { type: 'testPass', value: 'Initial value should be hello', context: 'StringTest' },
+        ], ['time'])
       })
     })
 
     //Test signed/unsigned integer weight
     describe('test number weight', function () {
       let filename = 'tests/number/number_test.sol'
-      let tests: any[] = [], results: ResultsInterface;
 
       before(function (done) {
         compileAndDeploy(filename, (_err, compilationData, contracts, accounts) => {
-          const testCallback: TestCbInterface = (err, test) => {
-            if (err) { throw err }
-            tests.push(test)
-          }
-          const resultsCallback: ResultCbInterface = (err, _results) => {
-            if (err) { throw err }
-            results = _results
-            done()
-          }
-          runTest('IntegerTest', contracts.IntegerTest, compilationData[filename]['IntegerTest'], { accounts }, testCallback, resultsCallback)
+          runTest('IntegerTest', contracts.IntegerTest, compilationData[filename]['IntegerTest'], { accounts }, testCallback, resultsCallback(done))
         })
       })
 
+      after(() => { tests = [] })
+
       it('should have 6 passing tests', function () {
         assert.equal(results.passingNum, 6)
       })
@@ -188,25 +211,15 @@ describe('testRunner', () => {
     // Test Transaction with different sender
     describe('various sender', function () {
       let filename = 'tests/various_sender/sender_test.sol'
-      let tests: any[] = [], results: ResultsInterface;
 
       before(function (done) {
         compileAndDeploy(filename, (_err, compilationData, contracts, accounts) => {
-          const testCallback: TestCbInterface = (err, test) => {
-            if (err) { throw err }
-            tests.push(test)
-          }
-          const resultsCallback: ResultCbInterface = (err, _results) => {
-            if (err) { throw err }
-            results = _results
-            done()
-          }
-
-          runTest('SenderTest', contracts.SenderTest, compilationData[filename]['SenderTest'], { accounts }, testCallback, resultsCallback)
-
+          runTest('SenderTest', contracts.SenderTest, compilationData[filename]['SenderTest'], { accounts }, testCallback, resultsCallback(done))
         })
       })
 
+      after(() => { tests = [] })
+
       it('should have 4 passing tests', function () {
         assert.equal(results.passingNum, 4)
       })