From 36fb6d8e6757666b8b2fdb062d006946d92c1c1e Mon Sep 17 00:00:00 2001 From: d1ll0n Date: Fri, 7 Jun 2019 02:17:10 -0500 Subject: [PATCH 1/8] fixed jump table destinations --- src/parser.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/parser.js b/src/parser.js index 6978c77f..1c1cc990 100644 --- a/src/parser.js +++ b/src/parser.js @@ -218,12 +218,13 @@ parser.processMacro = ( throw new Error(`expected to find ${tableInstance.label} in ${JSON.stringify(tableOffsets)}`); } const { offset } = tableInstance; - if (bytecode.slice((offset * 2) + 2, (offset * 2) + 6) !== 'xxxx') { - throw new Error(`expected ${tableInstance.offset} to be xxxx`); + const placeholder = bytecode.slice((offset * 2) + 2, (offset * 2) + 6); + if (placeholder !== 'xxxx') { + throw new Error(`expected ${placeholder} to be xxxx at offset ${tableInstance.offset}`); } const pre = bytecode.slice(0, (offset * 2) + 2); const post = bytecode.slice((offset * 2) + 6); - bytecode = `${pre}${formatEvenBytes(toHex(tableOffsets[tableInstance.label]))}${post}`; + bytecode = `${pre}${padNBytes(formatEvenBytes(toHex(tableOffsets[tableInstance.label])), 2)}${post}`; }); return { ...result, From f02afea5cd99b1ad1a9dc9eb36d060c10ca1c11c Mon Sep 17 00:00:00 2001 From: dillon Date: Mon, 23 Sep 2019 03:48:47 -0400 Subject: [PATCH 2/8] merged latest from upstream --- src/parser.js | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/parser.js b/src/parser.js index 2c5c4603..b6fa7da5 100644 --- a/src/parser.js +++ b/src/parser.js @@ -244,21 +244,12 @@ parser.processMacro = ( throw new Error(`expected to find ${tableInstance.label} in ${JSON.stringify(tableOffsets)}`); } const { offset } = tableInstance; -<<<<<<< HEAD - const placeholder = bytecode.slice((offset * 2) + 2, (offset * 2) + 6); - if (placeholder !== 'xxxx') { - throw new Error(`expected ${placeholder} to be xxxx at offset ${tableInstance.offset}`); - } - const pre = bytecode.slice(0, (offset * 2) + 2); - const post = bytecode.slice((offset * 2) + 6); -======= const placeholder = bytecode.slice(offset * 2 + 2, offset * 2 + 6); if (placeholder !== 'xxxx') { throw new Error(`expected ${placeholder} to be xxxx at offset ${tableInstance.offset}`); } const pre = bytecode.slice(0, offset * 2 + 2); const post = bytecode.slice(offset * 2 + 6); ->>>>>>> upstream/master bytecode = `${pre}${padNBytes(formatEvenBytes(toHex(tableOffsets[tableInstance.label])), 2)}${post}`; }); return { From 6610ccea21b08ffc953817b609fc31e3e8766141 Mon Sep 17 00:00:00 2001 From: dillon Date: Mon, 23 Sep 2019 03:50:40 -0400 Subject: [PATCH 3/8] added parser for sources object --- src/parser.js | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/src/parser.js b/src/parser.js index b6fa7da5..5ab30de2 100644 --- a/src/parser.js +++ b/src/parser.js @@ -750,4 +750,53 @@ parser.compileMacro = (macroName, filename, partialPath) => { return { bytecode, sourcemap }; }; +parser.getSourcesFileContents = (sources, entryFile) => { + const included = {}; + const parentPathOf = (str) => (m = /(.*)\/(?!\/)(.*)\.huff/g.exec(str)) && m[1]; + const recurse = (filename, parentPath) => { + const relativePath = path.join(parentPath || '', filename); + const fileString = sources[relativePath]; + let formatted = parser.removeComments(fileString); + let imported = []; + let test = formatted.match(grammar.topLevel.IMPORT); + while (test !== null) { + const importStatement = formatted.match(grammar.topLevel.IMPORT); + const empty = ' '.repeat(importStatement[0].length); + formatted = empty + formatted.slice(importStatement[0].length); + const importedFilename = importStatement[1]; + if (!included[importedFilename]) { + imported = [...imported, ...recurse(importedFilename, parentPathOf(relativePath))]; + included[importedFilename] = true; + } + test = formatted.match(grammar.topLevel.IMPORT); + } + const result = [ + ...imported, + { + filename: path.join('', filename), + data: formatted, + }, + ]; + return result; + }; + const filedata = recurse(entryFile); + const raw = filedata.reduce((acc, { data }) => { + return acc + data; + }, ''); + return { filedata, raw }; +}; + +/** + * @param source - can be either a filename or a sources object + * @param _path - if a sources object is given, path should be the entry file; otherwise, path should be the parent path +*/ +parser.parse = (source, _path) => { + const { filedata, raw } = (typeof source == 'string') + ? parser.getFileContents(source, _path) + : parser.getSourcesFileContents(source, _path) + const map = inputMaps.createInputMap(filedata); + const { macros, jumptables } = parser.parseTopLevel(raw, 0, map); + return { inputMap: map, macros, jumptables }; +}; + module.exports = parser; From ccac7908de71912e581af62b147d1388536ef4d3 Mon Sep 17 00:00:00 2001 From: dillon Date: Mon, 23 Sep 2019 04:24:08 -0400 Subject: [PATCH 4/8] added test for compiling from sources object --- src/parser.spec.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/parser.spec.js b/src/parser.spec.js index 8d550f9b..68dda38a 100644 --- a/src/parser.spec.js +++ b/src/parser.spec.js @@ -306,6 +306,12 @@ describe('parser tests', () => { expect(result.sourcemap.length).to.equal(result.bytecode.length / 2); }); + it(`can compile macro in sources object`, () => { + const sources = require(path.join(pathToTestData, 'sources.json')) + const result = parser.compileMacro('ADD_ONE', sources, 'd.huff'); + expect(result.bytecode).to.equal('600101') + }) + it('frozen version of MAIN_TWO_ENDO_MOD macro is correctly compiled', () => { const result = parser.compileMacro('MAIN_TWO_ENDO_MOD', './main_loop.huff', pathToTestData); const expected = fs.readFileSync(path.posix.resolve(pathToTestData, 'compiled.txt'), 'utf8'); From a34834ee5846bd9b6d59a50f46d6dca6a051f0d7 Mon Sep 17 00:00:00 2001 From: dillon Date: Mon, 23 Sep 2019 04:24:49 -0400 Subject: [PATCH 5/8] changed package version to 0.0.2 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index ba8e6859..7865a70c 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@aztec/huff", "description": "Compiler for Huff, a DSL for low-level Ethereum smart contract programming", - "version": "0.0.1", + "version": "0.0.2", "author": "AZTEC", "bugs": { "url": "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/AztecProtocol/huff/issues" From c11c5bf50248f466a6d13dd67871ccb54e711f0d Mon Sep 17 00:00:00 2001 From: dillon Date: Mon, 23 Sep 2019 04:50:25 -0400 Subject: [PATCH 6/8] feat: add solidity-style sources compilation --- src/parser.js | 62 ++++++++++++++++++++++++++--------------- src/parser.spec.js | 9 ++++-- src/runtime.js | 2 +- src/runtime.spec.js | 47 +++++++++++++++++++++++++++++++ testData/add-three.huff | 1 + testData/sources.json | 8 ++++++ 6 files changed, 104 insertions(+), 25 deletions(-) create mode 100644 testData/add-three.huff create mode 100644 testData/sources.json diff --git a/src/parser.js b/src/parser.js index 5ab30de2..2d160d6e 100644 --- a/src/parser.js +++ b/src/parser.js @@ -734,28 +734,21 @@ parser.getFileContents = (originalFilename, partialPath) => { return { filedata, raw }; }; -parser.parseFile = (filename, partialPath) => { - const { filedata, raw } = parser.getFileContents(filename, partialPath); - const map = inputMaps.createInputMap(filedata); - const { macros, jumptables } = parser.parseTopLevel(raw, 0, map); - return { inputMap: map, macros, jumptables }; -}; - -parser.compileMacro = (macroName, filename, partialPath) => { - const { filedata, raw } = parser.getFileContents(filename, partialPath); - const map = inputMaps.createInputMap(filedata); - const { macros, jumptables } = parser.parseTopLevel(raw, 0, map); - const { data: { bytecode, sourcemap } } = parser.processMacro(macroName, 0, [], macros, map, jumptables); // prettier-ignore - - return { bytecode, sourcemap }; -}; - +const normalizePath = (str) => path.join('', str); parser.getSourcesFileContents = (sources, entryFile) => { const included = {}; const parentPathOf = (str) => (m = /(.*)\/(?!\/)(.*)\.huff/g.exec(str)) && m[1]; const recurse = (filename, parentPath) => { - const relativePath = path.join(parentPath || '', filename); - const fileString = sources[relativePath]; + const fileName = normalizePath(filename) + const relativePath = path.join(parentPath || '', fileName); + let fileString = sources[relativePath]; + if (!fileString) { + if (!fs.existsSync(relativePath)) { + throw new Error(`Import not found: ${fileName} ${(parentPath && parentPath !== '.') ? `in ${parentPath}` : ''}`) + } + fileString = fs.readFileSync(relativePath, 'utf8'); + console.log(`Warning: File ${relativePath} does not exist in sources, found in filesystem.`) + } let formatted = parser.removeComments(fileString); let imported = []; let test = formatted.match(grammar.topLevel.IMPORT); @@ -773,7 +766,7 @@ parser.getSourcesFileContents = (sources, entryFile) => { const result = [ ...imported, { - filename: path.join('', filename), + filename: fileName, data: formatted, }, ]; @@ -786,14 +779,39 @@ parser.getSourcesFileContents = (sources, entryFile) => { return { filedata, raw }; }; +parser.getContents = (source, _path) => { + if (typeof source == 'string') { + return parser.getFileContents(source, _path); + } else { + const sources = Object.keys(source).reduce((obj, key) => ({ + ...obj, + [normalizePath(key)]: source[key] + }), {}); + return parser.getSourcesFileContents(sources, _path) + } +} + +parser.parseFile = (filename, partialPath) => { + const { filedata, raw } = parser.getFileContents(filename, partialPath); + const map = inputMaps.createInputMap(filedata); + const { macros, jumptables } = parser.parseTopLevel(raw, 0, map); + return { inputMap: map, macros, jumptables }; +}; + +parser.compileMacro = (macroName, source, _path) => { + const { filedata, raw } = parser.getContents(source, _path) + const map = inputMaps.createInputMap(filedata); + const { macros, jumptables } = parser.parseTopLevel(raw, 0, map); + const { data: { bytecode, sourcemap } } = parser.processMacro(macroName, 0, [], macros, map, jumptables); // prettier-ignore + return { bytecode, sourcemap }; +}; + /** * @param source - can be either a filename or a sources object * @param _path - if a sources object is given, path should be the entry file; otherwise, path should be the parent path */ parser.parse = (source, _path) => { - const { filedata, raw } = (typeof source == 'string') - ? parser.getFileContents(source, _path) - : parser.getSourcesFileContents(source, _path) + const { filedata, raw } = parser.getContents(source, _path) const map = inputMaps.createInputMap(filedata); const { macros, jumptables } = parser.parseTopLevel(raw, 0, map); return { inputMap: map, macros, jumptables }; diff --git a/src/parser.spec.js b/src/parser.spec.js index 68dda38a..63a8630c 100644 --- a/src/parser.spec.js +++ b/src/parser.spec.js @@ -12,6 +12,7 @@ const { opcodes } = require('./opcodes/opcodes'); const { expect } = chai; const pathToTestData = path.posix.resolve(__dirname, '../testData'); +const sources = require(path.join(pathToTestData, 'sources.json')); describe('parser tests', () => { describe('templates', () => { @@ -307,11 +308,15 @@ describe('parser tests', () => { }); it(`can compile macro in sources object`, () => { - const sources = require(path.join(pathToTestData, 'sources.json')) - const result = parser.compileMacro('ADD_ONE', sources, 'd.huff'); + const result = parser.compileMacro('ADD_ONE', sources, 'test.huff'); expect(result.bytecode).to.equal('600101') }) + it('can load file missing from sources object', () => { + const result = parser.compileMacro('ADD_THREE', sources, 'testData/external.huff'); + expect(result.bytecode).to.equal('600301') + }) + it('frozen version of MAIN_TWO_ENDO_MOD macro is correctly compiled', () => { const result = parser.compileMacro('MAIN_TWO_ENDO_MOD', './main_loop.huff', pathToTestData); const expected = fs.readFileSync(path.posix.resolve(pathToTestData, 'compiled.txt'), 'utf8'); diff --git a/src/runtime.js b/src/runtime.js index e390fb16..780f8440 100644 --- a/src/runtime.js +++ b/src/runtime.js @@ -95,7 +95,7 @@ function runCode(vm, bytecode, calldata, sourcemapOffset = 0, sourcemap = [], ca } function Runtime(filename, path, debug = false) { - const { inputMap, macros, jumptables } = newParser.parseFile(filename, path); + const { inputMap, macros, jumptables } = newParser.parse(filename, path); return async function runMacro(vm, macroName, stack = [], memory = [], calldata = null, callvalue = 0, callerAddr = 0) { const memoryCode = encodeMemory(memory); const stackCode = encodeStack(stack); diff --git a/src/runtime.spec.js b/src/runtime.spec.js index 6d6c99d7..9e519abf 100644 --- a/src/runtime.spec.js +++ b/src/runtime.spec.js @@ -1,6 +1,7 @@ const chai = require('chai'); const BN = require('bn.js'); const path = require('path'); +const fs = require('fs'); const { Runtime, getNewVM } = require('./runtime.js'); const vm = getNewVM(); @@ -64,3 +65,49 @@ describe('runtime tests using double algorithm', () => { expect(stack[2].eq(expected[2])).to.equal(true); }); }); + +const sources = { + 'test.huff': testHelper, + 'double.huff': fs.readFileSync(path.join(pathToTestData, 'double.huff'), 'utf8'), + 'constants.huff': fs.readFileSync(path.join(pathToTestData, 'constants.huff'), 'utf8') +}; + +describe('sources object', () => { + let double; + let inputStack; + let expected; + before(() => { + double = new Runtime(sources, 'test.huff'); + inputStack = [ + new BN('30644e72e131a029b85045b68181585d97816a916871ca8d3c208c16d87cfd47', 16), + new BN('bed36146b88c48f014c6b1ac1c8441ac0291e66462c432021914cc887dd0d75', 16), + new BN('14f5a63645c1cd318e0720ac3ae1f3c164462b5a96b288ce6f45b62e2fb220ab', 16), + new BN('29185780978972d99ce22b0543d90143dacc611fff75d4705b6ac867892ee2d8', 16), + ]; + }); + it('macro DOUBLE correctly calculates point doubling', async () => { + const { stack } = await double(vm, 'DOUBLE', inputStack); + expected = [ + new BN('30644e72e131a029b85045b68181585d97816a916871ca8d3c208c16d87cfd47', 16), + new BN('55f8cabad8ae94c14c1482e3e20f7ce889e3143949181f404b04da8df02029ba', 16), + new BN('43662fe9d0effbc8820577509b7454a46758301a4eedd592e61e6e357f4f4667', 16), + new BN('f8caadab040dc22bcd17b3dd99820ba0f9fc34582ab692cbca1f914c48cbad8', 16), + ]; + expect(stack[0].eq(expected[0])).to.equal(true); + expect(stack[1].eq(expected[1])).to.equal(true); + expect(stack[2].eq(expected[2])).to.equal(true); + expect(stack[3].eq(expected[3])).to.equal(true); + }); + + it('macro DOUBLE__MAIN correctly calculates point doubling (inverted y)', async () => { + const { stack } = await double(vm, 'DOUBLE__MAIN_IMPL', inputStack.slice(1)); + expected = [ + new BN('55f8cabad8ae94c14c1482e3e20f7ce889e3143949181f404b04da8df02029ba', 16), + new BN('4dc6bb6ed2a4e4b4a6eb59d2e90fb4745f2c0f99ea678a14ce43360f0a27b16e', 16), + new BN('20d7a39830f0c406fb7eca78a7e937a387e1a74be5c661607f7e930213f0426f', 16), + ]; + expect(stack[0].eq(expected[0])).to.equal(true); + expect(stack[1].eq(expected[1])).to.equal(true); + expect(stack[2].eq(expected[2])).to.equal(true); + }); +}); diff --git a/testData/add-three.huff b/testData/add-three.huff new file mode 100644 index 00000000..333e70c7 --- /dev/null +++ b/testData/add-three.huff @@ -0,0 +1 @@ +#define macro ADD_THREE = takes(0) returns(1) { 0x03 add } \ No newline at end of file diff --git a/testData/sources.json b/testData/sources.json new file mode 100644 index 00000000..9b9fcae0 --- /dev/null +++ b/testData/sources.json @@ -0,0 +1,8 @@ +{ + "a.huff": "#include 'b.huff'", + "b.huff": "#include './dir/c.huff'", + "dir/c.huff": "#include '../d.huff'", + "./dir/x.huff": "#include './y.huff'", + "dir/y.huff": "#define macro ADD_TWO = takes(0) returns(1) { 0x02 add }", + "d.huff": "#define macro ADD_ONE = takes(0) returns(1) { 0x01 add }" +} \ No newline at end of file From 65af11d4596ff4e93dae4f712fa83c20b2c6812a Mon Sep 17 00:00:00 2001 From: dillon Date: Mon, 23 Sep 2019 04:50:51 -0400 Subject: [PATCH 7/8] feat: add solidity-style sources compilation --- testData/sources.json | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/testData/sources.json b/testData/sources.json index 9b9fcae0..e923926b 100644 --- a/testData/sources.json +++ b/testData/sources.json @@ -1,8 +1,6 @@ { - "a.huff": "#include 'b.huff'", - "b.huff": "#include './dir/c.huff'", - "dir/c.huff": "#include '../d.huff'", - "./dir/x.huff": "#include './y.huff'", - "dir/y.huff": "#define macro ADD_TWO = takes(0) returns(1) { 0x02 add }", - "d.huff": "#define macro ADD_ONE = takes(0) returns(1) { 0x01 add }" + "testData/external.huff": "#include './add-three.huff'", + "add/two.huff": "#define macro ADD_TWO = takes(0) returns(1) { 0x02 add }", + "add/one.huff": "#define macro ADD_ONE = takes(0) returns(1) { 0x01 add }", + "test.huff": "#include './add/one.huff'" } \ No newline at end of file From 43dda4fc731cb29748b9b075d59c03848fa4dc9c Mon Sep 17 00:00:00 2001 From: dillon Date: Mon, 23 Sep 2019 05:02:49 -0400 Subject: [PATCH 8/8] syntax fixes --- src/parser.js | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/parser.js b/src/parser.js index 2d160d6e..76675968 100644 --- a/src/parser.js +++ b/src/parser.js @@ -782,14 +782,13 @@ parser.getSourcesFileContents = (sources, entryFile) => { parser.getContents = (source, _path) => { if (typeof source == 'string') { return parser.getFileContents(source, _path); - } else { - const sources = Object.keys(source).reduce((obj, key) => ({ - ...obj, - [normalizePath(key)]: source[key] - }), {}); - return parser.getSourcesFileContents(sources, _path) } -} + const sources = Object.keys(source).reduce((obj, key) => ({ + ...obj, + [normalizePath(key)]: source[key], + }), {}); + return parser.getSourcesFileContents(sources, _path); +}; parser.parseFile = (filename, partialPath) => { const { filedata, raw } = parser.getFileContents(filename, partialPath);