Merge branch 'master' into external-call-returndatasize

.github/workflows/build.yml

@@ -21,7 +21,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-20.04, macos-latest]
+        os: [ubuntu-latest, macos-latest]
 
     steps:
       - uses: actions/checkout@v4

.github/workflows/test.yml

@@ -60,6 +60,45 @@ jobs:
       - name: Run docs
         run: sphinx-build -E -b html docs dist/docs -n -q --color
 
+  # Symbolic tests
+  symbolic-tests:
+    runs-on: "ubuntu-latest"
+    name: symbolic-tests
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.11"
+        cache: "pip"
+
+    - name: Install dependencies
+      run: pip install .[test]
+
+    - name: Install hevm
+      run: |
+        # hevm dependency
+        sudo apt-get install z3
+
+        wget --no-verbose -O hevm https://github.com/ethereum/hevm/releases/download/release/0.54.2/hevm-x86_64-linux
+        chmod +x hevm
+        mkdir -p "$HOME/.local/bin/"
+        mv hevm "$HOME/.local/bin/"
+        echo "$HOME/.local/bin/" >> $GITHUB_PATH
+
+        hevm version
+
+    - name: Run tests
+      run: >
+        pytest
+        -m "hevm"
+        --hevm
+        --cov-config=setup.cfg
+        --cov=vyper
+        tests/
+ 
   # "Regular"/core tests.
   tests:
     runs-on: ${{ matrix.os || 'ubuntu' }}-latest
@@ -166,7 +205,7 @@ jobs:
     # summary result from test matrix.
     # see https://github.community/t/status-check-for-a-matrix-jobs/127354/7
     runs-on: ubuntu-latest
-    needs: [tests]
+    needs: [tests, symbolic-tests]
     steps:
       - name: Check tests tests all succeeded
         if: ${{ needs.tests.result != 'success' }}
@@ -271,20 +310,16 @@ jobs:
       run: |
         coverage xml
 
-    - name: Upload coverage sqlite artifact
+    - name: Upload coverage artifacts
       # upload coverage sqlite db for debugging
+      # upload coverage.xml artifact for downstream codecov upload action
       uses: actions/upload-artifact@v4
       with:
-        name: coverage-sqlite
+        name: coverage-artifacts
         include-hidden-files: true
-        path: .coverage
-        if-no-files-found: error
-
-    - name: Upload coverage.xml
-      uses: actions/upload-artifact@v4
-      with:
-        name: coverage-xml
-        path: coverage.xml
+        path: |
+          .coverage
+          coverage.xml
         if-no-files-found: error
 
   upload-coverage:
@@ -296,10 +331,10 @@ jobs:
     - uses: actions/checkout@v4
     - uses: actions/download-artifact@v4
       with:
-        name: coverage-xml
+        name: coverage-artifacts
 
     - name: Upload Coverage
       uses: codecov/codecov-action@v5
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
-        file: coverage.xml
+        files: coverage.xml

FUNDING.yml

@@ -1 +1 @@
-custom: https://etherscan.io/address/0x70CCBE10F980d80b7eBaab7D2E3A73e87D67B775
+custom: ["https://etherscan.io/address/0x70CCBE10F980d80b7eBaab7D2E3A73e87D67B775", "https://giveth.io/project/vyper"]

README.md

@@ -53,6 +53,10 @@ make dev-init
 ./quicktest.sh -m "not fuzzing"
 ```
 
+## Testing (with hevm)
+
+Install hevm by downloading it from the releases page (https://github.com/ethereum/hevm/releases/latest) and making sure it is in your PATH. hevm tests can be enabled with `--hevm` flag, and hevm tests can be selected with the `-m hevm` marker. For instance, `./quicktest.sh -m "hevm" --hevm`.
+
 ## Developing (working on the compiler)
 
 A useful script to have in your PATH is something like the following:
@@ -67,7 +71,9 @@ To run a python performance profile (to find compiler perf hotspots):
 PYTHONPATH=. python -m cProfile -s tottime vyper/cli/vyper_compile.py "$@"
 ```
 
-To get a call graph from a python profile, https://stackoverflow.com/a/23164271/ is helpful.
+To get a call graph from a python profile, pip install `gprof2dot` and `xdot`, and run it like `gprof2dot -f pstats stats | xdot -`. (See https://stackoverflow.com/a/23164271/).
+
+The utility timer functions `timeit`, `profileit` and `cumtimeit` are available in `vyper/utils.py`.
 
 
 # Contributing

setup.cfg

@@ -26,14 +26,15 @@ line_length = 100
 [tool:pytest]
 addopts = -n auto
 	--dist worksteal
+	--strict-markers
 python_files = test_*.py
 testpaths = tests
 xfail_strict = true
 markers =
 	fuzzing: Run Hypothesis fuzz test suite (deselect with '-m "not fuzzing"')
 	requires_evm_version(version): Mark tests that require at least a specific EVM version and would throw `EvmVersionException` otherwise
 	venom_xfail: mark a test case as a regression (expected to fail) under the venom pipeline
-
+	hevm: run tests marked for symbolic execution
 
 [coverage:run]
 branch = True

tests/conftest.py

@@ -7,6 +7,7 @@
 from eth_keys.datatypes import PrivateKey
 from hexbytes import HexBytes
 
+import tests.hevm
 import vyper.evm.opcodes as evm_opcodes
 from tests.evm_backends.base_env import BaseEnv, ExecutionReverted
 from tests.evm_backends.pyevm_env import PyEvmEnv
@@ -40,6 +41,7 @@ def pytest_addoption(parser):
     parser.addoption("--enable-compiler-debug-mode", action="store_true")
     parser.addoption("--experimental-codegen", action="store_true")
     parser.addoption("--tracing", action="store_true")
+    parser.addoption("--hevm", action="store_true")
 
     parser.addoption(
         "--evm-version",
@@ -114,6 +116,13 @@ def evm_version(pytestconfig):
     return pytestconfig.getoption("evm_version")
 
 
+@pytest.fixture(scope="session", autouse=True)
+def set_hevm(pytestconfig):
+    flag_value = pytestconfig.getoption("hevm")
+    assert isinstance(flag_value, bool)
+    tests.hevm.HAS_HEVM = flag_value
+
+
 @pytest.fixture(scope="session")
 def evm_backend(pytestconfig):
     backend_str = pytestconfig.getoption("evm_backend")

tests/functional/codegen/types/test_dynamic_array.py

@@ -13,6 +13,7 @@
     CompilerPanic,
     ImmutableViolation,
     OverflowException,
+    StackTooDeep,
     StateAccessViolation,
     StaticAssertionException,
     TypeMismatch,
@@ -290,6 +291,7 @@ def test_array(x: int128, y: int128, z: int128, w: int128) -> int128:
     assert c.test_array(2, 7, 1, 8) == -5454
 
 
+@pytest.mark.venom_xfail(raises=StackTooDeep, reason="stack scheduler regression")
 def test_four_d_array_accessor(get_contract):
     four_d_array_accessor = """
 @external

tests/functional/codegen/types/test_lists.py

@@ -7,7 +7,7 @@
 from tests.utils import check_precompile_asserts, decimal_to_int
 from vyper.compiler.settings import OptimizationLevel
 from vyper.evm.opcodes import version_check
-from vyper.exceptions import ArrayIndexException, OverflowException, TypeMismatch
+from vyper.exceptions import ArrayIndexException, OverflowException, StackTooDeep, TypeMismatch
 
 
 def _map_nested(f, xs):
@@ -193,6 +193,7 @@ def test_array(x: int128, y: int128, z: int128, w: int128) -> int128:
     assert c.test_array(2, 7, 1, 8) == -5454
 
 
+@pytest.mark.venom_xfail(raises=StackTooDeep, reason="stack scheduler regression")
 def test_four_d_array_accessor(get_contract):
     four_d_array_accessor = """
 @external

tests/hevm.py

@@ -0,0 +1,89 @@
+import subprocess
+
+from tests.venom_utils import parse_from_basic_block
+from vyper.ir.compile_ir import assembly_to_evm
+from vyper.venom import LowerDloadPass, StoreExpansionPass, VenomCompiler
+from vyper.venom.analysis import IRAnalysesCache
+from vyper.venom.basicblock import IRInstruction, IRLiteral
+
+HAS_HEVM: bool = False
+
+
+def _prep_hevm_venom(venom_source_code):
+    ctx = parse_from_basic_block(venom_source_code)
+
+    num_calldataloads = 0
+    for fn in ctx.functions.values():
+        for bb in fn.get_basic_blocks():
+            for inst in bb.instructions:
+                # transform `param` instructions into "symbolic" values for
+                # hevm via calldataload
+                if inst.opcode == "param":
+                    # hevm limit: 256 bytes of symbolic calldata
+                    assert num_calldataloads < 8
+
+                    inst.opcode = "calldataload"
+                    inst.operands = [IRLiteral(num_calldataloads * 32)]
+                    num_calldataloads += 1
+
+            term = bb.instructions[-1]
+            # test convention, terminate by `sink`ing the variables
+            # you want to check
+            if term.opcode != "sink":
+                continue
+
+            # testing convention: first 256 bytes can be symbolically filled
+            # with calldata
+            RETURN_START = 256
+
+            num_return_values = 0
+            for op in term.operands:
+                ptr = IRLiteral(RETURN_START + num_return_values * 32)
+                new_inst = IRInstruction("mstore", [op, ptr])
+                bb.insert_instruction(new_inst, index=-1)
+                num_return_values += 1
+
+            # return 0, 32 * num_variables
+            term.opcode = "return"
+            term.operands = [IRLiteral(num_return_values * 32), IRLiteral(RETURN_START)]
+
+        ac = IRAnalysesCache(fn)
+
+        # requirements for venom_to_assembly
+        LowerDloadPass(ac, fn).run_pass()
+        StoreExpansionPass(ac, fn).run_pass()
+
+    compiler = VenomCompiler([ctx])
+    return assembly_to_evm(compiler.generate_evm(no_optimize=True))[0].hex()
+
+
+def hevm_check_venom(pre, post, verbose=False):
+    global HAS_HEVM
+
+    if not HAS_HEVM:
+        return
+
+    # perform hevm equivalence check
+    if verbose:
+        print("HEVM COMPARE.")
+        print("BEFORE:", pre)
+        print("OPTIMIZED:", post)
+    bytecode1 = _prep_hevm_venom(pre)
+    bytecode2 = _prep_hevm_venom(post)
+
+    hevm_check_bytecode(bytecode1, bytecode2, verbose=verbose)
+
+
+def hevm_check_bytecode(bytecode1, bytecode2, verbose=False):
+    # debug:
+    if verbose:
+        print("RUN HEVM:")
+        print(bytecode1)
+        print(bytecode2)
+
+    subp_args = ["hevm", "equivalence", "--code-a", bytecode1, "--code-b", bytecode2]
+
+    if verbose:
+        subprocess.check_call(subp_args)
+    else:
+        subprocess.check_output(subp_args)