rebench.conf

# -*- mode: yaml -*-
# Config file for ReBench
default_experiment: benchmarks
default_data_file: 'rebench.data'

reporting:
    # Benchmark results will be reported to ReBenchDB
    rebenchdb:
        # this url needs to point to the API endpoint
        db_url: https://rebench.stefan-marr.de/rebenchdb
        repo_url: https://github.com/smarr/TruffleSOM
        record_all: true # make sure everything is recorded
        project_name: TruffleSOM

runs:
    max_invocation_time: 6000
    min_iteration_time: 1

benchmark_suites:
    macro-startup:
        gauge_adapter: RebenchLog
        command: &MACRO_CMD "-cp Smalltalk:Examples/Benchmarks/Richards:Examples/Benchmarks/DeltaBlue:Examples/Benchmarks/NBody:Examples/Benchmarks/Json:Examples/Benchmarks/GraphSearch Examples/Benchmarks/BenchmarkHarness.som --gc %(benchmark)s %(iterations)s "
        iterations: 1
        invocations: 5
        benchmarks:
            - Richards:     {extra_args:    1, tags: [yuria4]}
            - DeltaBlue:    {extra_args:  100, tags: [yuria4]}
            - NBody:        {extra_args: 1000, tags: [yuria3]}
            - Json:         {extra_args:    1, tags: [yuria4]}
            - GraphSearch:  {extra_args:    7, tags: [yuria2]}
            - PageRank:     {extra_args:   75, tags: [yuria3]}

    macro-steady:
        gauge_adapter: RebenchLog
        command: *MACRO_CMD
        benchmarks:
            - Richards:     {extra_args: 40,     warmup:  30,   iterations: 130, tags: [yuria4]}
            - DeltaBlue:    {extra_args: 10000,  warmup:  20,   iterations: 120, tags: [yuria4]}
            - NBody:        {extra_args: 200000, warmup:  20,   iterations: 120, tags: [yuria3]}
            - Json:         {extra_args: 80,     warmup:  20,   iterations: 120, tags: [yuria4]}
            - GraphSearch:  {extra_args: 25,     warmup: 100,   iterations: 250, tags: [yuria4]}
            - PageRank:     {extra_args: 1000,   warmup:  20,   iterations: 120, tags: [yuria3]}

    awfy-startup:
        gauge_adapter: RebenchLog
        command: &AWFY_CMD "-cp Examples/AreWeFastYet/CD:Examples/AreWeFastYet/Havlak:Examples/AreWeFastYet/Core:Smalltalk Examples/AreWeFastYet/Harness.som --gc %(benchmark)s %(iterations)s "
        iterations: 1
        invocations: 5
        benchmarks:
            - CD:     {extra_args: 10, tags: [yuria4]}
            - Havlak: {extra_args: 15, tags: [yuria4]}

    awfy-steady:
        gauge_adapter: RebenchLog
        command: *AWFY_CMD
        benchmarks:
            - CD:     {extra_args: 100, warmup: 30, iterations: 130, tags: [yuria4]}
            - Havlak: {extra_args: 150, warmup: 30, iterations: 130, tags: [yuria3]}

    micro-startup:
        gauge_adapter: RebenchLog
        command: &MICRO_CMD "-cp Smalltalk:Examples/Benchmarks/LanguageFeatures:Examples/Benchmarks/TestSuite Examples/Benchmarks/BenchmarkHarness.som --gc %(benchmark)s %(iterations)s "
        iterations: 1
        invocations: 5
        benchmarks:
            - Fannkuch:     {extra_args:   7, tags: [yuria4]}
            - Fibonacci:    {extra_args:  10, tags: [yuria2]}
            - Dispatch:     {extra_args:  10, tags: [yuria3]}
            - Bounce:       {extra_args:  10, tags: [yuria4]}
            - Loop:         {extra_args: 100, tags: [yuria2]}
            - Permute:      {extra_args:  10, tags: [yuria3]}
            - Queens:       {extra_args:  10, tags: [yuria4]}
            - List:         {extra_args:   2, tags: [yuria2]}
            - Recurse:      {extra_args:  12, tags: [yuria3]}
            - Storage:      {extra_args:   8, tags: [yuria4]}
            - Sieve:        {extra_args:  20, tags: [yuria2]}
            - BubbleSort:   {extra_args:  15, tags: [yuria3]}
            - QuickSort:    {extra_args:  15, tags: [yuria ]}
            - Sum:          {extra_args:  40, tags: [yuria2]}
            - Towers:       {extra_args:   2, tags: [yuria3]}
            - TreeSort:     {extra_args:   7, tags: [yuria ]}
            - IntegerLoop:  {extra_args:   7, tags: [yuria2]}
            - FieldLoop:    {extra_args:   1, tags: [yuria3]}
            - WhileLoop:    {extra_args:  30, tags: [yuria ]}
            - Mandelbrot:   {extra_args:  50, tags: [yuria2]}
            - IfNil:        {extra_args:  30, tags: [yuria ]}

            - Test:     {invocations: 10, tags: [yuria ]}
            - TestGC:   {invocations: 10, extra_args: 10, tags: [yuria ]}

    micro-steady:
        gauge_adapter: RebenchLog
        command: *MICRO_CMD
        benchmarks:
            - Fannkuch:     {extra_args: 9,      warmup:   5,   iterations:  55, tags: [yuria4]}
            - Fibonacci:    {extra_args: 1000,   warmup:  10,   iterations:  60, tags: [yuria4]}
            - Dispatch:     {extra_args: 10000,  warmup:   5,   iterations:  55, tags: [yuria4]}
            - Bounce:       {extra_args: 4000,   warmup:  10,   iterations:  60, tags: [yuria4]}
            - Loop:         {extra_args: 10000,  warmup:   5,   iterations:  55, tags: [yuria2]}
            - Permute:      {extra_args: 1500,   warmup:   5,   iterations:  55, tags: [yuria4]}
            - Queens:       {extra_args: 1000,   warmup:   5,   iterations:  55, tags: [yuria4]}
            - List:         {extra_args: 1000,   warmup:  15,   iterations:  65, tags: [yuria4]}
            - Recurse:      {extra_args: 2000,   warmup:  15,   iterations:  65, tags: [yuria4]}
            - Storage:      {extra_args: 1000,   warmup:  10,   iterations:  60, tags: [yuria4]}
            - Sieve:        {extra_args: 2500,   warmup:  10,   iterations:  60, tags: [yuria2]}
            - BubbleSort:   {extra_args: 3000,   warmup:   5,   iterations:  55, tags: [yuria4]}
            - QuickSort:    {extra_args: 2000,   warmup:   5,   iterations:  55, tags: [yuria4]}
            - Sum:          {extra_args: 10000,  warmup:   5,   iterations:  55, tags: [yuria2]}
            - Towers:       {extra_args: 1000,   warmup:   5,   iterations:  55, tags: [yuria4]}
            - TreeSort:     {extra_args: 1000,   warmup:  10,   iterations:  60, tags: [yuria4]}
            - IntegerLoop:  {extra_args: 8000,   warmup:   5,   iterations:  55, tags: [yuria2]}
            - FieldLoop:    {extra_args: 900,    warmup:   5,   iterations:  55, tags: [yuria4]}
            - WhileLoop:    {extra_args: 9000,   warmup:   5,   iterations:  55, tags: [yuria2]}
            - Mandelbrot:   {extra_args: 1000,   warmup:  10,   iterations: 110, tags: [yuria4]}
            - IfNil:        {extra_args: 9000,   warmup:   5,   iterations:  55, tags: [yuria2]}

    micro-somsom:
        gauge_adapter: RebenchLog
        command: "-cp Smalltalk:Examples/Benchmarks/LanguageFeatures Examples/Benchmarks/BenchmarkHarness.som --gc %(benchmark)s %(iterations)s "
        iterations: 1
        benchmarks:
            - Loop:         {extra_args: 1, tags: [yuria4]}
            - Queens:       {extra_args: 1, tags: [yuria4]}
            - List:         {extra_args: 1, tags: [yuria2]}
            - Recurse:      {extra_args: 1, tags: [yuria4]}
            - Mandelbrot:   {extra_args: 3, tags: [yuria4]}

    som-parse:
        gauge_adapter: RebenchLog
        command: "-cp Smalltalk:Examples:Examples/Benchmarks/DeltaBlue:Examples/Benchmarks/GraphSearch:Examples/Benchmarks/Json:Examples/Benchmarks/NBody:TestSuite:core-lib/SomSom/tests:core-lib/SomSom/src/vmobjects:core-lib/SomSom/src/primitives:core-lib/SomSom/src/compiler  Examples/Benchmarks/BenchmarkHarness.som --gc %(benchmark)s %(iterations)s "
        iterations: 1!
        invocations: 10
        benchmarks:
            - SomParse: {extra_args: 1, tags: [yuria2]}
            - SomInit:  {extra_args: 10000, tags: [yuria2]}

    interpreter:
        description: Basic interpreter benchmarks for comparing performance of most basic concepts.
        gauge_adapter: RebenchLog
        invocations: 5
        command: "-cp Smalltalk:Examples/Benchmarks/Interpreter Examples/Benchmarks/BenchmarkHarness.som %(benchmark)s %(iterations)s 1"
        benchmarks:
            - ArgRead:                           {tags: [yuria ]}
            - ArrayReadConst:                    {tags: [yuria ]}
            - ArrayWriteConstConst:              {tags: [yuria ]}
            - BlockSend0ConstReturn:             {tags: [yuria ]}
            - Const:                             {tags: [yuria ]}
            - FieldConstWrite:                   {tags: [yuria ]}
            - FieldRead:                         {tags: [yuria ]}
            - FieldReadIncWrite:                 {tags: [yuria ]}
            - FieldReadWrite:                    {tags: [yuria ]}
            - GlobalRead:                        {tags: [yuria ]}
            - LocalConstWrite:                   {tags: [yuria ]}
            - LocalRead:                         {tags: [yuria ]}
            - LocalReadIncWrite:                 {tags: [yuria ]}
            - LocalReadWrite:                    {tags: [yuria ]}
            - SelfSend0:                         {tags: [yuria ]}
            - SelfSend0BlockConstNonLocalReturn: {tags: [yuria ]}

executors:
    TruffleSOM-interp:
        path: .
        executable: som
        args: " -G "
    TruffleSOM-graal:
        path: .
        executable: som

    TruffleSOM-interp-bc:
        path: .
        executable: som
        args: " -G -Dsom.interp=BC "
    TruffleSOM-graal-bc:
        path: .
        executable: som
        args: " -Dsom.interp=BC "

    TruffleSOM-native:
        path: .
        executable: som-native-ast
    TruffleSOM-native-bc:
        path: .
        executable: som-native-bc

    TruffleSOM-native-interp-ast:
        path: .
        executable: som-native-interp-ast
        profiler:
          perf: {}

    TruffleSOM-native-interp-bc:
        path: .
        executable: som-native-interp-bc
        profiler:
          perf: {}

    TruffleSOM-native-ast-ee:
        path: .
        executable: som-native-ast-ee
    TruffleSOM-native-bc-ee:
        path: .
        executable: som-native-bc-ee

    TruffleSOM-native-interp-ast-ee:
        path: .
        executable: som-native-interp-ast-ee
        profiler:
          perf: {}

    TruffleSOM-native-interp-bc-ee:
        path: .
        executable: som-native-interp-bc-ee
        profiler:
          perf: {}

    SomSom-native-interp-ast:
        path: .
        executable: som-native-interp-ast
        args: "-cp core-lib/Smalltalk:core-lib/SomSom/src/compiler:core-lib/SomSom/src/vm:core-lib/SomSom/src/vmobjects:core-lib/SomSom/src/interpreter:core-lib/SomSom/src/primitives core-lib/SomSom/src/vm/MainLoadAll.som"
        profiler:
          perf: {}

    SomSom-native-interp-bc:
        path: .
        executable: som-native-interp-bc
        args: "-Dsom.interp=BC -cp core-lib/Smalltalk:core-lib/SomSom/src/compiler:core-lib/SomSom/src/vm:core-lib/SomSom/src/vmobjects:core-lib/SomSom/src/interpreter:core-lib/SomSom/src/primitives core-lib/SomSom/src/vm/MainLoadAll.som"
        profiler:
          perf: {}

    SomSom-native-interp-ast-ee:
        path: .
        executable: som-native-interp-ast-ee
        args: "-cp core-lib/Smalltalk:core-lib/SomSom/src/compiler:core-lib/SomSom/src/vm:core-lib/SomSom/src/vmobjects:core-lib/SomSom/src/interpreter:core-lib/SomSom/src/primitives core-lib/SomSom/src/vm/MainLoadAll.som"
        profiler:
          perf: {}

    SomSom-native-interp-bc-ee:
        path: .
        executable: som-native-interp-bc-ee
        args: "-Dsom.interp=BC -cp core-lib/Smalltalk:core-lib/SomSom/src/compiler:core-lib/SomSom/src/vm:core-lib/SomSom/src/vmobjects:core-lib/SomSom/src/interpreter:core-lib/SomSom/src/primitives core-lib/SomSom/src/vm/MainLoadAll.som"
        profiler:
          perf: {}

# define the benchmarks to be executed for a re-executable benchmark run
experiments:
    benchmarks:
        description: All benchmarks
        executions:
            # - TruffleSOM-interp:
            #     suites:
            #         - micro-startup
            #         - macro-startup
            # - TruffleSOM-interp-bc:
            #     suites:
            #         - micro-startup
            #         - macro-startup
            - TruffleSOM-graal:
                suites:
                    # - micro-startup
                    - micro-steady
                    # - macro-startup
                    - macro-steady
                    - awfy-steady
            - TruffleSOM-graal-bc:
                suites:
                    # - micro-startup
                    - micro-steady
                    # - macro-startup
                    - macro-steady
                    - awfy-steady
            - TruffleSOM-native-interp-ast:
                suites:
                    - micro-startup
                    - macro-startup
                    - awfy-startup
                    - som-parse
                    - interpreter
            - TruffleSOM-native-interp-bc:
                suites:
                    - micro-startup
                    - macro-startup
                    - awfy-startup
                    - som-parse
                    - interpreter
            - SomSom-native-interp-ast:
                suites:
                  - micro-somsom
            - SomSom-native-interp-bc:
                suites:
                  - micro-somsom

            - TruffleSOM-native-interp-ast-ee:
                suites:
                  - micro-startup
                  - macro-startup
                  - awfy-startup
                  - som-parse
                  - interpreter
            - TruffleSOM-native-interp-bc-ee:
                suites:
                  - micro-startup
                  - macro-startup
                  - awfy-startup
                  - som-parse
                  - interpreter
            - SomSom-native-interp-ast-ee:
                suites:
                  - micro-somsom
            - SomSom-native-interp-bc-ee:
                suites:
                  - micro-somsom

    profiling:
      description: Profile Native Image Interpreters
      action: profile
      executions:
        - TruffleSOM-native-interp-ast:
            iterations: 10!
            invocations: 1!
            suites:
              - micro-startup
              - macro-startup
              - awfy-startup
              - som-parse
        - TruffleSOM-native-interp-bc:
            iterations: 10!
            invocations: 1!
            suites:
              - micro-startup
              - macro-startup
              - awfy-startup
              - som-parse
        - SomSom-native-interp-ast:
            iterations: 1!
            invocations: 1!
            suites:
              - micro-somsom
        - SomSom-native-interp-bc:
            iterations: 1!
            invocations: 1!
            suites:
              - micro-somsom