tfhe-rs/.github/workflows/benchmark_cpu.yml at a976976996f88855fcec81ca31fb35658e4ff49f · zama-ai/tfhe-rs · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
# Run benchmarks on an AWS instance and return parsed results to Slab CI bot.
name: benchmark_cpu

run-name: ${{ inputs.command }}::${{ inputs.bench_type}} (${{ inputs.op_flavor }}, ${{ inputs.precisions_set }}, ${{ inputs.params_type }})

on:
  workflow_dispatch:
    inputs:
      profile:
        description: "Instance type"
        required: true
        type: choice
        default: "aws::bench-hpc8 (hpc8a.96xlarge)"
        options:
          - "aws::bench (hpc7a.96xlarge)"
          - "aws::bench-hpc8 (hpc8a.96xlarge)"
      command:
        description: "Benchmark command to run"
        type: choice
        options:
          - integer
          - signed_integer
          - integer_compression
          - integer_zk
          - msm_zk
          - shortint
          - shortint_oprf
          - hlapi_unsigned
          - hlapi_signed
          - hlapi_erc7984
          - hlapi_dex
          - hlapi_noise_squash
          - hlapi_kvstore
          - tfhe_zk_pok
          - boolean
          - pbs
          - pbs128
          - ks
          - ks_pbs
      op_flavor:
        description: "Operations set to run"
        type: choice
        default: default
        options:
          - default
          - fast_default
          - smart
          - unchecked
          - misc
      precisions_set:
        description: "Bit precisions set"
        type: choice
        default: fast
        options:
          - fast
          - all
          - documentation
      bench_type:
        description: "Benchmarks type"
        type: choice
        default: latency
        options:
          - latency
          - throughput
          - both
      params_type:
        description: "Parameters type"
        type: choice
        default: classical
        options:
          - classical
          - multi_bit
          - classical + multi_bit
          - classical_documentation
          - multi_bit_documentation
          - classical_documentation + multi_bit_documentation

permissions: {}

# zizmor: ignore[concurrency-limits] only Zama organization members can trigger this workflow

jobs:
  parse-inputs:
    name: benchmark_cpu/parse-inputs
    runs-on: ubuntu-latest
    outputs:
      backend: ${{ steps.parse_profile_and_hardware_name.outputs.backend }}
      profile: ${{ steps.parse_profile_and_hardware_name.outputs.profile }}
      hardware_name: ${{ steps.parse_profile_and_hardware_name.outputs.hardware }}
      additional_file_to_parse: ${{ steps.set_file_to_parse.outputs.additional_file_to_parse }}
    steps:
      - name: Checkout tfhe-rs repo
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
        with:
          persist-credentials: 'false'
          token: ${{ secrets.REPO_CHECKOUT_TOKEN }}

      - name: Parse profile and hardware name
        id: parse_profile_and_hardware_name
        env:
          INPUTS_PROFILE: ${{ inputs.profile }}
        run: python3 ci/parse_benchmark_profile.py "$INPUTS_PROFILE" "$GITHUB_OUTPUT"

      - name: Get additional file to parse
        id: set_file_to_parse
        shell: python
        env:
          INPUTS_COMMAND: ${{ inputs.command }}
        run: |
          import os

          inputs_command = os.environ["INPUTS_COMMAND"]
          output_file = os.environ["GITHUB_OUTPUT"]

          files_to_parse = []

          if inputs_command == "integer_zk":
            files_to_parse.append("pke_zk_crs_sizes.csv")
          elif inputs_command == "hlapi_erc7984":
            files_to_parse.append("erc7984_pbs_count.csv")
          elif inputs_command == "hlapi_dex":
            files_to_parse.extend(
              [
                "dex_swap_request_update_dex_balance_pbs_count.csv",
                "dex_swap_request_finalize_pbs_count.csv",
                "dex_swap_claim_prepare_pbs_count.csv",
                "dex_swap_claim_update_dex_balance_pbs_count.csv"
              ]
            )

          with open(output_file, "a") as f:
            f.write(f"""additional_file_to_parse={",".join(files_to_parse)}\n""")

  run-benchmarks:
    name: benchmark_cpu/run-benchmarks
    needs: parse-inputs
    uses: ./.github/workflows/benchmark_cpu_common.yml
    with:
      backend: ${{ needs.parse-inputs.outputs.backend }}
      profile: ${{ needs.parse-inputs.outputs.profile }}
      hardware_name: ${{ needs.parse-inputs.outputs.hardware_name }}
      command: ${{ inputs.command }}
      op_flavor: ${{ inputs.op_flavor }}
      bench_type: ${{ inputs.bench_type }}
      params_type: ${{ inputs.params_type }}
      precisions_set: ${{ inputs.precisions_set }}
      additional_file_to_parse: ${{ needs.parse-inputs.outputs.additional_file_to_parse }}
    secrets:
      BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
      SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
      SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
      REPO_CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN }}
      JOB_SECRET: ${{ secrets.JOB_SECRET }}
      SLAB_ACTION_TOKEN: ${{ secrets.SLAB_ACTION_TOKEN }}
      SLAB_URL: ${{ secrets.SLAB_URL }}
      SLAB_BASE_URL: ${{ secrets.SLAB_BASE_URL }}