HPU backend fixes

.github/workflows/benchmark_documentation.yml

@@ -138,8 +138,8 @@ jobs:
       op_flavor: default
       bench_type: both
       precisions_set: documentation
-      v80_pcie_dev: 24
-      v80_serial_number: XFL12NWY3ZKG
+      hpu_number: 4
+      runner_target: v80-marais
     secrets:
       BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
       SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}

.github/workflows/benchmark_hpu.yml

@@ -38,12 +38,12 @@ on:
           - latency
           - throughput
           - both
-      v80_pcie_dev:
-        description: "V80 PCIe device number"
-        default: 24
-      v80_serial_number:
-        description: "V80 serial number"
-        default: XFL12NWY3ZKG
+      hpu_number:
+        description: "number of hpu used"
+        default: 4
+      runner_target:
+        description: "machine used for bench"
+        default: v80-marais
 
 permissions: {}
 
@@ -58,8 +58,8 @@ jobs:
       op_flavor: ${{ inputs.op_flavor }}
       bench_type: ${{ inputs.bench_type }}
       precisions_set: ${{ inputs.precisions_set }}
-      v80_pcie_dev: ${{ inputs.v80_pcie_dev }}
-      v80_serial_number: ${{ inputs.v80_serial_number }}
+      hpu_number: ${{ inputs.hpu_number }}
+      runner_target: ${{ inputs.runner_target }}
     secrets:
       BOT_USERNAME: ${{ secrets.BOT_USERNAME }}
       SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}

.github/workflows/benchmark_hpu_common.yml

@@ -16,12 +16,12 @@ on:
       precisions_set:
         type: string
         default: fast
-      v80_pcie_dev:
+      hpu_number:
         type: string
-        default: 24
-      v80_serial_number:
+        default: 4
+      runner_target:
         type: string
-        default: XFL12NWY3ZKG
+        default: v80-marais
     secrets:
       REPO_CHECKOUT_TOKEN:
         required: true
@@ -107,7 +107,7 @@ jobs:
   hpu-benchmarks:
     name: benchmark_hpu_common/hpu-benchmarks
     needs: prepare-matrix
-    runs-on: v80-marais
+    runs-on: ${{ inputs.runner_target }}
     concurrency:
       group: ${{ github.workflow }}_${{ github.ref }}
       cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
@@ -151,17 +151,20 @@ jobs:
 
       - name: Select HPU board
         run: |
-          echo "V80_PCIE_DEV=${PCIE_DEV}" >> "${GITHUB_ENV}"
-          echo "V80_SERIAL_NUMBER=${SERIAL_NUMBER}" >> "${GITHUB_ENV}"
+          echo "HPU_NUMBER=${HPU_NUMBER}" >> "${GITHUB_ENV}"
         env:
-          PCIE_DEV: ${{ inputs.v80_pcie_dev }}
-          SERIAL_NUMBER: ${{ inputs.v80_serial_number }}
+          HPU_NUMBER: ${{ inputs.hpu_number }}
 
       - name: Run benchmarks
         run: |
-          echo "${V80_PCIE_DEV} ${V80_SERIAL_NUMBER}"
+          echo "using ${HPU_NUMBER} HPU"
+          # to get available list of boards
+          . /etc/profile.d/v80_pcie_dev.sh
+          # to modify hpu_config.toml to use only selected number of HPU
+          # will fail at start of backend if requesting more than available HPU
+          ids=$(seq -s, 0 $((HPU_NUMBER - 1))) && sed -i "s/^  node_id=.*/  node_id=[$ids]/" backends/tfhe-hpu-backend/config_store/v80/hpu_config.toml
           make pull_hpu_files
-          make BIT_SIZES_SET="${PRECISIONS_SET}" BENCH_OP_FLAVOR="${OP_FLAVOR}" BENCH_TYPE="${BENCH_TYPE}" BENCH_PARAM_TYPE="${BENCH_PARAMS_TYPE}" bench_"${BENCH_COMMAND}"_hpu
+          export RUST_LOG=debug && export PATH=$PATH:/usr/local/sbin && make BIT_SIZES_SET="${PRECISIONS_SET}" BENCH_OP_FLAVOR="${OP_FLAVOR}" BENCH_TYPE="${BENCH_TYPE}" BENCH_PARAM_TYPE="${BENCH_PARAMS_TYPE}" bench_"${BENCH_COMMAND}"_hpu
         env:
           OP_FLAVOR: ${{ matrix.op_flavor }}
           BENCH_TYPE: ${{ matrix.bench_type }}
@@ -172,7 +175,7 @@ jobs:
         run: |
           cargo run --release -p tfhe-benchmark-parser -- -i target/criterion -o "${RESULTS_FILENAME}" \
           --database tfhe_rs \
-          --hardware "hpu_x1" \
+          --hardware "hpu_x${HPU_NUMBER}" \
           --backend hpu \
           --project-version "${COMMIT_HASH}" \
           --branch "${REF_NAME}" \

backends/tfhe-hpu-backend/README.md

@@ -147,7 +147,8 @@ Following code snippet shows how to instantiate and configure a `HpuDevice`:
 ```rust
     // Following code snippets used the HighLevelApi abstraction
     // Instantiate HpuDevice --------------------------------------------------
-    let hpu_device = HpuDevice::from_config(&args.config.expand());
+    let hpu_device = HpuDevice::from_config(&config_path.expand(), false)
+        .expect("Hpu device init failed");
 
     // Generate keys ----------------------------------------------------------
     let config = Config::from_hpu_device(&hpu_device);

backends/tfhe-hpu-backend/scripts/v80-pcie-perms.sh

@@ -3,15 +3,15 @@
 GROUP=hw;
 # Enable Pcie rescan
 if [ -e /sys/bus/pci/rescan ]; then
-  sudo /usr/bin/chgrp $GROUP /sys/bus/pci/rescan && sudo /usr/bin/chmod g+w /sys/bus/pci/rescan
+  sudo --non-interactive /usr/bin/chgrp $GROUP /sys/bus/pci/rescan && sudo --non-interactive /usr/bin/chmod g+w /sys/bus/pci/rescan
 fi
 # Search V80 pcie boards PF0
 for dev in $(lspci -nn -d 10ee:50b4 | awk '{print $1}'); do
-  sudo /usr/bin/chgrp -R $GROUP /sys/bus/pci/devices/0000\:$dev/
-  sudo /usr/bin/chmod -R g=u    /sys/bus/pci/devices/0000\:$dev/
+  sudo --non-interactive /usr/bin/chgrp -R $GROUP /sys/bus/pci/devices/0000\:$dev/
+  sudo --non-interactive /usr/bin/chmod -R g=u    /sys/bus/pci/devices/0000\:$dev/
 done
 # Search V80 pcie boards PF1
 for dev in $(lspci -nn -d 10ee:50b5 | awk '{print $1}'); do
-  sudo /usr/bin/chgrp -R $GROUP /sys/bus/pci/devices/0000\:$dev/
-  sudo /usr/bin/chmod -R g=u    /sys/bus/pci/devices/0000\:$dev/
+  sudo --non-interactive /usr/bin/chgrp -R $GROUP /sys/bus/pci/devices/0000\:$dev/
+  sudo --non-interactive /usr/bin/chmod -R g=u    /sys/bus/pci/devices/0000\:$dev/
 done

backends/tfhe-hpu-backend/src/ffi/v80/ami.rs

@@ -119,7 +119,7 @@ impl AmiDriver {
             .open(&ami_proc_path)
             .unwrap();
 
-        let addr = unsafe {
+        let iop_ack_addr = unsafe {
             mmap(
                 None,
                 NonZero::new(AMI_ACK_PTR_LEN).unwrap(),
@@ -130,7 +130,7 @@ impl AmiDriver {
             )?
         };
 
-        let iop_ack_atomic_ptr: NonNull<AtomicU32> = addr.cast();
+        let iop_ack_atomic_ptr: NonNull<AtomicU32> = iop_ack_addr.cast();
 
         // Try to use direct register access if available
         let bar_reg_ptr = Self::map_bar_reg(&ami_dev).ok();
@@ -165,14 +165,6 @@ impl AmiDriver {
         Ok(bar_addr)
     }
 
-    pub fn munmap_cnt(&self) -> Result<(), Box<dyn Error>> {
-        let cnt_addr = self.iop_ack_atomic_ptr.cast();
-        unsafe {
-            munmap(cnt_addr, AMI_ACK_PTR_LEN)?;
-        }
-        Ok(())
-    }
-
     /// Read currently loaded UUID in BAR
     pub fn uuid(&self) -> String {
         let ami_fd = self.ami_dev.as_raw_fd();
@@ -451,6 +443,22 @@ impl AmiDriver {
     }
 }
 
+impl Drop for AmiDriver {
+    fn drop(&mut self) {
+        let iop_ack_addr = self.iop_ack_atomic_ptr.cast();
+        unsafe {
+            munmap(iop_ack_addr, AMI_ACK_PTR_LEN).expect("Unable to unmap iop_ack_ptr");
+        }
+
+        if let Some(bar_ptr) = self.bar_reg_ptr {
+            let bar_addr = bar_ptr.cast();
+            unsafe {
+                munmap(bar_addr, AMI_BAR_LEN).expect("Unable to unmap bar_reg_ptr");
+            }
+        }
+    }
+}
+
 // Define driver IOCTL command and associated payload -------------------------
 const AMI_IOC_MAGIC: u8 = b'a';
 

backends/tfhe-hpu-backend/src/ffi/v80/mod.rs

@@ -77,6 +77,7 @@ impl HpuHw {
             // Reload Ami driver
             tracing::info!("Load ami kernel module [{ami_path}]");
             Command::new("sudo")
+                .arg("--non-interactive")
                 .arg("/usr/sbin/insmod")
                 .arg(ami_path)
                 .status()
@@ -101,7 +102,6 @@ impl HpuHw {
                 // Check state and version
                 match AmiDriver::new(&board.pcie_id, &hpu_pdi.metadata.amc.his_version, None) {
                     Ok(ami) => {
-                        ami.munmap_cnt().unwrap();
                         if hpu_pdi.metadata.bitstream.uuid.to_lowercase()
                             == ami.uuid().to_lowercase()
                         {
@@ -180,6 +180,7 @@ impl HpuHw {
         // If either we fail to unload ami or qdma we must raise an error
         tracing::info!("Unload drivers ami/qdma_pf");
         let _ = Command::new("sudo")
+            .arg("--non-interactive")
             .arg("/usr/sbin/rmmod")
             .arg("--syslog") // Output to syslog instead of stderr
             .arg("ami")
@@ -189,6 +190,7 @@ impl HpuHw {
             "Failed to unload ami driver: module still loaded"
         );
         let _ = Command::new("sudo")
+            .arg("--non-interactive")
             .arg("/usr/sbin/rmmod")
             .arg("--syslog") // Output to syslog instead of stderr
             .arg("qdma_pf")
@@ -396,14 +398,25 @@ impl HpuHw {
     /// Thus dma_queues must be recreated for each node when any of the cluster node is reloaded -_-
     pub fn cfg_dma_queues(dev: &str) {
         // Configure maximum Dma queues
-        OpenOptions::new()
-            .write(true)
-            .open(format!(
-                "/sys/bus/pci/devices/0000:{dev}:00.1/qdma/qmax"
-            ))
-            .expect("Unable to open qdma qmax cmd file")
-            .write_all(b"100\n")
-        .unwrap_or_else(|_| tracing::debug!("Dma: Failed to configure qmax. Must have been already done after the last rescan"));
+        let qmax_path = format!("/sys/bus/pci/devices/0000:{dev}:00.1/qdma/qmax");
+
+        match std::fs::read_to_string(&qmax_path) {
+            Ok(current) if current.trim() == "100" => {
+                tracing::debug!("Dma: qmax already set to 100, skipping");
+            }
+            Ok(_) => {
+                if let Err(e) = OpenOptions::new()
+                    .write(true)
+                    .open(&qmax_path)
+                    .and_then(|mut f| f.write_all(b"100\n"))
+                {
+                    tracing::debug!("Dma: Failed to configure qmax ({e}). Must have been already done after the last rescan");
+                }
+            }
+            Err(e) => {
+                tracing::debug!("Dma: Failed to read qmax ({e}), skipping configuration");
+            }
+        }
 
         // Create user queues ----------------------------------------------
         let h2c_path = format!("/dev/qdma{dev}001-MM-1");

tfhe-benchmark/benches/high_level_api/bench_signed.rs

@@ -50,7 +50,8 @@ fn main() {
         let config_path = ShellString::new(
             "${HPU_BACKEND_DIR}/config_store/${HPU_CONFIG}/hpu_config.toml".to_string(),
         );
-        let hpu_device = HpuDevice::from_config(&config_path.expand());
+        let hpu_device =
+            HpuDevice::from_config(&config_path.expand(), false).expect("Hpu device init failed");
 
         let config = Config::from_hpu_device(&hpu_device);
         let cks = ClientKey::generate(config);

tfhe-benchmark/benches/high_level_api/bench_unsigned.rs

@@ -57,7 +57,8 @@ fn main() {
         let config_path = ShellString::new(
             "${HPU_BACKEND_DIR}/config_store/${HPU_CONFIG}/hpu_config.toml".to_string(),
         );
-        let hpu_device = HpuDevice::from_config(&config_path.expand());
+        let hpu_device =
+            HpuDevice::from_config(&config_path.expand(), false).expect("Hpu device init failed");
 
         let config = Config::from_hpu_device(&hpu_device);
         let cks = ClientKey::generate(config);