adds prometheus metrics for disk availability

wookietreiber · wookietreiber · commit 4aed0fe77cc5 · 2024-06-05T11:06:16.000+02:00
diff --git a/README.md b/README.md
@@ -29,6 +29,7 @@ Notable exceptions to parsing in the library and listing in tooling are:
 
 1.  Prometheus metrics for:
 
+    - disk availability
     - disk pool size
     - disk pool group I/O
     - quotas
diff --git a/src/bin/mmoxi/cli.rs b/src/bin/mmoxi/cli.rs
@@ -109,6 +109,13 @@ pub fn build_prometheus() -> Command {
         .arg(arg_output())
         .after_long_help("Run on cluster manager only.");
 
+    let prom_disk = Command::new("disk")
+        .about("Gather metrics from mmlsdisk.")
+        .disable_help_flag(true)
+        .disable_version_flag(true)
+        .arg(arg_output())
+        .after_long_help("Run on cluster manager only.");
+
     let prom_fileset = Command::new("fileset")
         .about("Gather fileset metrics.")
         .disable_help_flag(true)
@@ -172,6 +179,7 @@ pub fn build_prometheus() -> Command {
         .subcommand_required(true)
         .arg_required_else_help(true)
         .subcommand(prom_df)
+        .subcommand(prom_disk)
         .subcommand(prom_fileset)
         .subcommand(prom_pool)
         .subcommand(prom_quota)
diff --git a/src/bin/mmoxi/main.rs b/src/bin/mmoxi/main.rs
@@ -8,6 +8,7 @@ use std::path::PathBuf;
 
 use anyhow::{anyhow, Context, Result};
 use clap::ArgMatches;
+use mmoxi::prom::ToText;
 
 mod cli;
 
@@ -57,6 +58,7 @@ fn dispatch_list_manager(args: &ArgMatches) -> Result<()> {
 fn dispatch_prom(args: &ArgMatches) -> Result<()> {
     match args.subcommand() {
         Some(("df", args)) => run_prom_df(args),
+        Some(("disk", args)) => run_prom_disk(args),
         Some(("fileset", args)) => run_prom_fileset(args),
         Some(("pool", args)) => dispatch_prom_pool(args),
         Some(("quota", args)) => run_prom_quota(args),
@@ -229,6 +231,21 @@ fn run_prom_df(args: &ArgMatches) -> Result<()> {
     Ok(())
 }
 
+fn run_prom_disk(args: &ArgMatches) -> Result<()> {
+    let mut output = output_to_bufwriter(args)?;
+
+    let mut all_disks = HashMap::new();
+
+    for fs in mmoxi::fs::names()? {
+        let disks = mmoxi::disk::disks(&fs)?;
+        all_disks.insert(fs, disks);
+    }
+
+    all_disks.to_prom(&mut output)?;
+
+    Ok(())
+}
+
 fn run_prom_fileset(args: &ArgMatches) -> Result<()> {
     let mut output = output_to_bufwriter(args)?;
 
diff --git a/src/disk-example.in b/src/disk-example.in
@@ -1,4 +1,5 @@
 mmlsdisk::HEADER:version:reserved:reserved:nsdName:driverType:sectorSize:failureGroup:metadata:data:status:availability:diskID:storagePool:remarks:numQuorumDisks:readQuorumValue:writeQuorumValue:diskSizeKB:diskUID:thinDiskType:replicaType:
 mmlsdisk::0:1:::disk1:nsd:512:1:Yes:No:ready:up:42:system::3:2:2:1869606912:0A0A141A5AFAE37E:::
-mmlsdisk::0:1:::disk2:nsd:512:1:No:Yes:ready:up:43:nvme::3:2:2:1869606912:0A0A141A5AFAE386:::
-mmlsdisk::0:1:::disk3:nsd:512:1:No:Yes:ready:up:44:nlsas::3:2:2:1869606912:0A0A141A5AFAE382:::
+mmlsdisk::0:1:::disk2:nsd:512:1:No:Yes:ready:down:43:nvme::3:2:2:1869606912:0A0A141A5AFAE386:::
+mmlsdisk::0:1:::disk3:nsd:512:1:No:Yes:ready:recovering:44:nlsas::3:2:2:1869606912:0A0A141A5AFAE382:::
+mmlsdisk::0:1:::disk4:nsd:512:1:No:Yes:ready:unrecovered:45:nlsas::3:2:2:1869606912:0A0A141A5AFAE382:::
diff --git a/src/disk-example.prom b/src/disk-example.prom
@@ -0,0 +1,6 @@
+# HELP gpfs_disk_availability GPFS disk state.
+# TYPE gpfs_disk_availability gauge
+gpfs_disk_availability{name="disk1",fs="gpfs1",pool="system",availability="up"} 0
+gpfs_disk_availability{name="disk2",fs="gpfs1",pool="nvme",availability="down"} 1
+gpfs_disk_availability{name="disk3",fs="gpfs1",pool="nlsas",availability="recovering"} 1
+gpfs_disk_availability{name="disk4",fs="gpfs1",pool="nlsas",availability="unrecovered"} 1
diff --git a/src/disk.rs b/src/disk.rs
@@ -1,7 +1,10 @@
 //! `mmlsdisk` parsing.
 
-use std::io::BufRead;
+use std::collections::HashMap;
+use std::fmt::Display;
+use std::io::{BufRead, Write};
 use std::process::Command;
+use std::str::FromStr;
 
 use anyhow::{anyhow, Context, Result};
 
@@ -12,7 +15,7 @@ use crate::util::MMBool;
 /// # Errors
 ///
 /// Returns an error if running `mmlsdisk` fails or if parsing its output fails.
-pub fn disks<S: AsRef<str>>(fs_name: S) -> Result<Disks> {
+pub fn disks(fs_name: impl AsRef<str>) -> Result<Disks> {
     let fs_name = fs_name.as_ref();
 
     let mut cmd = Command::new("mmlsdisk");
@@ -102,12 +105,61 @@ impl FromIterator<Self> for Disks {
     }
 }
 
+/// Disk availability.
+#[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug)]
+#[non_exhaustive]
+pub enum Availability {
+    /// Disk is available for I/O operations.
+    Up,
+
+    /// No I/O operations can be performed.
+    Down,
+
+    /// Intermediate state for disks coming up.
+    Recovering,
+
+    /// Disk was not successfully brought up.
+    Unrecovered,
+
+    /// Unknown state.
+    Unknown(String),
+}
+
+impl Display for Availability {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let s = match self {
+            Self::Up => "up",
+            Self::Down => "down",
+            Self::Recovering => "recovering",
+            Self::Unrecovered => "unrecovered",
+            Self::Unknown(s) => s.as_str(),
+        };
+
+        write!(f, "{s}")
+    }
+}
+
+impl FromStr for Availability {
+    type Err = anyhow::Error;
+
+    fn from_str(s: &str) -> Result<Self> {
+        match s {
+            "up" => Ok(Self::Up),
+            "down" => Ok(Self::Down),
+            "recovering" => Ok(Self::Recovering),
+            "unrecovered" => Ok(Self::Unrecovered),
+            unknown => Ok(Self::Unknown(unknown.into())),
+        }
+    }
+}
+
 /// Disk data.
 #[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug)]
 pub struct Disk {
     nsd_name: String,
     is_metadata: bool,
     is_objectdata: bool,
+    availability: Availability,
     storage_pool: String,
 }
 
@@ -159,6 +211,12 @@ impl Disk {
         let is_objectdata =
             tokens[is_objectdata_index].parse::<MMBool>()?.as_bool();
 
+        let availability_index = index
+            .availability
+            .ok_or_else(|| anyhow!("no availability index"))?;
+        let availability =
+            tokens[availability_index].parse::<Availability>()?;
+
         let storage_pool_index = index
             .storage_pool
             .ok_or_else(|| anyhow!("no storage pool index"))?;
@@ -168,6 +226,7 @@ impl Disk {
             nsd_name,
             is_metadata,
             is_objectdata,
+            availability,
             storage_pool,
         })
     }
@@ -178,6 +237,7 @@ struct Index {
     nsd_name: Option<usize>,
     is_metadata: Option<usize>,
     is_objectdata: Option<usize>,
+    availability: Option<usize>,
     storage_pool: Option<usize>,
 }
 
@@ -187,57 +247,128 @@ fn header_to_index(tokens: &[&str], index: &mut Index) {
             "nsdName" => index.nsd_name = Some(i),
             "metadata" => index.is_metadata = Some(i),
             "data" => index.is_objectdata = Some(i),
+            "availability" => index.availability = Some(i),
             "storagePool" => index.storage_pool = Some(i),
             _ => {}
         }
     }
 }
 
+// ----------------------------------------------------------------------------
+// prometheus
+// ----------------------------------------------------------------------------
+
+impl<S: ::std::hash::BuildHasher> crate::prom::ToText
+    for HashMap<String, Disks, S>
+{
+    fn to_prom(&self, output: &mut impl Write) -> Result<()> {
+        for (fs, disks) in self {
+            writeln!(
+                output,
+                "# HELP gpfs_disk_availability GPFS disk state."
+            )?;
+            writeln!(output, "# TYPE gpfs_disk_availability gauge")?;
+
+            for disk in &disks.0 {
+                let status = match disk.availability {
+                    Availability::Up => 0,
+                    _ => 1,
+                };
+
+                writeln!(
+                output,
+                "gpfs_disk_availability{{name=\"{}\",fs=\"{}\",pool=\"{}\",availability=\"{}\"}} {}",
+                disk.nsd_name,
+                fs,
+                disk.storage_pool,
+                disk.availability,
+                status,
+            )?;
+            }
+        }
+
+        Ok(())
+    }
+}
+
 // ----------------------------------------------------------------------------
 // tests
 // ----------------------------------------------------------------------------
 
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::prom::ToText;
 
     #[test]
     fn parse() {
         let input = include_str!("disk-example.in");
 
-        let fs = Disks::from_reader(input.as_bytes()).unwrap();
-        let mut fs = fs.0.into_iter();
+        let disks = Disks::from_reader(input.as_bytes()).unwrap();
+        let mut disks = disks.0.into_iter();
 
         assert_eq!(
-            fs.next(),
+            disks.next(),
             Some(Disk {
                 nsd_name: "disk1".into(),
                 is_metadata: true,
                 is_objectdata: false,
+                availability: Availability::Up,
                 storage_pool: "system".into(),
             })
         );
 
         assert_eq!(
-            fs.next(),
+            disks.next(),
             Some(Disk {
                 nsd_name: "disk2".into(),
                 is_metadata: false,
                 is_objectdata: true,
+                availability: Availability::Down,
                 storage_pool: "nvme".into(),
             })
         );
 
         assert_eq!(
-            fs.next(),
+            disks.next(),
             Some(Disk {
                 nsd_name: "disk3".into(),
                 is_metadata: false,
                 is_objectdata: true,
+                availability: Availability::Recovering,
                 storage_pool: "nlsas".into(),
             })
         );
 
-        assert_eq!(fs.next(), None);
+        assert_eq!(
+            disks.next(),
+            Some(Disk {
+                nsd_name: "disk4".into(),
+                is_metadata: false,
+                is_objectdata: true,
+                availability: Availability::Unrecovered,
+                storage_pool: "nlsas".into(),
+            })
+        );
+
+        assert_eq!(disks.next(), None);
+    }
+
+    #[test]
+    fn prometheus() {
+        let input = include_str!("disk-example.in");
+
+        let disks = Disks::from_reader(input.as_bytes()).unwrap();
+
+        let mut all_disks = HashMap::new();
+        all_disks.insert(String::from("gpfs1"), disks);
+
+        let mut output = vec![];
+        all_disks.to_prom(&mut output).unwrap();
+
+        let metrics = std::str::from_utf8(output.as_slice()).unwrap();
+
+        let expected = include_str!("disk-example.prom");
+        assert_eq!(metrics, expected);
     }
 }
diff --git a/src/prom.rs b/src/prom.rs
@@ -10,6 +10,17 @@ use crate::fileset::Fileset;
 use crate::nsd::FsPoolId;
 use crate::sysfs;
 
+/// Convert `Data` to Prometheus metrics.
+pub trait ToText {
+    /// Converts the `Data` to Prometheus' text-based format.
+    ///
+    /// # Errors
+    ///
+    /// This function uses [`writeln`] to write to `output`. It can only fail
+    /// if any of these [`writeln`] fails.
+    fn to_prom(&self, output: &mut impl Write) -> Result<()>;
+}
+
 /// Writes the `mmdf` NSD data as prometheus metrics to `output`.
 ///
 /// # Errors
