Add links to grafana from instance detail pages, add grafana provisio…

…n script (#943)

## Done

- add grafana base url to the settings
- add links to grafana from instance detail page if its base url is
configured
- add script to setup grafana

## QA

1. Run the LXD-UI:
- On the demo server via the link posted by @webteam-app below. This is
only available for PRs created by collaborators of the repo. Ask
@mas-who or @edlerd for access.
- With a local copy of this branch, [build and run as described in the
docs](../CONTRIBUTING.md#setting-up-for-development).
2. Perform the following QA steps:
    - setup o11y stack for lxd: 
```
lxc launch ubuntu:24.04 grafana
curl https://localhost:8407/ui/assets/scripts/setup-grafana.sh --insecure > /tmp/setup-grafana.sh
bash /tmp/setup-grafana.sh grafana default
rm /tmp/setup-grafana.sh
```
    - create the dashboard:
      - open grafana with the link from the script above, then
      - login with admin/admin and chose a new password
      - after login, go to dashboards > new > import
      - enter 19131 and click load
      - confirm the options for prometheus and loki, the click import
- go back to the lxd-ui and ensure the links from instance detail page
to "metrics" work fine.

public/assets/scripts/setup-grafana.sh

@@ -0,0 +1,129 @@
+#!/bin/bash
+
+if [ "$#" -ne 2 ]; then
+    echo "Usage: $0 <instance> <project>"
+    echo "Error: Both 'instance' and 'project' arguments are required."
+    exit 1
+fi
+
+INSTANCE=$1
+PROJECT=$2
+
+IS_LXD_CLUSTERED=$(lxc info | grep "server_clustered:" | grep "false")
+if [ -z "$IS_LXD_CLUSTERED" ]; then
+    echo "Error: LXD is clustered, this script only works for single node installations."
+    echo "See https://documentation.ubuntu.com/lxd/en/latest/metrics/ for more information."
+    exit 1
+fi
+
+CONTAINER_IP=$(lxc info "$INSTANCE" --project="$PROJECT" | grep inet: | grep "global" | head -n1 | cut -d ":" -f2 | cut -d " " -f3 | cut -d "/" -f1)
+CONTAINER_UPLINK_IP=$(echo "$CONTAINER_IP" | cut -d "." -f1,2,3)".1"
+echo "Found container IP as '$CONTAINER_IP' and uplink as '$CONTAINER_UPLINK_IP'"
+
+set -e
+set -x
+
+# upload server.crt to container
+lxc info | sed -n "/BEGIN CERTIFICATE/,/END CERTIFICATE/p" | sed 's/^[ \t]*//;s/[ \t]*$//' > /tmp/server.crt
+lxc file push /tmp/server.crt "$INSTANCE"/root/server.crt --project="$PROJECT"
+rm /tmp/server.crt
+
+# install and configure grafana and prometheus in container
+lxc exec "$INSTANCE" --project="$PROJECT" bash <<EOF
+set -x
+set -e
+# install grafana and prometheus
+apt-get update
+apt-get install -y apt-transport-https software-properties-common wget prometheus
+mkdir -p /etc/apt/keyrings/
+wget -q -O - https://apt.grafana.com/gpg.key | gpg --dearmor | tee /etc/apt/keyrings/grafana.gpg > /dev/null
+echo "deb [signed-by=/etc/apt/keyrings/grafana.gpg] https://apt.grafana.com stable main" | tee -a /etc/apt/sources.list.d/grafana.list
+apt-get update
+apt-get install -y grafana=11.4.0 loki=3.3.2 promtail=3.3.2
+systemctl daemon-reload
+systemctl start grafana-server
+systemctl enable grafana-server.service
+sed -ie '44d' /etc/loki/config.yml # fix the loki configuration, see https://github.com/grafana/loki/issues/15039
+systemctl start loki
+systemctl enable loki
+systemctl start promtail
+systemctl enable promtail
+
+# generate ssl key for grafana to serve via https
+openssl req -x509 -newkey rsa:4096 -keyout /etc/grafana/grafana.key -out /etc/grafana/grafana.crt -days 365 -nodes -subj "/CN=metrics.local"
+chown grafana:grafana /etc/grafana/grafana.crt
+chown grafana:grafana /etc/grafana/grafana.key
+chmod 400 /etc/grafana/grafana.key /etc/grafana/grafana.crt
+sed -i "s#;protocol = http#protocol = https#" /etc/grafana/grafana.ini
+cat <<EOT > /etc/grafana/provisioning/datasources/lxd-sources.yaml
+apiVersion: 1
+
+datasources:
+  - name: prometheus
+    type: prometheus
+    access: proxy
+    url: http://$CONTAINER_IP:9090
+  - name: loki
+    type: loki
+    access: proxy
+    url: http://$CONTAINER_IP:3100
+EOT
+systemctl restart grafana-server
+
+# generate certs for prometheus
+openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:secp384r1 -sha384 -keyout metrics.key -nodes -out metrics.crt -days 3650 -subj "/CN=metrics.local"
+mkdir /etc/prometheus/tls
+mv metrics.* /etc/prometheus/tls/
+mv server.crt /etc/prometheus/tls/
+chown -R prometheus:root /etc/prometheus/tls
+chmod 400 /etc/prometheus/tls/metrics.key /etc/prometheus/tls/metrics.crt /etc/prometheus/tls/server.crt
+
+# configure prometheus
+cat <<EOT > /etc/prometheus/prometheus.yml
+global:
+  scrape_interval:     15s
+  evaluation_interval: 15s
+  scrape_timeout: 15s
+scrape_configs:
+  - job_name: lxd
+    scrape_interval: 15s
+    scrape_timeout: 15s
+    metrics_path: '/1.0/metrics'
+    scheme: 'https'
+    static_configs:
+      - targets: ['$CONTAINER_UPLINK_IP:8443']
+    tls_config:
+      ca_file: '/etc/prometheus/tls/server.crt'
+      cert_file: '/etc/prometheus/tls/metrics.crt'
+      key_file: '/etc/prometheus/tls/metrics.key'
+      # XXX: server_name is required if the target name
+      #      is not covered by the certificate (not in the SAN list)
+      server_name: '$HOSTNAME'
+EOT
+systemctl daemon-reload
+systemctl start prometheus
+systemctl enable prometheus.service
+EOF
+
+# download metrics.crt from container and add to host lxd trust store
+lxc file pull "$INSTANCE"/etc/prometheus/tls/metrics.crt --project="$PROJECT" /tmp/metrics.crt
+lxc config trust add /tmp/metrics.crt --type=metrics
+rm -rf /tmp/metrics.crt
+
+# configure host lxd for loki and grafana
+lxc config set user.grafana_base_url=https://"$CONTAINER_IP":3000/d/bGY-LSB7k/lxd?orgId=1
+lxc config set loki.api.url=http://"$CONTAINER_IP":3100 loki.instance=lxd &
+
+# restart container
+lxc exec "$INSTANCE" --project="$PROJECT" reboot
+sleep 10
+
+set +x
+
+# print grafana url
+echo "Successfully initialized grafana"
+echo "Next steps:"
+echo "1. Wait for the container to finish booting"
+echo "2. Sign in with admin/admin to grafana at https://$CONTAINER_IP:3000"
+echo "3. Change password"
+echo "4. Create a dashboard, see https://documentation.ubuntu.com/lxd/en/latest/howto/grafana/ for more details."

src/pages/instances/InstanceDetail.tsx

@@ -1,5 +1,5 @@
 import { FC } from "react";
-import { Notification, Row, Strip } from "@canonical/react-components";
+import { Icon, Notification, Row, Strip } from "@canonical/react-components";
 import InstanceOverview from "./InstanceOverview";
 import InstanceTerminal from "./InstanceTerminal";
 import { useParams } from "react-router-dom";
@@ -14,6 +14,8 @@ import EditInstance from "./EditInstance";
 import InstanceDetailHeader from "pages/instances/InstanceDetailHeader";
 import CustomLayout from "components/CustomLayout";
 import TabLinks from "components/TabLinks";
+import { useSettings } from "context/useSettings";
+import { TabLink } from "@canonical/react-components/dist/components/Tabs/Tabs";
 
 const tabs: string[] = [
   "Overview",
@@ -25,6 +27,8 @@ const tabs: string[] = [
 ];
 
 const InstanceDetail: FC = () => {
+  const { data: settings } = useSettings();
+
   const { name, project, activeTab } = useParams<{
     name: string;
     project: string;
@@ -48,6 +52,22 @@ const InstanceDetail: FC = () => {
     queryFn: () => fetchInstance(name, project),
   });
 
+  const renderTabs: (string | TabLink)[] = [...tabs];
+
+  const grafanaBaseUrl = settings?.config?.["user.grafana_base_url"] ?? "";
+  if (grafanaBaseUrl) {
+    renderTabs.push({
+      label: (
+        <div>
+          <Icon name="external-link" /> Metrics
+        </div>
+      ) as unknown as string,
+      href: `${grafanaBaseUrl}&var-job=lxd&var-project=${project}&var-name=${instance?.name}&var-top=5`,
+      target: "_blank",
+      rel: "noopener noreferrer",
+    });
+  }
+
   return (
     <CustomLayout
       header={
@@ -72,7 +92,7 @@ const InstanceDetail: FC = () => {
       {!isLoading && instance && (
         <Row>
           <TabLinks
-            tabs={tabs}
+            tabs={renderTabs}
             activeTab={activeTab}
             tabUrl={`/ui/project/${project}/instance/${name}`}
           />

src/pages/settings/Settings.tsx

@@ -89,6 +89,16 @@ const Settings: FC = () => {
     });
   }
 
+  configFields.push({
+    key: "user.grafana_base_url",
+    category: "user",
+    default: "",
+    longdesc: "e.g. https://192.0.2.1:3000/d/bGY-LSB7k/lxd?orgId=1",
+    shortdesc:
+      " See {ref}`grafana` for more information. Pages link to metrics, when set.",
+    type: "string",
+  });
+
   let lastCategory = "";
   const rows = configFields
     .filter((configField) => {

tests/server.spec.ts

@@ -56,5 +56,5 @@ test("only user server setting available for lxd v5.0/edge", async ({
   await visitServerSettings(page);
   await page.waitForSelector(`text=Get more server settings`);
   const allSettingRows = await page.locator("#settings-table tbody tr").all();
-  expect(allSettingRows.length).toEqual(1);
+  expect(allSettingRows.length).toEqual(2);
 });