Skip to content

Tree-based APC storage engine #37

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 4 commits into from
Closed

Tree-based APC storage engine #37

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
4eb9277
Much faster tree-based APC storage instead of APCUIterator
Feb 4, 2021
963afe3
Prevent possible infinite loop with APCu check-and-set
Feb 4, 2021
f96072f
Clean up phpstan output
Feb 4, 2021
f7d7ade
Clean up phpcs output
Feb 4, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
239 changes: 215 additions & 24 deletions src/Prometheus/Storage/APC.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,14 @@

namespace Prometheus\Storage;

use APCUIterator;
use Prometheus\Exception\StorageException;
use Prometheus\MetricFamilySamples;
use RuntimeException;

class APC implements Adapter
{
const PROMETHEUS_PREFIX = 'prom';
const CAS_LOOP_RETRIES = 30;

/**
* APC constructor.
Expand Down Expand Up @@ -51,17 +51,22 @@ public function updateHistogram(array $data): void
// If sum does not exist, assume a new histogram and store the metadata
if ($new) {
apcu_store($this->metaKey($data), json_encode($this->metaData($data)));
$this->storeLabelKeys($data);
}

// Atomically increment the sum
// Taken from https://github.com/prometheus/client_golang/blob/66058aac3a83021948e5fb12f1f408ff556b9037/prometheus/value.go#L91
$done = false;
while (!$done) {
$loopCatcher = self::CAS_LOOP_RETRIES;
while (!$done && $loopCatcher-- > 0) {
$old = apcu_fetch($sumKey);
if ($old !== false) {
$done = apcu_cas($sumKey, $old, $this->toBinaryRepresentationAsInteger($this->fromBinaryRepresentationAsInteger($old) + $data['value']));
}
}
if ($loopCatcher <= 0) {
throw new RuntimeException('Caught infinite loop in ' . __METHOD__ . '()');
}

// Figure out in which bucket the observation belongs
$bucketToIncrease = '+Inf';
Expand All @@ -86,19 +91,25 @@ public function updateGauge(array $data): void
if ($data['command'] === Adapter::COMMAND_SET) {
apcu_store($valueKey, $this->toBinaryRepresentationAsInteger($data['value']));
apcu_store($this->metaKey($data), json_encode($this->metaData($data)));
$this->storeLabelKeys($data);
} else {
$new = apcu_add($valueKey, $this->toBinaryRepresentationAsInteger(0));
if ($new) {
apcu_store($this->metaKey($data), json_encode($this->metaData($data)));
$this->storeLabelKeys($data);
}
// Taken from https://github.com/prometheus/client_golang/blob/66058aac3a83021948e5fb12f1f408ff556b9037/prometheus/value.go#L91
$done = false;
while (!$done) {
$loopCatcher = self::CAS_LOOP_RETRIES;
while (!$done && $loopCatcher-- > 0) {
$old = apcu_fetch($valueKey);
if ($old !== false) {
$done = apcu_cas($valueKey, $old, $this->toBinaryRepresentationAsInteger($this->fromBinaryRepresentationAsInteger($old) + $data['value']));
}
}
if ($loopCatcher <= 0) {
throw new RuntimeException('Caught infinite loop in ' . __METHOD__ . '()');
}
}
}

Expand All @@ -109,19 +120,75 @@ public function updateCounter(array $data): void
{
$valueKey = $this->valueKey($data);
// Check if value key already exists
if (apcu_exists($this->valueKey($data)) === false) {
apcu_add($this->valueKey($data), 0);
if (apcu_exists($valueKey) === false) {
apcu_add($valueKey, 0);
apcu_store($this->metaKey($data), json_encode($this->metaData($data)));
$this->storeLabelKeys($data);
}

// Taken from https://github.com/prometheus/client_golang/blob/66058aac3a83021948e5fb12f1f408ff556b9037/prometheus/value.go#L91
$done = false;
while (!$done) {
$loopCatcher = self::CAS_LOOP_RETRIES;
while (!$done && $loopCatcher-- > 0) {
$old = apcu_fetch($valueKey);
if ($old !== false) {
$done = apcu_cas($valueKey, $old, $this->toBinaryRepresentationAsInteger($this->fromBinaryRepresentationAsInteger($old) + $data['value']));
}
}
if ($loopCatcher <= 0) {
throw new RuntimeException('Caught infinite loop in ' . __METHOD__ . '()');
}
}

/**
* @param array<string> $metaData
* @param string $labels
* @return string
*/
private function assembleLabelKey(array $metaData, string $labels): string
{
return implode(':', [ self::PROMETHEUS_PREFIX, $metaData['type'], $metaData['name'], $labels, 'label' ]);
}

/**
* Store ':label' keys for each metric's labelName in APC.
*
* @param array<mixed> $data
* @return void
*/
private function storeLabelKeys(array $data): void
{
// Store metadata key in tree root
$this->addItemToKey($this->rootNode(), $this->metaKey($data));

// Store labelValues in each labelName key
foreach ($data['labelNames'] as $seq => $label) {
$this->addItemToKey(implode(':', [
self::PROMETHEUS_PREFIX,
$data['type'],
$data['name'],
$label,
'label'
]), isset($data['labelValues']) ? $data['labelValues'][$seq] : ''); // may not need the isset check
}
}

/**
* Ensures an array serialized into APCu contains exactly one copy of a given string
*
* @return void
*/
private function addItemToKey(string $key, string $item): void
{
$arr = apcu_fetch($key);
if (false === $arr) {
$arr = [];
}
if (in_array($item, $arr, true)) {
return;
}
$arr[] = $item;
apcu_store($key, $arr);
}

/**
Expand All @@ -141,15 +208,27 @@ public function flushAPC(): void
*/
public function wipeStorage(): void
{
// / / | PCRE expresion boundary
// ^ | match from first character only
// %s: | common prefix substitute with colon suffix
// .+ | at least one additional character
$matchAll = sprintf('/^%s:.+/', self::PROMETHEUS_PREFIX);

foreach (new APCUIterator($matchAll) as $key => $value) {
apcu_delete($key);
$root = apcu_fetch($this->rootNode());

if (is_array($root)) {
$metricTypeList = ['counter', 'gauge', 'histogram'];
foreach ($metricTypeList as $metricType) {
foreach ($this->getMetas($metricType) as $metaRecord) {
$metaData = json_decode($metaRecord['value'], true);
if ($metricType === 'histogram') {
$metaData['buckets'][] = '+Inf';
}
foreach ($this->getValues($metricType, $metaData) as $value) {
apcu_delete($value['key']);
}
foreach (array_values($metaData['labelNames']) as $label) {
apcu_delete($this->assembleLabelKey($metaData, $label));
}
apcu_delete($metaRecord['key']);
}
}
}
apcu_delete($this->rootNode());
}

/**
Expand Down Expand Up @@ -204,13 +283,59 @@ private function metaData(array $data): array
return $metricsMetaData;
}

/**
* When given a ragged 2D array $labelValues of arbitrary size, and a 1D array $labelNames containing one
* string labeling each row of $labelValues, return an array-of-arrays containing all possible permutations
* of labelValues, with the sub-array elements in order of labelName.
*
* Example input:
* $labelNames: ['endpoint', 'method', 'result']
* $labelValues: [0] => ['/', '/private', '/metrics'], // "endpoint"
* [1] => ['put', 'get', 'post'], // "method"
* [2] => ['success', 'fail'] // "result"
* Returned array:
* [0] => ['/', 'put', 'success'], [1] => ['/', 'put', 'fail'], [2] => ['/', 'get', 'success'],
* [3] => ['/', 'get', 'fail'], [4] => ['/', 'post', 'success'], [5] => ['/', 'post', 'fail'],
* [6] => ['/private', 'put', 'success'], [7] => ['/private', 'put', 'fail'], [8] => ['/private', 'get', 'success'],
* [9] => ['/private', 'get', 'fail'], [10] => ['/private', 'post', 'success'], [11] => ['/private', 'post', 'fail'],
* [12] => ['/metrics', 'put', 'success'], [13] => ['/metrics', 'put', 'fail'], [14] => ['/metrics', 'get', 'success'],
* [15] => ['/metrics', 'get', 'fail'], [16] => ['/metrics', 'post', 'success'], [17] => ['/metrics', 'post', 'fail']
* @param array<string> $labelNames
* @param array<array> $labelValues
* @return array<array>
*/
private function buildPermutationTree(array $labelNames, array $labelValues): array
{
$treeRowCount = count(array_keys($labelNames));
$numElements = 1;
$treeInfo = [];
for ($i = $treeRowCount - 1; $i >= 0; $i--) {
$treeInfo[$i]['numInRow'] = count($labelValues[$i]);
$numElements *= $treeInfo[$i]['numInRow'];
$treeInfo[$i]['numInTree'] = $numElements;
}

$map = array_fill(0, $numElements, []);
for ($row = 0; $row < $treeRowCount; $row++) {
$col = $i = 0;
while ($i < $numElements) {
$val = $labelValues[$row][$col];
$map[$i] = array_merge($map[$i], array($val));
if (++$i % ($treeInfo[$row]['numInTree'] / $treeInfo[$row]['numInRow']) == 0) {
$col = ++$col % $treeInfo[$row]['numInRow'];
}
}
}
return $map;
}

/**
* @return MetricFamilySamples[]
*/
private function collectCounters(): array
{
$counters = [];
foreach (new APCUIterator('/^prom:counter:.*:meta/') as $counter) {
foreach ($this->getMetas('counter') as $counter) {
$metaData = json_decode($counter['value'], true);
$data = [
'name' => $metaData['name'],
Expand All @@ -219,7 +344,7 @@ private function collectCounters(): array
'labelNames' => $metaData['labelNames'],
'samples' => [],
];
foreach (new APCUIterator('/^prom:counter:' . $metaData['name'] . ':.*:value/') as $value) {
foreach ($this->getValues('counter', $metaData) as $value) {
$parts = explode(':', $value['key']);
$labelValues = $parts[3];
$data['samples'][] = [
Expand All @@ -235,13 +360,71 @@ private function collectCounters(): array
return $counters;
}

/**
* When given a type ('histogram', 'gauge', or 'counter'), return an iterable array of matching records retrieved from APCu
*
* @param string $type
* @return array<array>
*/
private function getMetas(string $type): array
{
$arr = [];
$root = apcu_fetch($this->rootNode());
if (is_array($root)) {
foreach ($root as $metaKey) {
if ((1 === preg_match('/' . self::PROMETHEUS_PREFIX . ':' . $type . ':.*:meta/', $metaKey)) && false !== ($gauge = apcu_fetch($metaKey))) {
$arr[] = [ 'key' => $metaKey, 'value' => $gauge ];
}
}
}
return $arr;
}

/**
* When given a type ('histogram', 'gauge', or 'counter') and metaData array, return an iterable array of matching records retrieved from APCu
*
* @param string $type
* @param array<mixed> $metaData
* @return array<array>
*/
private function getValues(string $type, array $metaData): array
{
$labels = $arr = [];
foreach (array_values($metaData['labelNames']) as $label) {
$labelKey = $this->assembleLabelKey($metaData, $label);
if (is_array($tmp = apcu_fetch($labelKey))) {
$labels[] = $tmp;
}
}
// Append the histogram bucket-list and the histogram-specific label 'sum' to labels[] then generate the permutations
if (isset($metaData['buckets'])) {
$metaData['buckets'][] = 'sum';
$labels[] = $metaData['buckets'];
$metaData['labelNames'][] = '__histogram_buckets';
}
$labelValuesList = $this->buildPermutationTree($metaData['labelNames'], $labels);
unset($labels);
$histogramBucket = '';
foreach ($labelValuesList as $labelValues) {
// Extract bucket value from permuted element, if present, then construct the key and retrieve
if (isset($metaData['buckets'])) {
$histogramBucket = ':' . array_pop($labelValues);
}
$key = self::PROMETHEUS_PREFIX . ":{$type}:{$metaData['name']}:" . $this->encodeLabelValues($labelValues) . $histogramBucket . ':value';
if (false !== ($value = apcu_fetch($key))) {
$arr[] = [ 'key' => $key, 'value' => $value ];
}
}
return $arr;
}

/**
* @return MetricFamilySamples[]
*/
private function collectGauges(): array
{
$gauges = [];
foreach (new APCUIterator('/^prom:gauge:.*:meta/') as $gauge) {
foreach ($this->getMetas('gauge') as $gauge) {
$metaData = json_decode($gauge['value'], true);
$data = [
'name' => $metaData['name'],
Expand All @@ -250,7 +433,7 @@ private function collectGauges(): array
'labelNames' => $metaData['labelNames'],
'samples' => [],
];
foreach (new APCUIterator('/^prom:gauge:' . $metaData['name'] . ':.*:value/') as $value) {
foreach ($this->getValues('gauge', $metaData) as $value) {
$parts = explode(':', $value['key']);
$labelValues = $parts[3];
$data['samples'][] = [
Expand All @@ -260,7 +443,6 @@ private function collectGauges(): array
'value' => $this->fromBinaryRepresentationAsInteger($value['value']),
];
}

$this->sortSamples($data['samples']);
$gauges[] = new MetricFamilySamples($data);
}
Expand All @@ -273,8 +455,12 @@ private function collectGauges(): array
private function collectHistograms(): array
{
$histograms = [];
foreach (new APCUIterator('/^prom:histogram:.*:meta/') as $histogram) {
foreach ($this->getMetas('histogram') as $histogram) {
$metaData = json_decode($histogram['value'], true);

// Add the Inf bucket so we can compute it later on
$metaData['buckets'][] = '+Inf';

$data = [
'name' => $metaData['name'],
'help' => $metaData['help'],
Expand All @@ -283,11 +469,8 @@ private function collectHistograms(): array
'buckets' => $metaData['buckets'],
];

// Add the Inf bucket so we can compute it later on
$data['buckets'][] = '+Inf';

$histogramBuckets = [];
foreach (new APCUIterator('/^prom:histogram:' . $metaData['name'] . ':.*:value/') as $value) {
foreach ($this->getValues('histogram', $metaData) as $value) {
$parts = explode(':', $value['key']);
$labelValues = $parts[3];
$bucket = $parts[4];
Expand Down Expand Up @@ -417,4 +600,12 @@ private function decodeLabelValues(string $values): array
}
return $decodedValues;
}

/**
* @return string
*/
private function rootNode(): string
{
return implode(':', [ self::PROMETHEUS_PREFIX, 'rootnode', ]);
}
}