Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/Usage/Adapter/ClickHouse.php
Original file line number Diff line number Diff line change
Expand Up @@ -1215,6 +1215,8 @@ private function getColumnType(string $id, string $type = 'event'): string
'connectionUsageType', 'autonomousSystemNumber',
// sdk identity
'sdk', 'sdkVersion',
// gauge replica ordinal
'ordinal',
];

if (in_array($id, $lowCardinality, true)) {
Expand Down
19 changes: 18 additions & 1 deletion src/Usage/Adapter/Database.php
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,8 @@ public function setup(): void
// Event schema is a superset of the gauge schema for the dimensions
// that exist in both (resourceId, resourceInternalId, teamId,
// teamInternalId), so a single Database collection backed by the
// event schema works for both types.
// event schema works for both types. Gauge-only columns (ordinal)
// are appended below.
$attributes = $this->getAttributeDocuments('event');
$indexDocs = $this->getIndexDocuments('event');

Expand All @@ -97,6 +98,22 @@ public function setup(): void
'attributes' => ['type'],
]);

// Gauge-only replica ordinal dimension.
$attributes[] = new Document([
'$id' => 'ordinal',
'type' => 'string',
'size' => 255,
'required' => false,
'signed' => true,
'array' => false,
'filters' => [],
]);
$indexDocs[] = new Document([
'$id' => 'index-ordinal',
'type' => 'key',
'attributes' => ['ordinal'],
]);

try {
$this->db->createCollection(
$this->collection,
Expand Down
17 changes: 14 additions & 3 deletions src/Usage/Metric.php
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ class Metric extends ArrayObject
/**
* Gauge-specific column names that are extracted from tags into dedicated columns.
*/
public const GAUGE_COLUMNS = ['service', 'resourceType', 'teamId', 'teamInternalId', 'resourceId', 'resourceInternalId'];
public const GAUGE_COLUMNS = ['service', 'resourceType', 'teamId', 'teamInternalId', 'resourceId', 'resourceInternalId', 'ordinal'];

/**
* Construct a new metric object.
Expand Down Expand Up @@ -99,6 +99,7 @@ class Metric extends ArrayObject
*
* Gauge-only dimension columns (see GAUGE_COLUMNS):
* - teamId / teamInternalId / resourceId / resourceInternalId
* - ordinal: replica ordinal for multi-node resources (0 is the primary)
*
* @param array<string, mixed> $input Metric data
*/
Expand Down Expand Up @@ -279,6 +280,15 @@ public function getResourceInternalId(): ?string
return is_string($v) ? $v : null;
}

/**
* Get replica ordinal (gauge metrics). 0 is the primary; 1+ are replicas.
*/
public function getOrdinal(): ?string
{
$v = $this->getAttribute('ordinal', null);
return is_string($v) ? $v : null;
}

/**
* Get team id (event/gauge metrics).
*/
Expand Down Expand Up @@ -721,6 +731,7 @@ public static function getGaugeSchema(): array
$stringColumn('teamInternalId', 255),
$stringColumn('resourceId', 255),
$stringColumn('resourceInternalId', 255),
$stringColumn('ordinal', 255),
];
}

Expand Down Expand Up @@ -778,9 +789,9 @@ static function (string $col) use ($setIndexed): array {
*/
public static function getGaugeIndexes(): array
{
$indexed = ['service', 'resourceType', 'resourceId', 'resourceInternalId', 'teamId', 'teamInternalId'];
$indexed = ['service', 'resourceType', 'resourceId', 'resourceInternalId', 'teamId', 'teamInternalId', 'ordinal'];

$setIndexed = ['service', 'resourceType'];
$setIndexed = ['service', 'resourceType', 'ordinal'];

return array_map(
static fn (string $col): array => [
Expand Down
21 changes: 17 additions & 4 deletions tests/Usage/Adapter/ClickHouseColumnTypeTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,11 @@ protected function setUp(): void
$this->getColumnType->setAccessible(true);
}

private function columnType(string $id): string
private function columnType(string $id, string $type = 'event'): string
{
/** @var string $type */
$type = $this->getColumnType->invoke($this->adapter, $id, 'event');
return $type;
/** @var string $columnType */
$columnType = $this->getColumnType->invoke($this->adapter, $id, $type);
return $columnType;
}

/**
Expand Down Expand Up @@ -86,4 +86,17 @@ public function testLowCardinalitySdkColumns(): void
);
}
}

/**
* The gauge replica ordinal holds a handful of distinct values, so it
* must map to LowCardinality(Nullable(String)).
*/
public function testLowCardinalityOrdinalColumn(): void
{
$this->assertSame(
'LowCardinality(Nullable(String))',
$this->columnType('ordinal', 'gauge'),
'ordinal should be LowCardinality(Nullable(String))'
);
}
}
1 change: 1 addition & 0 deletions tests/Usage/Adapter/ClickHouseSchemaTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,7 @@ private function expectedDimAssertions(array $columns, string $type): array
'continentCode', 'subdivisions', 'connectionType',
'connectionUsageType', 'autonomousSystemNumber',
'sdk', 'sdkVersion',
'ordinal',
];

$baseKey = ['id', 'metric', 'value', 'time', 'tenant'];
Expand Down
50 changes: 50 additions & 0 deletions tests/Usage/Adapter/ClickHouseTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -314,6 +314,7 @@ public function testGaugeColumnsRoundTrip(): void
'teamInternalId' => '7',
'resourceId' => 'r1',
'resourceInternalId' => '42',
'ordinal' => '0',
],
],
], Usage::TYPE_GAUGE));
Expand All @@ -330,6 +331,55 @@ public function testGaugeColumnsRoundTrip(): void
$this->assertEquals('7', $metric->getTeamInternalId());
$this->assertEquals('r1', $metric->getResourceId());
$this->assertEquals('42', $metric->getResourceInternalId());
$this->assertEquals('0', $metric->getOrdinal());
}

/**
* Per-replica gauge rows for one resource stay distinct series via the
* ordinal dimension: filtering by ordinal isolates one node, grouping by
* ordinal returns the latest snapshot per node.
*/
public function testGaugeOrdinalSeparatesReplicaSeries(): void
{
$this->usage->purge('1', [], Usage::TYPE_GAUGE);

$this->assertTrue($this->usage->addBatch([
[
'tenant' => '1',
'metric' => 'gauge-ordinal-test',
'value' => 10,
'tags' => ['resourceType' => 'databases', 'resourceId' => 'db1', 'ordinal' => '0'],
],
[
'tenant' => '1',
'metric' => 'gauge-ordinal-test',
'value' => 20,
'tags' => ['resourceType' => 'databases', 'resourceId' => 'db1', 'ordinal' => '1'],
],
], Usage::TYPE_GAUGE));

$primary = $this->usage->find('1', [
\Utopia\Query\Query::equal('metric', ['gauge-ordinal-test']),
\Utopia\Query\Query::equal('ordinal', ['0']),
], Usage::TYPE_GAUGE);

$this->assertCount(1, $primary);
$this->assertEquals(10, $primary[0]->getValue());
$this->assertEquals('0', $primary[0]->getOrdinal());

$perNode = $this->usage->find('1', [
\Utopia\Query\Query::equal('metric', ['gauge-ordinal-test']),
UsageQuery::groupBy('ordinal'),
], Usage::TYPE_GAUGE);

$this->assertCount(2, $perNode);
$byOrdinal = [];
foreach ($perNode as $row) {
$ordinal = $row->getOrdinal();
$this->assertNotNull($ordinal);
$byOrdinal[$ordinal] = (int) $row->getValue();
}
$this->assertEquals(['0' => 10, '1' => 20], $byOrdinal);
}

public function testUnknownTagKeyThrows(): void
Expand Down
30 changes: 29 additions & 1 deletion tests/Usage/MetricTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -626,10 +626,38 @@ public function testEventColumnsConstant(): void
*/
public function testGaugeColumnsConstant(): void
{
$expected = ['service', 'resourceType', 'teamId', 'teamInternalId', 'resourceId', 'resourceInternalId'];
$expected = ['service', 'resourceType', 'teamId', 'teamInternalId', 'resourceId', 'resourceInternalId', 'ordinal'];
$this->assertSame($expected, Metric::GAUGE_COLUMNS);
}

/**
* The replica ordinal is a gauge-only dimension: present in GAUGE_COLUMNS,
* the gauge schema and gauge indexes, extracted from tags into its column,
* and readable via the typed accessor. Events must not carry it.
*/
public function testOrdinalIsGaugeOnly(): void
{
$this->assertContains('ordinal', Metric::GAUGE_COLUMNS);
$this->assertNotContains('ordinal', Metric::EVENT_COLUMNS);

$gaugeIds = array_column(Metric::getGaugeSchema(), '$id');
$this->assertContains('ordinal', $gaugeIds);

$eventIds = array_column(Metric::getEventSchema(), '$id');
$this->assertNotContains('ordinal', $eventIds);

$indexIds = array_column(Metric::getGaugeIndexes(), '$id');
$this->assertContains('index-ordinal', $indexIds);

$columns = Metric::extractColumns(['resourceId' => 'db_a', 'ordinal' => 1], 'gauge');
$this->assertSame('1', $columns['ordinal']);
$this->assertSame('db_a', $columns['resourceId']);

$metric = new Metric(['ordinal' => '2']);
$this->assertSame('2', $metric->getOrdinal());
$this->assertNull((new Metric([]))->getOrdinal());
}

/**
* Test that the event schema contains every new dimension column.
*/
Expand Down
Loading