Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -8,19 +8,27 @@


class BlobStorageIntegrationFileType(enum.StrEnum):
"""
File format for exported data. `PARQUET` is a columnar binary format encoded and compressed by the storage engine; gzip compression does not apply to it. Note that the model-price columns (`input_price`, `output_price`, `total_price`) are not included in Parquet observation exports.
"""

JSON = "JSON"
CSV = "CSV"
JSONL = "JSONL"
PARQUET = "PARQUET"

def visit(
self,
json: typing.Callable[[], T_Result],
csv: typing.Callable[[], T_Result],
jsonl: typing.Callable[[], T_Result],
parquet: typing.Callable[[], T_Result],
) -> T_Result:
if self is BlobStorageIntegrationFileType.JSON:
return json()
if self is BlobStorageIntegrationFileType.CSV:
return csv()
if self is BlobStorageIntegrationFileType.JSONL:
return jsonl()
if self is BlobStorageIntegrationFileType.PARQUET:
return parquet()
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

class BlobStorageIntegrationFileTypeResponse(enum.StrEnum):
"""
File type reported for an existing integration. Includes `PARQUET`, which a project may enable through the Langfuse UI but cannot yet be set via this API (the request `fileType` omits it).
File type reported for an existing integration.
"""

JSON = "JSON"
Expand Down
12 changes: 8 additions & 4 deletions langfuse/api/trace/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ def list(
```json
[
{
"type": string, // Required. One of: "datetime", "string", "number", "stringOptions", "categoryOptions", "arrayOptions", "stringObject", "numberObject", "boolean", "null"
"type": string, // Required. One of: "datetime", "string", "number", "stringOptions", "categoryOptions", "arrayOptions", "stringObject", "numberObject", "booleanObject", "boolean", "null"
"column": string, // Required. Column to filter on (see available columns below)
"operator": string, // Required. Operator based on type:
// - datetime: ">", "<", ">=", "<="
Expand All @@ -191,10 +191,11 @@ def list(
// - number: "=", ">", "<", ">=", "<="
// - stringObject: "=", "contains", "does not contain", "starts with", "ends with"
// - numberObject: "=", ">", "<", ">=", "<="
// - booleanObject: "=", "<>"
// - boolean: "=", "<>"
// - null: "is null", "is not null"
"value": any, // Required (except for null type). Value to compare against. Type depends on filter type
"key": string // Required only for stringObject, numberObject, and categoryOptions types when filtering on nested fields like metadata
"key": string // Required only for stringObject, numberObject, booleanObject, and categoryOptions types when filtering on nested fields like metadata or score names
}
]
```
Expand Down Expand Up @@ -237,6 +238,7 @@ def list(
### Scores (requires join with scores table)
- `scores_avg` (number) - Average of numeric scores (alias: `scores`)
- `score_categories` (categoryOptions) - Categorical score values
- `score_booleans` (booleanObject) - Boolean score values. Use `key` for the score name and a boolean `value`, e.g. `{"type": "booleanObject", "column": "score_booleans", "key": "is_correct", "operator": "=", "value": true}`. The `<>` operator also matches traces without a score of that name.

## Filter Examples
```json
Expand Down Expand Up @@ -546,7 +548,7 @@ async def list(
```json
[
{
"type": string, // Required. One of: "datetime", "string", "number", "stringOptions", "categoryOptions", "arrayOptions", "stringObject", "numberObject", "boolean", "null"
"type": string, // Required. One of: "datetime", "string", "number", "stringOptions", "categoryOptions", "arrayOptions", "stringObject", "numberObject", "booleanObject", "boolean", "null"
"column": string, // Required. Column to filter on (see available columns below)
"operator": string, // Required. Operator based on type:
// - datetime: ">", "<", ">=", "<="
Expand All @@ -557,10 +559,11 @@ async def list(
// - number: "=", ">", "<", ">=", "<="
// - stringObject: "=", "contains", "does not contain", "starts with", "ends with"
// - numberObject: "=", ">", "<", ">=", "<="
// - booleanObject: "=", "<>"
// - boolean: "=", "<>"
// - null: "is null", "is not null"
"value": any, // Required (except for null type). Value to compare against. Type depends on filter type
"key": string // Required only for stringObject, numberObject, and categoryOptions types when filtering on nested fields like metadata
"key": string // Required only for stringObject, numberObject, booleanObject, and categoryOptions types when filtering on nested fields like metadata or score names
}
]
```
Expand Down Expand Up @@ -603,6 +606,7 @@ async def list(
### Scores (requires join with scores table)
- `scores_avg` (number) - Average of numeric scores (alias: `scores`)
- `score_categories` (categoryOptions) - Categorical score values
- `score_booleans` (booleanObject) - Boolean score values. Use `key` for the score name and a boolean `value`, e.g. `{"type": "booleanObject", "column": "score_booleans", "key": "is_correct", "operator": "=", "value": true}`. The `<>` operator also matches traces without a score of that name.

## Filter Examples
```json
Expand Down
12 changes: 8 additions & 4 deletions langfuse/api/trace/raw_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -308,7 +308,7 @@ def list(
```json
[
{
"type": string, // Required. One of: "datetime", "string", "number", "stringOptions", "categoryOptions", "arrayOptions", "stringObject", "numberObject", "boolean", "null"
"type": string, // Required. One of: "datetime", "string", "number", "stringOptions", "categoryOptions", "arrayOptions", "stringObject", "numberObject", "booleanObject", "boolean", "null"
"column": string, // Required. Column to filter on (see available columns below)
"operator": string, // Required. Operator based on type:
// - datetime: ">", "<", ">=", "<="
Expand All @@ -319,10 +319,11 @@ def list(
// - number: "=", ">", "<", ">=", "<="
// - stringObject: "=", "contains", "does not contain", "starts with", "ends with"
// - numberObject: "=", ">", "<", ">=", "<="
// - booleanObject: "=", "<>"
// - boolean: "=", "<>"
// - null: "is null", "is not null"
"value": any, // Required (except for null type). Value to compare against. Type depends on filter type
"key": string // Required only for stringObject, numberObject, and categoryOptions types when filtering on nested fields like metadata
"key": string // Required only for stringObject, numberObject, booleanObject, and categoryOptions types when filtering on nested fields like metadata or score names
}
]
```
Expand Down Expand Up @@ -365,6 +366,7 @@ def list(
### Scores (requires join with scores table)
- `scores_avg` (number) - Average of numeric scores (alias: `scores`)
- `score_categories` (categoryOptions) - Categorical score values
- `score_booleans` (booleanObject) - Boolean score values. Use `key` for the score name and a boolean `value`, e.g. `{"type": "booleanObject", "column": "score_booleans", "key": "is_correct", "operator": "=", "value": true}`. The `<>` operator also matches traces without a score of that name.

## Filter Examples
```json
Expand Down Expand Up @@ -910,7 +912,7 @@ async def list(
```json
[
{
"type": string, // Required. One of: "datetime", "string", "number", "stringOptions", "categoryOptions", "arrayOptions", "stringObject", "numberObject", "boolean", "null"
"type": string, // Required. One of: "datetime", "string", "number", "stringOptions", "categoryOptions", "arrayOptions", "stringObject", "numberObject", "booleanObject", "boolean", "null"
"column": string, // Required. Column to filter on (see available columns below)
"operator": string, // Required. Operator based on type:
// - datetime: ">", "<", ">=", "<="
Expand All @@ -921,10 +923,11 @@ async def list(
// - number: "=", ">", "<", ">=", "<="
// - stringObject: "=", "contains", "does not contain", "starts with", "ends with"
// - numberObject: "=", ">", "<", ">=", "<="
// - booleanObject: "=", "<>"
// - boolean: "=", "<>"
// - null: "is null", "is not null"
"value": any, // Required (except for null type). Value to compare against. Type depends on filter type
"key": string // Required only for stringObject, numberObject, and categoryOptions types when filtering on nested fields like metadata
"key": string // Required only for stringObject, numberObject, booleanObject, and categoryOptions types when filtering on nested fields like metadata or score names
}
]
```
Expand Down Expand Up @@ -967,6 +970,7 @@ async def list(
### Scores (requires join with scores table)
- `scores_avg` (number) - Average of numeric scores (alias: `scores`)
- `score_categories` (categoryOptions) - Categorical score values
- `score_booleans` (booleanObject) - Boolean score values. Use `key` for the score name and a boolean `value`, e.g. `{"type": "booleanObject", "column": "score_booleans", "key": "is_correct", "operator": "=", "value": true}`. The `<>` operator also matches traces without a score of that name.

## Filter Examples
```json
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,8 @@ class EvaluationRuleMapping(UniversalBaseModel):
Source field that should populate the prompt variable.

Quick reference:
- `target=observation`: `input`, `output`, `metadata`
- `target=experiment`: `input`, `output`, `metadata`, `expected_output`, `experiment_item_metadata`
- `target=observation`: `input`, `output`, `metadata`, `tool_calls`
- `target=experiment`: `input`, `output`, `metadata`, `tool_calls`, `expected_output`, `experiment_item_metadata`
"""

json_path: typing_extensions.Annotated[
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,20 +14,22 @@ class EvaluationRuleMappingSource(enum.StrEnum):
Use these values when mapping evaluator prompt variables to live data.

Target-specific rules:
- `target=observation` supports `input`, `output`, and `metadata`
- `target=experiment` supports `input`, `output`, `metadata`, `expected_output`, and `experiment_item_metadata`
- `target=observation` supports `input`, `output`, `metadata`, and `tool_calls`
- `target=experiment` supports `input`, `output`, `metadata`, `tool_calls`, `expected_output`, and `experiment_item_metadata`

Source semantics:
- `input`: the observation or experiment input payload
- `output`: the observation or experiment output payload
- `metadata`: the metadata object for the target. Combine with `jsonPath` when you need one nested field instead of the whole object.
- `tool_calls`: the tool calls recorded on the observation, as an array of `{id, name, arguments, type, index}` objects in the order the model emitted them. Combine with `jsonPath` (for example `$[*].name`) to select parts of each call.
- `expected_output`: the experiment item's expected output. Only valid for `target=experiment`.
- `experiment_item_metadata`: the experiment item's metadata object. Only valid for `target=experiment`.
"""

INPUT = "input"
OUTPUT = "output"
METADATA = "metadata"
TOOL_CALLS = "tool_calls"
EXPECTED_OUTPUT = "expected_output"
EXPERIMENT_ITEM_METADATA = "experiment_item_metadata"

Expand All @@ -36,6 +38,7 @@ def visit(
input: typing.Callable[[], T_Result],
output: typing.Callable[[], T_Result],
metadata: typing.Callable[[], T_Result],
tool_calls: typing.Callable[[], T_Result],
expected_output: typing.Callable[[], T_Result],
experiment_item_metadata: typing.Callable[[], T_Result],
) -> T_Result:
Expand All @@ -45,6 +48,8 @@ def visit(
return output()
if self is EvaluationRuleMappingSource.METADATA:
return metadata()
if self is EvaluationRuleMappingSource.TOOL_CALLS:
return tool_calls()
if self is EvaluationRuleMappingSource.EXPECTED_OUTPUT:
return expected_output()
if self is EvaluationRuleMappingSource.EXPERIMENT_ITEM_METADATA:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ class EvaluationRuleTarget(enum.StrEnum):

Choose the target first, because it changes both the valid filter columns and the valid variable-mapping sources:
- `observation` evaluates live-ingested observations such as generations, spans, and events.
It supports mapping from `input`, `output`, and `metadata`.
It supports mapping from `input`, `output`, `metadata`, and `tool_calls`.
- `experiment` evaluates live experiment executions and can additionally map `expected_output` and `experiment_item_metadata`.
It currently supports filtering by `datasetId`.
Discover valid dataset IDs with `GET /api/public/v2/datasets`, then use the returned dataset `id` values in your filter.
Expand Down