274 lines
7.9 KiB
TypeScript
274 lines
7.9 KiB
TypeScript
import { APIResource } from "../../core/resource.js";
|
|
import * as ResponsesAPI from "../responses/responses.js";
|
|
export declare class GraderModels extends APIResource {
|
|
}
|
|
/**
|
|
* A LabelModelGrader object which uses a model to assign labels to each item in
|
|
* the evaluation.
|
|
*/
|
|
export interface LabelModelGrader {
|
|
input: Array<LabelModelGrader.Input>;
|
|
/**
|
|
* The labels to assign to each item in the evaluation.
|
|
*/
|
|
labels: Array<string>;
|
|
/**
|
|
* The model to use for the evaluation. Must support structured outputs.
|
|
*/
|
|
model: string;
|
|
/**
|
|
* The name of the grader.
|
|
*/
|
|
name: string;
|
|
/**
|
|
* The labels that indicate a passing result. Must be a subset of labels.
|
|
*/
|
|
passing_labels: Array<string>;
|
|
/**
|
|
* The object type, which is always `label_model`.
|
|
*/
|
|
type: 'label_model';
|
|
}
|
|
export declare namespace LabelModelGrader {
|
|
/**
|
|
* A message input to the model with a role indicating instruction following
|
|
* hierarchy. Instructions given with the `developer` or `system` role take
|
|
* precedence over instructions given with the `user` role. Messages with the
|
|
* `assistant` role are presumed to have been generated by the model in previous
|
|
* interactions.
|
|
*/
|
|
interface Input {
|
|
/**
|
|
* Inputs to the model - can contain template strings.
|
|
*/
|
|
content: string | ResponsesAPI.ResponseInputText | Input.OutputText | Input.InputImage | Array<unknown>;
|
|
/**
|
|
* The role of the message input. One of `user`, `assistant`, `system`, or
|
|
* `developer`.
|
|
*/
|
|
role: 'user' | 'assistant' | 'system' | 'developer';
|
|
/**
|
|
* The type of the message input. Always `message`.
|
|
*/
|
|
type?: 'message';
|
|
}
|
|
namespace Input {
|
|
/**
|
|
* A text output from the model.
|
|
*/
|
|
interface OutputText {
|
|
/**
|
|
* The text output from the model.
|
|
*/
|
|
text: string;
|
|
/**
|
|
* The type of the output text. Always `output_text`.
|
|
*/
|
|
type: 'output_text';
|
|
}
|
|
/**
|
|
* An image input to the model.
|
|
*/
|
|
interface InputImage {
|
|
/**
|
|
* The URL of the image input.
|
|
*/
|
|
image_url: string;
|
|
/**
|
|
* The type of the image input. Always `input_image`.
|
|
*/
|
|
type: 'input_image';
|
|
/**
|
|
* The detail level of the image to be sent to the model. One of `high`, `low`, or
|
|
* `auto`. Defaults to `auto`.
|
|
*/
|
|
detail?: string;
|
|
}
|
|
}
|
|
}
|
|
/**
|
|
* A MultiGrader object combines the output of multiple graders to produce a single
|
|
* score.
|
|
*/
|
|
export interface MultiGrader {
|
|
/**
|
|
* A formula to calculate the output based on grader results.
|
|
*/
|
|
calculate_output: string;
|
|
/**
|
|
* A StringCheckGrader object that performs a string comparison between input and
|
|
* reference using a specified operation.
|
|
*/
|
|
graders: StringCheckGrader | TextSimilarityGrader | PythonGrader | ScoreModelGrader | LabelModelGrader;
|
|
/**
|
|
* The name of the grader.
|
|
*/
|
|
name: string;
|
|
/**
|
|
* The object type, which is always `multi`.
|
|
*/
|
|
type: 'multi';
|
|
}
|
|
/**
|
|
* A PythonGrader object that runs a python script on the input.
|
|
*/
|
|
export interface PythonGrader {
|
|
/**
|
|
* The name of the grader.
|
|
*/
|
|
name: string;
|
|
/**
|
|
* The source code of the python script.
|
|
*/
|
|
source: string;
|
|
/**
|
|
* The object type, which is always `python`.
|
|
*/
|
|
type: 'python';
|
|
/**
|
|
* The image tag to use for the python script.
|
|
*/
|
|
image_tag?: string;
|
|
}
|
|
/**
|
|
* A ScoreModelGrader object that uses a model to assign a score to the input.
|
|
*/
|
|
export interface ScoreModelGrader {
|
|
/**
|
|
* The input text. This may include template strings.
|
|
*/
|
|
input: Array<ScoreModelGrader.Input>;
|
|
/**
|
|
* The model to use for the evaluation.
|
|
*/
|
|
model: string;
|
|
/**
|
|
* The name of the grader.
|
|
*/
|
|
name: string;
|
|
/**
|
|
* The object type, which is always `score_model`.
|
|
*/
|
|
type: 'score_model';
|
|
/**
|
|
* The range of the score. Defaults to `[0, 1]`.
|
|
*/
|
|
range?: Array<number>;
|
|
/**
|
|
* The sampling parameters for the model.
|
|
*/
|
|
sampling_params?: unknown;
|
|
}
|
|
export declare namespace ScoreModelGrader {
|
|
/**
|
|
* A message input to the model with a role indicating instruction following
|
|
* hierarchy. Instructions given with the `developer` or `system` role take
|
|
* precedence over instructions given with the `user` role. Messages with the
|
|
* `assistant` role are presumed to have been generated by the model in previous
|
|
* interactions.
|
|
*/
|
|
interface Input {
|
|
/**
|
|
* Inputs to the model - can contain template strings.
|
|
*/
|
|
content: string | ResponsesAPI.ResponseInputText | Input.OutputText | Input.InputImage | Array<unknown>;
|
|
/**
|
|
* The role of the message input. One of `user`, `assistant`, `system`, or
|
|
* `developer`.
|
|
*/
|
|
role: 'user' | 'assistant' | 'system' | 'developer';
|
|
/**
|
|
* The type of the message input. Always `message`.
|
|
*/
|
|
type?: 'message';
|
|
}
|
|
namespace Input {
|
|
/**
|
|
* A text output from the model.
|
|
*/
|
|
interface OutputText {
|
|
/**
|
|
* The text output from the model.
|
|
*/
|
|
text: string;
|
|
/**
|
|
* The type of the output text. Always `output_text`.
|
|
*/
|
|
type: 'output_text';
|
|
}
|
|
/**
|
|
* An image input to the model.
|
|
*/
|
|
interface InputImage {
|
|
/**
|
|
* The URL of the image input.
|
|
*/
|
|
image_url: string;
|
|
/**
|
|
* The type of the image input. Always `input_image`.
|
|
*/
|
|
type: 'input_image';
|
|
/**
|
|
* The detail level of the image to be sent to the model. One of `high`, `low`, or
|
|
* `auto`. Defaults to `auto`.
|
|
*/
|
|
detail?: string;
|
|
}
|
|
}
|
|
}
|
|
/**
|
|
* A StringCheckGrader object that performs a string comparison between input and
|
|
* reference using a specified operation.
|
|
*/
|
|
export interface StringCheckGrader {
|
|
/**
|
|
* The input text. This may include template strings.
|
|
*/
|
|
input: string;
|
|
/**
|
|
* The name of the grader.
|
|
*/
|
|
name: string;
|
|
/**
|
|
* The string check operation to perform. One of `eq`, `ne`, `like`, or `ilike`.
|
|
*/
|
|
operation: 'eq' | 'ne' | 'like' | 'ilike';
|
|
/**
|
|
* The reference text. This may include template strings.
|
|
*/
|
|
reference: string;
|
|
/**
|
|
* The object type, which is always `string_check`.
|
|
*/
|
|
type: 'string_check';
|
|
}
|
|
/**
|
|
* A TextSimilarityGrader object which grades text based on similarity metrics.
|
|
*/
|
|
export interface TextSimilarityGrader {
|
|
/**
|
|
* The evaluation metric to use. One of `cosine`, `fuzzy_match`, `bleu`, `gleu`,
|
|
* `meteor`, `rouge_1`, `rouge_2`, `rouge_3`, `rouge_4`, `rouge_5`, or `rouge_l`.
|
|
*/
|
|
evaluation_metric: 'cosine' | 'fuzzy_match' | 'bleu' | 'gleu' | 'meteor' | 'rouge_1' | 'rouge_2' | 'rouge_3' | 'rouge_4' | 'rouge_5' | 'rouge_l';
|
|
/**
|
|
* The text being graded.
|
|
*/
|
|
input: string;
|
|
/**
|
|
* The name of the grader.
|
|
*/
|
|
name: string;
|
|
/**
|
|
* The text being graded against.
|
|
*/
|
|
reference: string;
|
|
/**
|
|
* The type of grader.
|
|
*/
|
|
type: 'text_similarity';
|
|
}
|
|
export declare namespace GraderModels {
|
|
export { type LabelModelGrader as LabelModelGrader, type MultiGrader as MultiGrader, type PythonGrader as PythonGrader, type ScoreModelGrader as ScoreModelGrader, type StringCheckGrader as StringCheckGrader, type TextSimilarityGrader as TextSimilarityGrader, };
|
|
}
|
|
//# sourceMappingURL=grader-models.d.ts.map
|