Gateway Config Object

The config object is used to configure API interactions with various providers. It supports multiple modes such as single provider access, load balancing between providers, and fallback strategies. The following JSON schema is used to validate the config object:

Portkey Config JSON Schema

{
  $schema: 'http://json-schema.org/draft-07/schema#',
  type: 'object',
  properties: {
  	after_request_hooks: {
  		type: 'array',
  		items: {
  			properties: {
  				id: {
  					type: 'string',
  				},
  				type: {
  					type: 'string',
  				},
  				async: {
  					type: 'boolean',
  				},
  				on_fail: {
  					type: 'object',
  					properties: {
  						feedback: {
  							type: 'object',
  							properties: {
  								value: {
  									type: 'number',
  								},
  								weight: {
  									type: 'number',
  								},
  								metadata: {
  									type: 'object',
  								},
  							},
  						},
  					},
  				},
  				on_success: {
  					type: 'object',
  					properties: {
  						feedback: {
  							type: 'object',
  							properties: {
  								value: {
  									type: 'number',
  								},
  								weight: {
  									type: 'number',
  								},
  								metadata: {
  									type: 'object',
  								},
  							},
  						},
  					},
  				},
  				checks: {
  					type: 'array',
  					items: {
  						type: 'object',
  						properties: {
  							id: {
  								type: 'string',
  							},
  							parameters: {
  								type: 'object',
  							},
  						},
  						required: ['id', 'parameters'],
  					},
  				},
  			},
  			required: ['id'],
  		},
  	},
  	input_guardrails: {
  		type: 'array',
  		items: {
  			oneOf: [
  				{
  					type: 'object',
  					properties: {
  						id: {
  							type: 'string',
  						},
  						deny: {
  							type: 'boolean',
  						},
  						on_fail: {
  							type: 'object',
  							properties: {
  								feedback: {
  									type: 'object',
  									properties: {
  										value: {
  											type: 'number',
  										},
  										weight: {
  											type: 'number',
  										},
  										metadata: {
  											type: 'object',
  										},
  									},
  								},
  							},
  						},
  						on_success: {
  							type: 'object',
  							properties: {
  								feedback: {
  									type: 'object',
  									properties: {
  										value: {
  											type: 'number',
  										},
  										weight: {
  											type: 'number',
  										},
  										metadata: {
  											type: 'object',
  										},
  									},
  								},
  							},
  						},
  						async: {
  							type: 'boolean',
  						},
  					},
  					additionalProperties: {
  						type: 'object',
  						additionalProperties: true,
  					},
  				},
  				{
  					type: 'string',
  				},
  			],
  		},
  	},
  	output_guardrails: {
  		type: 'array',
  		items: {
  			oneOf: [
  				{
  					type: 'object',
  					properties: {
  						id: {
  							type: 'string',
  						},
  						deny: {
  							type: 'boolean',
  						},
  						on_fail: {
  							type: 'object',
  							properties: {
  								feedback: {
  									type: 'object',
  									properties: {
  										value: {
  											type: 'number',
  										},
  										weight: {
  											type: 'number',
  										},
  										metadata: {
  											type: 'object',
  										},
  									},
  								},
  								deny: {
  									type: 'boolean',
  								},
  							},
  						},
  						on_success: {
  							type: 'object',
  							properties: {
  								feedback: {
  									type: 'object',
  									properties: {
  										value: {
  											type: 'number',
  										},
  										weight: {
  											type: 'number',
  										},
  										metadata: {
  											type: 'object',
  										},
  									},
  								},
  								deny: {
  									type: 'boolean',
  								},
  							},
  						},
  						async: {
  							type: 'boolean',
  						},
  					},
  					additionalProperties: {
  						type: 'object',
  						additionalProperties: true,
  					},
  				},
  				{
  					type: 'string',
  				},
  			],
  		},
  	},
  	before_request_hooks: {
  		type: 'array',
  		items: {
  			properties: {
  				id: {
  					type: 'string',
  				},
  				type: {
  					type: 'string',
  				},
  				on_fail: {
  					type: 'object',
  					properties: {
  						feedback: {
  							type: 'object',
  							properties: {
  								value: {
  									type: 'number',
  								},
  								weight: {
  									type: 'number',
  								},
  								metadata: {
  									type: 'object',
  								},
  							},
  						},
  						deny: {
  							type: 'boolean',
  						},
  					},
  				},
  				on_success: {
  					type: 'object',
  					properties: {
  						feedback: {
  							type: 'object',
  							properties: {
  								value: {
  									type: 'number',
  								},
  								weight: {
  									type: 'number',
  								},
  								metadata: {
  									type: 'object',
  								},
  							},
  						},
  						deny: {
  							type: 'boolean',
  						},
  					},
  				},
  				checks: {
  					type: 'array',
  					items: {
  						type: 'object',
  						properties: {
  							id: {
  								type: 'string',
  							},
  							parameters: {
  								type: 'object',
  							},
  						},
  						required: ['id', 'parameters'],
  					},
  				},
  			},
  			required: ['id'],
  		},
  	},
  	strategy: {
  		type: 'object',
  		properties: {
  			mode: {
  				type: 'string',
  				enum: ['single', 'loadbalance', 'fallback', 'conditional'],
  			},
  			conditions: {
  				type: 'array',
  				items: {
  					type: 'object',
  					properties: {
  						query: {
  							type: 'object',
  						},
  						then: {
  							type: 'string',
  						},
  					},
  					required: ['query', 'then'],
  				},
  			},
  			default: {
  				type: 'string',
  			},
  			on_status_codes: {
  				type: 'array',
  				items: {
  					type: 'integer',
  				},
  				optional: true,
  			},
  		},
  		allOf: [
  			{
  				if: {
  					properties: {
  						mode: {
  							const: 'conditional',
  						},
  					},
  				},
  				then: {
  					required: ['conditions', 'default'],
  				},
  			},
  		],
  		required: ['mode'],
  	},
  	name: {
  		type: 'string',
  	},
  	strict_open_ai_compliance: {
  		type: 'boolean',
  	},
  	provider: {
  		type: 'string',
  	},
  	resource_name: {
  		type: 'string',
  		optional: true,
  	},
  	deployment_id: {
  		type: 'string',
  		optional: true,
  	},
  	api_version: {
  		type: 'string',
  		optional: true,
  	},
  	deployments: {
  		type: 'array',
  		optional: true,
  		items: {
  			type: 'object',
  			properties: {
  				deployment_id: {
  					type: 'string',
  				},
  				alias: {
  					type: 'string',
  				},
  				api_version: {
  					type: 'string',
  				},
  				is_default: {
  					type: 'boolean',
  				},
  			},
  			required: ['deployment_id', 'alias', 'api_version'],
  		},
  	},
  	override_params: {
  		type: 'object',
  	},
  	api_key: {
  		type: 'string',
  	},
  	virtual_key: {
  		type: 'string',
  	},
  	prompt_id: {
  		type: 'string',
  	},
  	request_timeout: {
  		type: 'integer',
  	},
  	cache: {
  		type: 'object',
  		properties: {
  			mode: {
  				type: 'string',
  				enum: ['simple', 'semantic'],
  			},
  			max_age: {
  				type: 'integer',
  				optional: true,
  			},
  		},
  		required: ['mode'],
  	},
  	cb_config: {
  		type: 'object',
  		properties: {
  			failure_threshold: {
  				type: 'number',
  				minimum: 1,
  			},
  			cooldown_interval: {
  				type: 'number',
  				minimum: 30000,
  			},
  			failure_status_codes: {
  				type: 'array',
  				items: {
  					type: 'integer',
  				},
  				optional: true,
  			},
  		},
  		required: ['failure_threshold', 'cooldown_interval'],
  	},
  	retry: {
  		type: 'object',
  		properties: {
  			attempts: {
  				type: 'integer',
  			},
  			use_retry_after_headers: {
  				type: 'boolean',
  			},
  			on_status_codes: {
  				type: 'array',
  				items: {
  					type: 'number',
  				},
  				optional: true,
  			},
  		},
  		required: ['attempts'],
  	},
  	weight: {
  		type: 'number',
  	},
  	on_status_codes: {
  		type: 'array',
  		items: {
  			type: 'integer',
  		},
  	},
  	custom_host: {
  		type: 'string',
  	},
  	forward_headers: {
  		type: 'array',
  		items: {
  			type: 'string',
  		},
  	},
  	targets: {
  		type: 'array',
  		items: {
  			$ref: '#',
  		},
  	},
  	aws_access_key_id: {
  		type: 'string',
  	},
  	aws_secret_access_key: {
  		type: 'string',
  	},
  	aws_region: {
  		type: 'string',
  	},
  	aws_session_token: {
  		type: 'string',
  	},
  	openai_organization: {
  		type: 'string',
  	},
  	openai_project: {
  		type: 'string',
  	},
  	vertex_project_id: {
  		type: 'string',
  	},
  	vertex_region: {
  		type: 'string',
  	},
  	vertex_service_account_json: {
  		type: 'object',
  	},
  	azure_region: {
  		type: 'string',
  	},
  	azure_deployment_name: {
  		type: 'string',
  	},
  	azure_deployment_type: {
  		type: 'string',
  		enum: ['serverless', 'managed'],
  	},
  	azure_endpoint_name: {
  		type: 'string',
  	},
  	azure_api_version: {
  		type: 'string',
  	},
  },
  anyOf: [
  	{
  		required: ['provider', 'api_key'],
  	},
  	{
  		required: ['provider', 'custom_host'],
  	},
  	{
  		required: ['virtual_key'],
  	},
  	{
  		required: ['strategy', 'targets'],
  	},
  	{
  		required: ['cache'],
  	},
  	{
  		required: ['retry'],
  	},
  	{
  		required: ['prompt_id'],
  	},
  	{
  		required: ['forward_headers'],
  	},
  	{
  		required: ['request_timeout'],
  	},
  	{
  		required: ['provider', 'aws_access_key_id', 'aws_secret_access_key'],
  	},
  	{
  		required: ['provider', 'vertex_region', 'vertex_service_account_json'],
  	},
  	{
  		required: ['provider', 'vertex_region', 'vertex_project_id'],
  	},
  	{
  		required: [
  			'provider',
  			'azure_deployment_name',
  			'azure_deployment_type',
  			'azure_region',
  			'azure_api_version',
  		],
  	},
  	{
  		required: ['provider', 'azure_endpoint_name', 'azure_deployment_type'],
  	},
  	{
  		required: ['after_request_hooks'],
  	},
  	{
  		required: ['before_request_hooks'],
  	},
  	{
  		required: ['input_guardrails'],
  	},
  	{
  		required: ['output_guardrails'],
  	},
  ],
  additionalProperties: false,
}

Example Configs

// Simple config with cache and retry
{
  "virtual_key": "***", // Your Virtual Key
  "cache": { // Optional
    "mode": "semantic",
    "max_age": 10000
  },
  "retry": { // Optional
    "attempts": 5,
    "on_status_codes": []
  }
}

// Load balancing with 2 OpenAI keys
{
  "strategy": {
      "mode": "loadbalance"
    },
  "targets": [
    {
      "provider": "openai",
      "api_key": "sk-***"
    },
    {
      "provider": "openai",
      "api_key": "sk-***"
    }
  ]
}

You can find more examples of schemas below.

Schema Details

Key Name	Description	Type	Required	Enum Values	Additional Info
`strategy`	Operational strategy for the config or any individual target	object	Yes (if no `provider` or `virtual_key`)	-	See Strategy Object Details
`provider`	Name of the service provider	string	Yes (if no `mode` or `virtual_key`)	“openai”, “anthropic”, “azure-openai”, “anyscale”, “cohere”	-
`api_key`	API key for the service provider	string	Yes (if `provider` is specified)	-	-
`virtual_key`	Virtual key identifier	string	Yes (if no `mode` or `provider`)	-	-
`cache`	Caching configuration	object	No	-	See Cache Object Details
`retry`	Retry configuration	object	No	-	See Retry Object Details
`weight`	Weight for load balancing	number	No	-	Used in `loadbalance` mode
`on_status_codes`	Status codes triggering fallback	array of strings	No	-	Used in `fallback` mode
`targets`	List of target configurations	array	Yes (if `mode` is specified)	-	Each item follows the config schema
`request_timeout`	Request timeout configuration	number	No	-	-
`custom_host`	Route to privately hosted model	string	No	-	Used in combination with `provider` + `api_key`
`forward_headers`	Forward sensitive headers directly	array of strings	No	-	-
`override_params`	Pass model name and other hyper parameters	object	No	”model”, “temperature”, “frequency_penalty”, “logit_bias”, “logprobs”, “top_logprobs”, “max_tokens”, “n”, “presence_penalty”, “response_format”, “seed”, “stop”, “top_p”, etc.	Pass everything that’s typically part of the payload

Strategy Object Details

Key Name	Description	Type	Required	Enum Values	Additional Info
`mode`	strategy mode for the config	string	Yes	”loadbalance”, “fallback”
`on_status_codes`	status codes to apply the strategy. This field is only used when strategy mode is “fallback”	array of numbers	No		Optional

Cache Object Details

Key Name	Description	Type	Required	Enum Values	Additional Info
`mode`	Cache mode	string	Yes	”simple”, “semantic”	-
`max_age`	Maximum age for cache entries	integer	No	-	Optional

Retry Object Details

Key Name	Description	Type	Required	Enum Values	Additional Info
`attempts`	Number of retry attempts	integer	Yes	-	-
`on_status_codes`	Status codes to trigger retries	array of strings	No	-	Optional
`use_retry_after_headers`	Whether to respect provider’s Retry-After and Retry-After-ms headers	boolean	Default: false

Circuit Breaker Object Details

Key Name	Description	Type	Required	Enum Values	Additional Info
`failure_threshold`	Number of failures after which the circuit opens	number	Yes	-	Minimum value: 1
`cooldown_interval`	Time (in milliseconds) to wait before allowing retries	number	Yes	-	Minimum value: 30000 (30 seconds)
`failure_status_codes`	Specific HTTP status codes considered as failures	array of integers	No	-	Optional, defaults to status codes >500

Cloud Provider Params (Azure OpenAI, Google Vertex, AWS Bedrock)

Azure OpenAI

Key Name	Type	Required
`azure_resource_name`	string	No
`azure_deployment_id`	string	No
`azure_api_version`	string	No
`azure_model_name`	string	No
`Authorization`	string (“Bearer $API_KEY”)	No

Google Vertex AI

Key Name	Type	Required
`vertex_project_id`	string	No
`vertex_region`	string	No

AWS Bedrock

Key Name	Type	Required
`aws_access_key_id`	string	No
`aws_secret_access_key`	string	No
`aws_region`	string	No
`aws_session_token`	string	No

Notes

The strategy mode key determines the operational mode of the config. If strategy mode is not specified, a single provider mode is assumed, requiring either provider and api_key or virtual_key.
In loadbalance and fallback modes, the targets array specifies the configurations for each target.
The cache and retry objects provide additional configurations for caching and retry policies, respectively.

Examples

Single Provider with API Key

{
  "provider": "openai",
  "api_key": "sk-***"
}

Passing Model & Hyperparameters with Override Option

{
  "provider": "anthropic",
  "api_key": "xxx",
  "override_params": {
    "model": "claude-3-sonnet-20240229",
    "max_tokens": 512,
    "temperature": 0
  }
}

Single Provider with Virtual Key

{
  "virtual_key": "***"
}

Single Provider with Virtual Key, Cache and Retry

{
  "virtual_key": "***",
  "cache": {
    "mode": "semantic",
    "max_age": 10000
  },
  "retry": {
    "attempts": 5,
    "on_status_codes": [429]
  }
}

Load Balancing with Two OpenAI API Keys

{
  "strategy": {
    "mode": "loadbalance"
  },
  "targets": [
    {
      "provider": "openai",
      "api_key": "sk-***"
    },
    {
      "provider": "openai",
      "api_key": "sk-***"
    }
  ]
}

Load Balancing and Fallback Combination

  {
    "strategy": {
      "mode": "loadbalance"
    },
    "targets": [
      {
        "provider": "openai",
        "api_key": "sk-***"
      },
      {
        "strategy": {
          "mode": "fallback",
          "on_status_codes": [429, 241]
        },
        "targets": [
          {
            "virtual_key": "***"
          },
          {
            "virtual_key": "***"
          }
        ]
      }
    ]
  }

API Reference

SDK

Chat Completions

Responses

Portkey Endpoints

Embeddings

Images

Models

Audio

Other APIs

Completions

Moderations

Fine-tuning

Batch

Files

Assistants

Example Configs

Schema Details

Strategy Object Details

Cache Object Details

Retry Object Details

Circuit Breaker Object Details

Cloud Provider Params (Azure OpenAI, Google Vertex, AWS Bedrock)

Azure OpenAI

Google Vertex AI

AWS Bedrock

Notes

Examples

​Example Configs

​Schema Details

​Strategy Object Details

​Cache Object Details

​Retry Object Details

​Circuit Breaker Object Details

​Cloud Provider Params (Azure OpenAI, Google Vertex, AWS Bedrock)

​Azure OpenAI

​Google Vertex AI

​AWS Bedrock

​Notes

​Examples

Example Configs

Schema Details

Strategy Object Details

Cache Object Details

Retry Object Details

Circuit Breaker Object Details

Cloud Provider Params (Azure OpenAI, Google Vertex, AWS Bedrock)

Azure OpenAI

Google Vertex AI

AWS Bedrock

Notes

Examples