Microsoft Foundry Series (Part 10) — CI/CD with GitLab: Evaluation Gates, Cost Optimization and Monitoring

 1 Merge request pipeline                     Default branch (main) pipeline
 2 ──────────────────────                     ──────────────────────────────
 3 ┌──────────┐  bicep-lint, bicep-build,     ┌──────────┐
 4 │ validate │  ruff                         │ validate │
 5 └────┬─────┘                               └────┬─────┘
 6      ▼                                          ▼
 7 ┌──────────┐  pytest (JUnit),              ┌──────────┐
 8 │   test   │  what-if:dev (artifact)       │   test   │  pytest
 9 └────┬─────┘                               └────┬─────┘
10      ▼                                          ▼
11 ┌──────────┐  eval:mr — golden dataset     ┌──────────┐
12 │ evaluate │  against the MR's prompts     │  build   │  az acr build → image:sha
13 └──────────┘  (in-process, dev model)      └────┬─────┘
14                                                 ▼
15                                            ┌────────────┐
16                                            │ deploy-dev │  Bicep + containerapp update
17                                            └────┬───────┘
18                                                 ▼
19                                            ┌──────────┐  eval:dev — golden dataset
20                                            │ evaluate │  against foundry-demo-api
21                                            └────┬─────┘  what-if:prod (artifact)
22                                                 ▼
23                                            ┌─────────────┐  manual + protected env
24                                            │ deploy-prod │  same image, approval
25                                            └─────────────┘
26
27 Scheduled pipeline (weekly): redteam — AI Red Teaming Agent scan against dev

1curl --request PUT \
2  --header "PRIVATE-TOKEN: $GITLAB_TOKEN" \
3  --header "Content-Type: application/json" \
4  --data '{"ci_id_token_sub_claim_components": ["project_path", "ref_type", "ref", "deployment_tier"]}' \
5  "https://gitlab.com/api/v4/projects/<your-project-id>"

1project_path:<group>/<project>:ref_type:branch:ref:main:deployment_tier:production

 1PROD_APP_ID=$(az ad app create --display-name gitlab-foundry-prod --query appId -o tsv)
 2az ad sp create --id $PROD_APP_ID
 3PROD_OBJECT_ID=$(az ad app show --id $PROD_APP_ID --query id -o tsv)
 4
 5cat <<EOF > fic-prod.json
 6{
 7  "name": "gitlab-main-production",
 8  "issuer": "https://gitlab.com",
 9  "subject": "project_path:<group>/<project>:ref_type:branch:ref:main:deployment_tier:production",
10  "description": "GitLab main branch, production tier only",
11  "audiences": ["https://gitlab.com"]
12}
13EOF
14
15az rest --method POST \
16  --uri "https://graph.microsoft.com/beta/applications/$PROD_OBJECT_ID/federatedIdentityCredentials" \
17  --body @fic-prod.json

 1{
 2  "name": "gitlab-any-branch-dev",
 3  "issuer": "https://gitlab.com",
 4  "subject": null,
 5  "claimsMatchingExpression": {
 6    "value": "claims['sub'] matches 'project_path:<group>/<project>:ref_type:branch:ref:*' and claims['project_id'] eq '<your-project-id>'",
 7    "languageVersion": 1
 8  },
 9  "description": "GitLab, any branch of this project, dev only",
10  "audiences": ["https://gitlab.com"]
11}

  1include:
  2  - component: $CI_SERVER_FQDN/components/code-quality-oss/codequality-os-scanners-integration/bicep-lint@1.3.2
  3    inputs:
  4      stage: validate
  5
  6stages: [validate, test, build, deploy-dev, evaluate, deploy-prod, scheduled]
  7
  8variables:
  9  AZURE_TENANT_ID: "<your-tenant-id>"
 10  AZURE_SUBSCRIPTION_ID: "<your-subscription-id>"
 11  AZ_IMAGE: mcr.microsoft.com/azure-cli:2.85.0
 12  PYTHON_IMAGE: python:3.12-slim
 13  ACR_NAME: "<your-acr-name>"          # the registry from Part 6, shared by dev and prod
 14  IMAGE_NAME: product-api
 15  IMAGE: $ACR_NAME.azurecr.io/$IMAGE_NAME:$CI_COMMIT_SHORT_SHA
 16  DEV_RG: rg-foundry-demo
 17  DEV_APP: foundry-demo-api
 18  PROD_RG: rg-foundry-demo-prod
 19  PROD_APP: foundry-demo-prod-api
 20  DEV_APP_URL: https://foundry-demo-api.<your-env-domain>.swedencentral.azurecontainerapps.io
 21  PROD_APP_URL: https://foundry-demo-prod-api.<your-env-domain>.swedencentral.azurecontainerapps.io
 22  PROJECT_ENDPOINT: https://foundry-demo-ai.services.ai.azure.com/api/projects/foundry-demo-project
 23  AZURE_OPENAI_ENDPOINT: https://foundry-demo-ai.services.ai.azure.com
 24  JUDGE_DEPLOYMENT: gpt-4.1
 25  AZURE_CORE_NO_COLOR: "true"
 26  AZURE_EXTENSION_USE_DYNAMIC_INSTALL: yes_without_prompt
 27
 28workflow:
 29  rules:
 30    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
 31    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
 32      when: never
 33    - if: $CI_PIPELINE_SOURCE == "schedule"
 34    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
 35
 36.not-scheduled:
 37  rules:
 38    - if: $CI_PIPELINE_SOURCE == "schedule"
 39      when: never
 40    - when: on_success
 41
 42.main-only:
 43  rules:
 44    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push"
 45
 46# .az-login-prod is identical, with $AZURE_CLIENT_ID_PROD
 47.az-login-dev:
 48  image: $AZ_IMAGE
 49  id_tokens:
 50    GITLAB_OIDC_TOKEN:
 51      aud: https://gitlab.com
 52  before_script:
 53    - az login --service-principal -u "$AZURE_CLIENT_ID_DEV" -t "$AZURE_TENANT_ID" --federated-token "$GITLAB_OIDC_TOKEN" --output none
 54    - az account set -s "$AZURE_SUBSCRIPTION_ID"
 55
 56# Python jobs: DefaultAzureCredential picks up workload identity from these env vars
 57.python-wif-dev:
 58  image: $PYTHON_IMAGE
 59  id_tokens:
 60    GITLAB_OIDC_TOKEN:
 61      aud: https://gitlab.com
 62  variables:
 63    AZURE_CLIENT_ID: $AZURE_CLIENT_ID_DEV
 64    AZURE_FEDERATED_TOKEN_FILE: /tmp/gitlab-oidc-token
 65  before_script:
 66    - echo "$GITLAB_OIDC_TOKEN" > "$AZURE_FEDERATED_TOKEN_FILE"
 67    - pip install -q -r requirements-dev.txt
 68
 69build-image:
 70  stage: build
 71  extends: [.az-login-dev, .main-only]
 72  needs: [pytest, ruff]
 73  script:
 74    - az acr build --registry "$ACR_NAME" --image "$IMAGE_NAME:$CI_COMMIT_SHORT_SHA" --file Dockerfile .
 75
 76.eval:
 77  stage: evaluate
 78  extends: .python-wif-dev
 79  artifacts:
 80    when: always
 81    expose_as: "Evaluation results"
 82    paths: [reports/]
 83    reports:
 84      junit: reports/eval-junit.xml
 85
 86eval:mr:
 87  extends: .eval
 88  needs: [pytest]
 89  rules:
 90    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
 91      changes: [app/**/*, eval/**/*]   # includes app/prompts/
 92  variables:
 93    MODEL_NAME: gpt-4.1-mini
 94  script:
 95    - python -m eval.gate --target local --out reports/
 96
 97eval:dev:
 98  extends: [.eval, .main-only]
 99  needs: [deploy-dev]
100  script:
101    - python -m eval.gate --target http --endpoint "$DEV_APP_URL" --out reports/
102
103what-if:prod:
104  stage: evaluate
105  extends: [.az-login-prod, .main-only]
106  needs: [bicep-build]
107  environment:
108    name: production
109    deployment_tier: production
110    action: prepare
111  script:
112    - export CONTAINER_IMAGE=$(az containerapp show -n "$PROD_APP" -g "$PROD_RG" --query "properties.template.containers[0].image" -o tsv)
113    - az deployment group what-if
114        --resource-group "$PROD_RG"
115        --template-file infra/main.bicep
116        --parameters infra/env/prod.bicepparam
117        --exclude-change-types NoChange Ignore | tee whatif-prod.txt
118  artifacts:
119    expose_as: "Bicep what-if (prod)"
120    paths: [whatif-prod.txt]
121
122# deploy-dev is the same job against $DEV_RG / $DEV_APP, with .az-login-dev and no manual gate
123deploy-prod:
124  stage: deploy-prod
125  extends: .az-login-prod
126  needs: [build-image, eval:dev, what-if:prod]
127  rules:
128    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push"
129      when: manual
130  allow_failure: false
131  environment:
132    name: production
133    deployment_tier: production
134    url: $PROD_APP_URL
135  resource_group: production
136  script:
137    # Infra first, keeping whatever image is currently running (or the new one on first deploy)
138    - export CONTAINER_IMAGE=$(az containerapp show -n "$PROD_APP" -g "$PROD_RG" --query "properties.template.containers[0].image" -o tsv 2>/dev/null || echo "$IMAGE")
139    - az deployment group create
140        --name "foundry-$CI_PIPELINE_ID"
141        --resource-group "$PROD_RG"
142        --template-file infra/main.bicep
143        --parameters infra/env/prod.bicepparam
144        --output none
145    # Then roll the app to the new image as a new revision
146    - az containerapp update
147        --name "$PROD_APP" --resource-group "$PROD_RG"
148        --image "$IMAGE" --revision-suffix "sha-$CI_COMMIT_SHORT_SHA"
149        --output none
150    - curl --fail --silent --show-error "$PROD_APP_URL/healthz"
151
152redteam:
153  stage: scheduled
154  extends: .python-wif-dev
155  rules:
156    - if: $CI_PIPELINE_SOURCE == "schedule" && $SCHEDULED_TASK == "redteam"
157  needs: []
158  timeout: 2h
159  script:
160    - pip install -q "azure-ai-evaluation[redteam]"
161    - python -m redteam.scan   # the Part 9 scan script, saved as redteam/scan.py; writes redteam-scorecard.json
162    - python -c "import json, sys;
163        asr = json.load(open('redteam-scorecard.json'))['redteaming_scorecard']['risk_category_summary'][0]['overall_asr'];
164        print(f'Overall ASR {asr:.1%}'); sys.exit(asr > 0)"
165  artifacts:
166    when: always
167    paths: [redteam-scorecard.json]
168    expire_in: 90 days

1using '../main.bicep'
2
3param baseName = 'foundry-demo'          // prod.bicepparam: 'foundry-demo-prod'
4param location = 'swedencentral'
5param deploymentSku = 'GlobalStandard'
6param enablePrivateEndpoint = false
7param containerImage = readEnvironmentVariable('CONTAINER_IMAGE')

 1{
 2  "groundedness.binary_aggregate": { "min": 0.95 },
 3  "relevance.binary_aggregate":    { "min": 0.90 },
 4  "coherence.binary_aggregate":    { "min": 0.95 },
 5  "schema.schema_valid":           { "min": 1.0 },
 6  "rules.rules_ok":                { "min": 0.95 },
 7  "mentions.coverage":             { "min": 0.95 },
 8  "claims.no_forbidden_claims":    { "min": 1.0 },
 9  "violence.violence_defect_rate": { "max": 0.0 },
10  "budget.within_budget":          { "min": 0.95 }
11}

  1"""Evaluation gate: run the golden dataset, compare with thresholds, exit 1 on regression."""
  2import argparse
  3import asyncio
  4import json
  5import os
  6import sys
  7from pathlib import Path
  8from xml.etree.ElementTree import Element, ElementTree, SubElement
  9
 10import httpx
 11from azure.ai.evaluation import (AzureOpenAIModelConfiguration, CoherenceEvaluator,
 12                                 GroundednessEvaluator, RelevanceEvaluator, ViolenceEvaluator, evaluate)
 13from azure.ai.projects.aio import AIProjectClient
 14from azure.identity import DefaultAzureCredential
 15from azure.identity.aio import DefaultAzureCredential as AsyncDefaultAzureCredential
 16
 17from app.generator import generate_description
 18from app.models import ProductRequest
 19from app.prompts import load_prompt
 20from eval.evaluators import (BusinessRulesEvaluator, ForbiddenClaimsEvaluator,  # from Part 8
 21                             RequiredAttributesEvaluator, SchemaValidEvaluator)
 22
 23PROJECT_ENDPOINT = os.environ["PROJECT_ENDPOINT"]
 24GOLDEN = Path("eval/golden.jsonl")
 25MIN_ROWS = 30             # refuse to "pass" on a truncated dataset
 26MAX_OUTPUT_TOKENS = 220   # per description, used by the cost gate
 27
 28
 29class TokenBudgetEvaluator:
 30    def __call__(self, *, output_tokens: int) -> dict:
 31        return {"output_tokens": output_tokens, "within_budget": float(output_tokens <= MAX_OUTPUT_TOKENS)}
 32
 33
 34def as_text(d: dict) -> str:
 35    return "\n".join([d["title"], d["tagline"], d["description"], *d["bullet_points"]])
 36
 37
 38async def generate_local(rows: list[dict], model: str) -> None:
 39    """The MR's own prompt and generator, in-process, against the dev model."""
 40    prompt = load_prompt("product-description")
 41    print(f"prompt product-description@{prompt.version}, model {model}")
 42    async with AsyncDefaultAzureCredential() as credential, AIProjectClient(
 43        endpoint=PROJECT_ENDPOINT, credential=credential
 44    ) as project:
 45        openai = project.get_openai_client()
 46        for row in rows:
 47            req = ProductRequest(sku=row["sku"], locale=row["locale"])
 48            desc, usage = await generate_description(openai, req, prompt, model)
 49            row.update(raw_json=desc.model_dump_json(), response=as_text(desc.model_dump()),
 50                       output_tokens=usage.output_tokens)
 51
 52
 53async def generate_http(rows: list[dict], endpoint: str) -> None:
 54    """The deployed dev API, exactly as clients call it."""
 55    async with httpx.AsyncClient(base_url=endpoint, timeout=60) as client:
 56        for row in rows:
 57            r = await client.post("/descriptions", json={"sku": row["sku"], "locale": row["locale"]})
 58            r.raise_for_status()
 59            row.update(raw_json=r.text, response=as_text(r.json()))
 60
 61
 62def write_junit(checks: list[dict], path: Path) -> None:
 63    suite = Element("testsuite", name="evaluation-gate", tests=str(len(checks)),
 64                    failures=str(sum(not c["passed"] for c in checks)))
 65    for c in checks:
 66        case = SubElement(suite, "testcase", classname="evaluation", name=c["metric"])
 67        if not c["passed"]:
 68            SubElement(case, "failure", message=f"{c['metric']} = {c['value']} ({c['rule']})")
 69    ElementTree(suite).write(path, encoding="utf-8", xml_declaration=True)
 70
 71
 72def main() -> int:
 73    parser = argparse.ArgumentParser()
 74    parser.add_argument("--target", choices=["local", "http"], required=True)
 75    parser.add_argument("--endpoint")
 76    parser.add_argument("--out", default="reports/")
 77    args = parser.parse_args()
 78    out = Path(args.out)
 79    out.mkdir(parents=True, exist_ok=True)
 80
 81    rows = [json.loads(line) for line in GOLDEN.read_text().splitlines() if line.strip()]
 82    if args.target == "local":
 83        asyncio.run(generate_local(rows, os.environ["MODEL_NAME"]))
 84    else:
 85        asyncio.run(generate_http(rows, args.endpoint))
 86    outputs = out / "outputs.jsonl"
 87    outputs.write_text("\n".join(json.dumps(r, ensure_ascii=False) for r in rows), encoding="utf-8")
 88
 89    judge = AzureOpenAIModelConfiguration(azure_endpoint=os.environ["AZURE_OPENAI_ENDPOINT"],
 90                                          azure_deployment=os.environ["JUDGE_DEPLOYMENT"],
 91                                          api_version="2024-10-21")
 92    evaluators = {
 93        "groundedness": GroundednessEvaluator(judge, threshold=4),
 94        "relevance": RelevanceEvaluator(judge, threshold=4),
 95        "coherence": CoherenceEvaluator(judge),
 96        "violence": ViolenceEvaluator(credential=DefaultAzureCredential(), azure_ai_project=PROJECT_ENDPOINT),
 97        "schema": SchemaValidEvaluator(),
 98        "rules": BusinessRulesEvaluator(),
 99        "mentions": RequiredAttributesEvaluator(),
100        "claims": ForbiddenClaimsEvaluator(),
101    }
102    mapping = {"query": "${data.query}", "context": "${data.context}", "sku": "${data.sku}",
103               "required_attributes": "${data.required_attributes}",
104               "response": "${data.response}", "raw_json": "${data.raw_json}"}
105    if args.target == "local":  # the API doesn't return token usage, so the cost gate runs in-process only
106        evaluators["budget"] = TokenBudgetEvaluator()
107        mapping["output_tokens"] = "${data.output_tokens}"
108
109    result = evaluate(
110        data=str(outputs),
111        evaluators=evaluators,
112        evaluator_config={"default": {"column_mapping": mapping}},
113        evaluation_name=f"gate-{args.target}-{os.getenv('CI_PIPELINE_ID', 'local')}",
114        azure_ai_project=PROJECT_ENDPOINT,  # every gate run shows up in the Foundry project
115        output_path=str(out / "eval-results.json"),
116    )
117
118    metrics = result["metrics"]
119    checks = [{"metric": "rows", "value": len(rows), "rule": f">= {MIN_ROWS}", "passed": len(rows) >= MIN_ROWS}]
120    for metric, rule in json.loads(Path("eval/thresholds.json").read_text()).items():
121        if metric.split(".")[0] not in evaluators:
122            continue
123        value = metrics.get(metric)
124        if "min" in rule:
125            passed, text = value is not None and value >= rule["min"], f">= {rule['min']}"
126        else:
127            passed, text = value is not None and value <= rule["max"], f"<= {rule['max']}"
128        checks.append({"metric": metric, "value": None if value is None else round(value, 3),
129                       "rule": text, "passed": passed})
130
131    write_junit(checks, out / "eval-junit.xml")
132    (out / "eval-summary.json").write_text(json.dumps(checks, indent=2))
133    for c in checks:
134        print(f"{'PASS' if c['passed'] else 'FAIL'}  {c['metric']:<32} {c['value']}  ({c['rule']})")
135    return 0 if all(c["passed"] for c in checks) else 1
136
137
138if __name__ == "__main__":
139    sys.exit(main())

 1prompt product-description@9c41e0d2, model gpt-4.1-mini
 2PASS  rows                             30  (>= 30)
 3PASS  groundedness.binary_aggregate    0.967  (>= 0.95)
 4PASS  relevance.binary_aggregate       0.933  (>= 0.9)
 5FAIL  coherence.binary_aggregate       0.867  (>= 0.95)
 6PASS  schema.schema_valid              1.0  (>= 1.0)
 7FAIL  rules.rules_ok                   0.733  (>= 0.95)
 8PASS  mentions.coverage                0.967  (>= 0.95)
 9PASS  claims.no_forbidden_claims       1.0  (>= 1.0)
10PASS  violence.violence_defect_rate    0.0  (<= 0.0)
11FAIL  budget.within_budget             0.633  (>= 0.95)

1  properties: {
2    model: { name: modelName, format: 'OpenAI', version: modelVersion }
3    versionUpgradeOption: 'NoAutoUpgrade'
4  }

1Pay-as-you-go / month = (input_tokens × input_price + output_tokens × output_price) / 1,000,000
2PTU / month           = PTUs × hourly_rate × 730   (less with a monthly/yearly reservation)
3
4PTU pays off when:  PTU / month  <  Pay-as-you-go / month  at your *sustained* volume
5
6Example: 1M descriptions/month, ~1,200 input + ~180 output tokens each, gpt-4.1-mini (Part 3 list prices)
7Input:  1,200,000,000 tokens × $0.40 / 1M = $480
8Output:   180,000,000 tokens × $1.60 / 1M = $288
9Total:                                       $768 / month

 1resource ptuDeployment 'Microsoft.CognitiveServices/accounts/deployments@2025-04-01-preview' = {
 2  parent: foundry
 3  name: 'gpt-4.1-mini-ptu'
 4  sku: {
 5    name: 'GlobalProvisionedManaged'
 6    capacity: 15 // PTUs; check the minimum and increments for your model
 7  }
 8  properties: {
 9    model: { format: 'OpenAI', name: 'gpt-4.1-mini', version: '2025-04-14' }
10    spilloverDeploymentName: 'gpt-4.1-mini' // an existing Standard deployment of the same model/version
11  }
12}

 1import json
 2
 3from azure.ai.projects import AIProjectClient
 4from azure.identity import DefaultAzureCredential
 5
 6from app.catalog import get_product_details  # Part 7
 7from app.prompts import load_prompt          # Part 5
 8
 9project = AIProjectClient(
10    endpoint="https://foundry-demo-prod-ai.services.ai.azure.com/api/projects/foundry-demo-prod-project",
11    credential=DefaultAzureCredential(),
12)
13client = project.get_openai_client()
14prompt = load_prompt("product-description")
15
16with open("backfill.jsonl", "w") as f:
17    for sku in skus_without_description():  # from PostgreSQL (Part 7)
18        f.write(json.dumps({
19            "custom_id": f"{sku}:{prompt.version}",
20            "method": "POST",
21            "url": "/v1/responses",
22            "body": {
23                "model": "gpt-4.1-mini-batch",
24                "instructions": prompt.instructions,
25                "input": f"SKU: {sku}\nLocale: en-GB\nProduct details:\n{get_product_details(sku)}",
26                "max_output_tokens": 300,
27            },
28        }) + "\n")
29
30batch_file = client.files.create(file=open("backfill.jsonl", "rb"), purpose="batch")
31batch = client.batches.create(input_file_id=batch_file.id, endpoint="/v1/responses", completion_window="24h")
32print(batch.id)  # poll later, then write results back by custom_id

1┌────────────────────────────────────────────┐  ← identical for every request
2│ system.md (copywriting rules, brand voice) │
3│ examples.json (few-shot descriptions)      │
4│ Tool definitions / output schema           │
5├────────────────────────────────────────────┤  ← changes per request
6│ Retrieved sources [S1]..[Sn] (Part 7)      │
7│ SKU + locale                               │
8│ get_product_details tool output            │
9└────────────────────────────────────────────┘

 1import hashlib
 2import json
 3
 4from app.generator import generate_description
 5from app.models import ProductRequest
 6
 7
 8def description_hash(facts: dict, prompt_version: str, model: str) -> str:
 9    payload = json.dumps({"f": facts, "p": prompt_version, "m": model}, sort_keys=True)
10    return hashlib.sha256(payload.encode()).hexdigest()
11
12
13async def get_or_generate(openai, product, prompt, model: str):
14    h = description_hash(product.facts, prompt.version, model)
15    if product.description_hash == h:
16        return product.description  # cache hit: zero tokens
17    description, _usage = await generate_description(
18        openai, ProductRequest(sku=product.sku, locale=product.locale), prompt, model
19    )
20    await save_description(product.sku, description, h)  # UPDATE products SET description=..., description_hash=...
21    return description

 1@description('Monthly budget in the billing currency')
 2param monthlyBudget int = 200
 3
 4@description('First day of a month, e.g. 2026-09-01. Keep it fixed after the first deployment.')
 5param budgetStartDate string
 6
 7param alertEmail string
 8
 9resource budget 'Microsoft.Consumption/budgets@2023-05-01' = {
10  name: 'budget-${baseName}'
11  properties: {
12    category: 'Cost'
13    amount: monthlyBudget
14    timeGrain: 'Monthly'
15    timePeriod: { startDate: budgetStartDate }
16    notifications: {
17      actual80: { enabled: true, operator: 'GreaterThanOrEqualTo', threshold: 80, thresholdType: 'Actual', contactEmails: [alertEmail] }
18      forecast100: { enabled: true, operator: 'GreaterThan', threshold: 100, thresholdType: 'Forecasted', contactEmails: [alertEmail] }
19    }
20  }
21}

 1param foundryName string // foundry-demo-ai / foundry-demo-prod-ai
 2param alertEmail string
 3
 4@description('Hourly token ceiling before someone gets paged')
 5param hourlyTokenThreshold int = 2000000
 6
 7resource foundry 'Microsoft.CognitiveServices/accounts@2025-04-01-preview' existing = {
 8  name: foundryName
 9}
10
11resource actionGroup 'Microsoft.Insights/actionGroups@2023-01-01' = {
12  name: 'ag-${foundryName}'
13  location: 'global'
14  properties: {
15    groupShortName: 'foundry'
16    enabled: true
17    emailReceivers: [{ name: 'platform-team', emailAddress: alertEmail, useCommonAlertSchema: true }]
18  }
19}
20
21var alerts = [
22  { name: 'throttling-429', severity: 2, metricName: 'AzureOpenAIRequests', timeAggregation: 'Total', threshold: 10, windowSize: 'PT15M', dimensions: [{ name: 'StatusCode', operator: 'Include', values: ['429'] }] }
23  { name: 'latency-time-to-response', severity: 3, metricName: 'AzureOpenAITimeToResponse', timeAggregation: 'Average', threshold: 5000, windowSize: 'PT15M', dimensions: [] }
24  { name: 'token-consumption', severity: 2, metricName: 'TokenTransaction', timeAggregation: 'Total', threshold: hourlyTokenThreshold, windowSize: 'PT1H', dimensions: [] }
25]
26
27resource metricAlerts 'Microsoft.Insights/metricAlerts@2018-03-01' = [
28  for alert in alerts: {
29    name: '${foundryName}-${alert.name}'
30    location: 'global'
31    properties: {
32      severity: alert.severity
33      enabled: true
34      scopes: [foundry.id]
35      evaluationFrequency: 'PT5M'
36      windowSize: alert.windowSize
37      criteria: {
38        'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria'
39        allOf: [{
40          name: alert.name
41          criterionType: 'StaticThresholdCriterion'
42          metricNamespace: 'Microsoft.CognitiveServices/accounts'
43          metricName: alert.metricName
44          dimensions: alert.dimensions
45          operator: 'GreaterThan'
46          threshold: alert.threshold
47          timeAggregation: alert.timeAggregation
48        }]
49      }
50      actions: [{ actionGroupId: actionGroup.id }]
51    }
52  }
53]

1az group delete --name rg-foundry-demo --yes --no-wait
2az group delete --name rg-foundry-demo-prod --yes --no-wait
3
4# Remove the CI identities
5az ad app delete --id $(az ad app list --display-name gitlab-foundry-dev --query "[0].appId" -o tsv)
6az ad app delete --id $(az ad app list --display-name gitlab-foundry-prod --query "[0].appId" -o tsv)