Building High Availability with Gurobi Instant Cloud Pools

Create a Pool in Gurobi Instant Cloud

1. Sign in to Gurobi Cloud Manager.
2. In the Pools section, click “Create Pool”.
3. Choose:
   • Region (e.g., us-east-1, us-west-2, or eu-west-1)
   • Machine type (e.g., m6i.large)
   • Idle shutdown and Job timeout parameters
4. Click Save to create the pool.
5. It will appear in your list with a unique Pool Name (e.g., prod-us-east-1).
6. (Optional) Repeat the process to create one or more secondary pools in other regions.

Automate Pool Creation via the REST API

You can also automate pool creation using the Instant Cloud REST API.

The endpoint to create a pool is:

POST https://cloud.gurobi.com/api/v2/pools

Example JSON body:

{
  "name": "string",
  "licenseId": "string",
  "licenseType": "light compute server",
  "description": "string",
  "provider": "aws",
  "region": "us-east-1",
  "machineType": "c5.4xlarge",
  "GRBVersion": "latest",
  "idleShutdown": 0,
  "idleJobTimeout": 0,
  "jobLimit": 0,
  "nbComputeServers": 0,
  "nbDistributedWorkers": 0,
  "jobHistory": true
}

Reference: Interactive Swagger Console

# Gurobi Instant Cloud license (US East)
# Save this file as gurobi.lic in a default location like your home directory
# or set the GRB_LICENSE_FILE environment variable to the file's full path
CLOUDACCESSID=XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
CLOUDSECRETKEY=YYYYYYYY-YYYY-YYYY-YYYY-YYYYYYYYYYYY
CLOUDPOOL=prod-us-east-1

# Gurobi Instant Cloud license (US West)

# Save this file as gurobi.lic in a default location like your home directory
# or set the GRB_LICENSE_FILE environment variable to the file's full path
CLOUDACCESSID=XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
CLOUDSECRETKEY=YYYYYYYY-YYYY-YYYY-YYYY-YYYYYYYYYYYY
CLOUDPOOL=prod-us-west-2

import threading
import queue
import gurobipy as gp
from gurobipy import GurobiError

# --- configure ---
ACCESS_ID = "<your-access-id>"
SECRET_KEY = "<your-secret-key>"
POOLS = ["prod-us-east-1", "prod-us-west-2", "prod-eu-west-1"]  # priority order
ENV_START_TIMEOUT_S = 60  # how long to wait for env.start() per pool
# -----------------


def _start_env_worker(pool_name: str, out: queue.Queue):
    """Run env.start() off-thread so we can enforce a timeout."""
    try:
        env = gp.Env(empty=True)
        env.setParam("CloudAccessID", ACCESS_ID)
        env.setParam("CloudSecretKey", SECRET_KEY)
        env.setParam("CloudPool", pool_name)
        # Optional: reduce client verbosity while failing over
        # env.setParam("OutputFlag", 0)
        env.start()
        out.put(("ok", env))
    except Exception as e:
        out.put(("err", e))


def start_env_with_timeout(pool_name: str, timeout_s: int):
    q = queue.Queue(maxsize=1)
    t = threading.Thread(target=_start_env_worker, args=(pool_name, q), daemon=True)
    t.start()
    try:
        status, payload = q.get(timeout=timeout_s)
    except queue.Empty:
        raise TimeoutError(
            f"env.start() timed out for pool '{pool_name}' after {timeout_s}s"
        )
    if status == "ok":
        return payload
    raise payload if isinstance(payload, Exception) else RuntimeError(payload)


def solve_with_failover(build_model_fn):
    """
    Try each pool:
      1) start environment with timeout
      2) build model bound to that env
      3) optimize
    Fall back on any error or startup timeout.
    """
    last_err = None
    for pool in POOLS:
        print(f"Trying pool: {pool}")
        try:
            env = start_env_with_timeout(pool, ENV_START_TIMEOUT_S)
        except Exception as e:
            print(f"  env.start() failed/timed out on {pool}: {e}")
            last_err = e
            continue

        try:
            model = build_model_fn(
                env
            )  # e.g., lambda env: gp.read("model.mps", env=env)
            model.optimize()
            print(f"Success on pool: {pool}")
            return model, pool
        except GurobiError as e:
            print(f"  Optimize failed on {pool}: {e}")
            last_err = e
            # try next pool
        except Exception as e:
            print(f"  Unexpected error on {pool}: {e}")
            last_err = e
            # try next pool

    raise RuntimeError(f"All pools failed. Last error: {last_err}")


# Example usage:
# model, used_pool = solve_with_failover(lambda env: gp.read("model.mps", env=env))
# print("Solved using pool:", used_pool)

import time
import requests
from requests.auth import AuthBase


# ---- Custom auth (adds v2 headers while keeping auth=auth call sites) ----
class GurobiCloudAuth(AuthBase):
    def __init__(self, access_id: str, secret_key: str):
        self.access_id = access_id.strip()
        self.secret_key = secret_key.strip()

    def __call__(self, r: requests.PreparedRequest) - requests.PreparedRequest:
        r.headers["X-GUROBI-ACCESS-ID"] = self.access_id
        r.headers["X-GUROBI-SECRET-KEY"] = self.secret_key
        r.headers.setdefault("Accept", "application/json")
        r.headers.setdefault("Content-Type", "application/json")
        return r


# --- configure ---
ACCESS_ID = ""
SECRET_KEY = ""
BASE_URL = "https://cloud.gurobi.com/api/v2"
POOL_NAME = "prod-us-west-2"  # backup pool
DESIRED_COUNT = 1  # how many machines to warm-start
READY_TIMEOUT_S = 180
POLL_EVERY_S = 5
# -----------------

auth = GurobiCloudAuth(ACCESS_ID, SECRET_KEY)
READY_STATES = {"idle", "running", "busy", "ready"}  # "ready enough" states


def machine_state(m: dict) - str:
    # Prefer "state", fallback to "status"; lowercased
    return (m.get("state") or m.get("status") or "").lower()


def get_machines(pool_id: str) - list[dict]:
    r = requests.get(f"{BASE_URL}/pools/{pool_id}/machines", auth=auth, timeout=30)
    r.raise_for_status()
    raw = r.json()
    machines = raw["computeServers"]
    return machines


def any_ready(machines: list[dict]) - bool:
    return any(machine_state(m) in READY_STATES for m in machines)


def count_ready(machines: list[dict]) - int:
    return sum(1 for m in machines if machine_state(m) in READY_STATES)


# 1) Locate the pool
resp = requests.get(f"{BASE_URL}/pools", auth=auth, timeout=30)
resp.raise_for_status()
pools = resp.json()
pool = next((p for p in pools if p.get("name") == POOL_NAME), None)
if not pool:
    raise RuntimeError(
        f"Pool {POOL_NAME} not found. Available: {[p['name'] for p in pools]}"
    )
pool_id = pool["id"]

# 2) Check existing machines and readiness
machines = get_machines(pool_id)
if any_ready(machines):
    print(
        f"Pool {POOL_NAME} already ready "
        f"({count_ready(machines)}/{len(machines)} machine(s) in ready state)"
    )
else:
    print(f"No ready machines in {POOL_NAME}. Launching {DESIRED_COUNT}...")
    launch = requests.post(
        f"{BASE_URL}/pools/{pool_id}/machines",
        auth=auth,
        json={"machineCount": DESIRED_COUNT},
        timeout=30,
    )
    launch.raise_for_status()

    # 3) Poll until at least one machine is ready
    deadline = time.time() + READY_TIMEOUT_S
    while time.time() = 1:
            print(f"Pool {POOL_NAME} is ready ({ready}/{total} ready).")
            break
        time.sleep(POLL_EVERY_S)
    else:
        raise TimeoutError(f"Pool {POOL_NAME} did not become ready in time.")

# Continue with model building and execution with launched pool...