[{"data":1,"prerenderedAt":4459},["ShallowReactive",2],{"content-query-lCvHNvQFtG":3},[4,991,1644,2349,3052,3857],{"_path":5,"_dir":6,"_draft":7,"_partial":7,"_locale":8,"title":9,"description":10,"date":11,"aiGenerated":12,"body":13,"_type":985,"_id":986,"_source":987,"_file":988,"_stem":989,"_extension":990},"\u002Farticles\u002Fdocker-for-ml-engineers","articles",false,"","Docker for ML Engineers Who Just Want to Ship a Model","A practical Docker guide for ML engineers — writing Dockerfiles for model serving, keeping images small with multi-stage builds, handling GPUs with NVIDIA base images, and pushing to a registry.","2025-07-11",true,{"type":14,"children":15,"toc":962},"root",[16,24,30,35,42,47,57,70,81,86,97,102,113,126,133,228,234,255,283,292,297,303,308,314,319,328,349,355,373,422,477,485,513,519,524,533,538,583,595,601,606,638,647,667,672,683,688,694,699,707,712,718,723,729,738,743,749,758,764,769,778,784,797,802,871,876,882,887,896,901,910,915,924,937,941],{"type":17,"tag":18,"props":19,"children":21},"element","h1",{"id":20},"docker-for-ml-engineers-who-just-want-to-ship-a-model",[22],{"type":23,"value":9},"text",{"type":17,"tag":25,"props":26,"children":27},"p",{},[28],{"type":23,"value":29},"You know Python. You can train a model, write a serving script, and get predictions out of an API endpoint. But someone keeps saying you need to \"containerize it,\" and the Docker documentation reads like it was written for infrastructure engineers.",{"type":17,"tag":25,"props":31,"children":32},{},[33],{"type":23,"value":34},"This article skips the container philosophy and goes straight to: here's how to put your model in a Docker image so other people can run it.",{"type":17,"tag":36,"props":37,"children":39},"h2",{"id":38},"the-minimum-viable-dockerfile",[40],{"type":23,"value":41},"The minimum viable Dockerfile",{"type":17,"tag":25,"props":43,"children":44},{},[45],{"type":23,"value":46},"Say you have a directory with these files:",{"type":17,"tag":48,"props":49,"children":51},"pre",{"code":50},"my-model\u002F\n├── serve.py\n├── model.pkl\n└── requirements.txt\n",[52],{"type":17,"tag":53,"props":54,"children":55},"code",{"__ignoreMap":8},[56],{"type":23,"value":50},{"type":17,"tag":25,"props":58,"children":59},{},[60,62,68],{"type":23,"value":61},"Your ",{"type":17,"tag":53,"props":63,"children":65},{"className":64},[],[66],{"type":23,"value":67},"serve.py",{"type":23,"value":69}," is a FastAPI app:",{"type":17,"tag":48,"props":71,"children":76},{"code":72,"language":73,"meta":8,"className":74},"import pickle\nfrom fastapi import FastAPI\n\napp = FastAPI()\n\nwith open(\"model.pkl\", \"rb\") as f:\n model = pickle.load(f)\n\n@app.get(\"\u002Fhealth\")\ndef health():\n return {\"status\": \"ok\"}\n\n@app.post(\"\u002Fpredict\")\ndef predict(features: list[float]):\n result = model.predict([features])\n return {\"prediction\": int(result[0])}\n","python",[75],"language-python",[77],{"type":17,"tag":53,"props":78,"children":79},{"__ignoreMap":8},[80],{"type":23,"value":72},{"type":17,"tag":25,"props":82,"children":83},{},[84],{"type":23,"value":85},"Here's the Dockerfile:",{"type":17,"tag":48,"props":87,"children":92},{"code":88,"language":89,"meta":8,"className":90},"FROM python:3.11-slim\n\nWORKDIR \u002Fapp\n\nCOPY requirements.txt .\nRUN pip install --no-cache-dir -r requirements.txt\n\nCOPY serve.py model.pkl .\u002F\n\nCMD [\"uvicorn\", \"serve:app\", \"--host\", \"0.0.0.0\", \"--port\", \"8000\"]\n","dockerfile",[91],"language-dockerfile",[93],{"type":17,"tag":53,"props":94,"children":95},{"__ignoreMap":8},[96],{"type":23,"value":88},{"type":17,"tag":25,"props":98,"children":99},{},[100],{"type":23,"value":101},"Build and run:",{"type":17,"tag":48,"props":103,"children":108},{"code":104,"language":105,"meta":8,"className":106},"docker build -t my-model:v1 .\ndocker run -p 8000:8000 my-model:v1\n","bash",[107],"language-bash",[109],{"type":17,"tag":53,"props":110,"children":111},{"__ignoreMap":8},[112],{"type":23,"value":104},{"type":17,"tag":25,"props":114,"children":115},{},[116,118,124],{"type":23,"value":117},"That's it. Hit ",{"type":17,"tag":53,"props":119,"children":121},{"className":120},[],[122],{"type":23,"value":123},"http:\u002F\u002Flocalhost:8000\u002Fhealth",{"type":23,"value":125}," and you should get a response.",{"type":17,"tag":127,"props":128,"children":130},"h3",{"id":129},"what-each-line-does",[131],{"type":23,"value":132},"What each line does",{"type":17,"tag":134,"props":135,"children":136},"ul",{},[137,161,175,200,214],{"type":17,"tag":138,"props":139,"children":140},"li",{},[141,151,153,159],{"type":17,"tag":142,"props":143,"children":144},"strong",{},[145],{"type":17,"tag":53,"props":146,"children":148},{"className":147},[],[149],{"type":23,"value":150},"FROM python:3.11-slim",{"type":23,"value":152}," — start from a base image that already has Python installed. The ",{"type":17,"tag":53,"props":154,"children":156},{"className":155},[],[157],{"type":23,"value":158},"slim",{"type":23,"value":160}," variant is a stripped-down Debian image (~150 MB instead of ~900 MB).",{"type":17,"tag":138,"props":162,"children":163},{},[164,173],{"type":17,"tag":142,"props":165,"children":166},{},[167],{"type":17,"tag":53,"props":168,"children":170},{"className":169},[],[171],{"type":23,"value":172},"WORKDIR \u002Fapp",{"type":23,"value":174}," — set the working directory inside the container. All subsequent commands run from here.",{"type":17,"tag":138,"props":176,"children":177},{},[178,187,189,198],{"type":17,"tag":142,"props":179,"children":180},{},[181],{"type":17,"tag":53,"props":182,"children":184},{"className":183},[],[185],{"type":23,"value":186},"COPY requirements.txt .",{"type":23,"value":188}," then ",{"type":17,"tag":142,"props":190,"children":191},{},[192],{"type":17,"tag":53,"props":193,"children":195},{"className":194},[],[196],{"type":23,"value":197},"RUN pip install",{"type":23,"value":199}," — install dependencies first, before copying the rest of the code. Docker caches each layer, so if your code changes but your dependencies don't, pip install won't re-run.",{"type":17,"tag":138,"props":201,"children":202},{},[203,212],{"type":17,"tag":142,"props":204,"children":205},{},[206],{"type":17,"tag":53,"props":207,"children":209},{"className":208},[],[210],{"type":23,"value":211},"COPY serve.py model.pkl .\u002F",{"type":23,"value":213}," — copy your application code and model into the container.",{"type":17,"tag":138,"props":215,"children":216},{},[217,226],{"type":17,"tag":142,"props":218,"children":219},{},[220],{"type":17,"tag":53,"props":221,"children":223},{"className":222},[],[224],{"type":23,"value":225},"CMD [...]",{"type":23,"value":227}," — the command that runs when the container starts.",{"type":17,"tag":127,"props":229,"children":231},{"id":230},"the-layer-caching-trick",[232],{"type":23,"value":233},"The layer caching trick",{"type":17,"tag":25,"props":235,"children":236},{},[237,239,245,247,253],{"type":23,"value":238},"Docker builds images in layers. Each ",{"type":17,"tag":53,"props":240,"children":242},{"className":241},[],[243],{"type":23,"value":244},"COPY",{"type":23,"value":246}," or ",{"type":17,"tag":53,"props":248,"children":250},{"className":249},[],[251],{"type":23,"value":252},"RUN",{"type":23,"value":254}," instruction creates a layer. If a layer's input hasn't changed since the last build, Docker reuses the cached version.",{"type":17,"tag":25,"props":256,"children":257},{},[258,260,266,268,274,276,281],{"type":23,"value":259},"This is why you copy ",{"type":17,"tag":53,"props":261,"children":263},{"className":262},[],[264],{"type":23,"value":265},"requirements.txt",{"type":23,"value":267}," and install dependencies ",{"type":17,"tag":269,"props":270,"children":271},"em",{},[272],{"type":23,"value":273},"before",{"type":23,"value":275}," copying your code. If you change ",{"type":17,"tag":53,"props":277,"children":279},{"className":278},[],[280],{"type":23,"value":67},{"type":23,"value":282},", Docker reuses the cached pip install layer. If you did it all in one step, changing any file would re-trigger a full pip install.",{"type":17,"tag":48,"props":284,"children":287},{"code":285,"language":89,"meta":8,"className":286},"# Good: dependencies are cached separately from code\nCOPY requirements.txt .\nRUN pip install --no-cache-dir -r requirements.txt\nCOPY serve.py model.pkl .\u002F\n\n# Bad: any file change re-installs all dependencies\nCOPY . .\nRUN pip install --no-cache-dir -r requirements.txt\n",[91],[288],{"type":17,"tag":53,"props":289,"children":290},{"__ignoreMap":8},[291],{"type":23,"value":285},{"type":17,"tag":25,"props":293,"children":294},{},[295],{"type":23,"value":296},"This matters when your dependencies include PyTorch (2+ GB). You don't want to re-download it every time you fix a typo in your serving code.",{"type":17,"tag":36,"props":298,"children":300},{"id":299},"keeping-images-small",[301],{"type":23,"value":302},"Keeping images small",{"type":17,"tag":25,"props":304,"children":305},{},[306],{"type":23,"value":307},"ML images get big fast. PyTorch alone is over 2 GB. Add CUDA, your model weights, and some data processing libraries and you're looking at 10+ GB images. Big images mean slow pulls, slow deploys, and wasted disk space.",{"type":17,"tag":127,"props":309,"children":311},{"id":310},"multi-stage-builds",[312],{"type":23,"value":313},"Multi-stage builds",{"type":17,"tag":25,"props":315,"children":316},{},[317],{"type":23,"value":318},"A multi-stage build uses one image to install\u002Fbuild things and a second image to run them. Only the second image ships.",{"type":17,"tag":48,"props":320,"children":323},{"code":321,"language":89,"meta":8,"className":322},"# Stage 1: install dependencies\nFROM python:3.11-slim AS builder\n\nWORKDIR \u002Fapp\nCOPY requirements.txt .\nRUN pip install --no-cache-dir --prefix=\u002Finstall -r requirements.txt\n\n# Stage 2: runtime image\nFROM python:3.11-slim\n\nWORKDIR \u002Fapp\nCOPY --from=builder \u002Finstall \u002Fusr\u002Flocal\nCOPY serve.py model.pkl .\u002F\n\nCMD [\"uvicorn\", \"serve:app\", \"--host\", \"0.0.0.0\", \"--port\", \"8000\"]\n",[91],[324],{"type":17,"tag":53,"props":325,"children":326},{"__ignoreMap":8},[327],{"type":23,"value":321},{"type":17,"tag":25,"props":329,"children":330},{},[331,333,339,341,347],{"type":23,"value":332},"The ",{"type":17,"tag":53,"props":334,"children":336},{"className":335},[],[337],{"type":23,"value":338},"builder",{"type":23,"value":340}," stage installs packages into ",{"type":17,"tag":53,"props":342,"children":344},{"className":343},[],[345],{"type":23,"value":346},"\u002Finstall",{"type":23,"value":348},". The runtime stage copies only the installed packages — no pip cache, no build tools, no compiler artifacts. The final image is smaller because it doesn't contain anything needed only at build time.",{"type":17,"tag":127,"props":350,"children":352},{"id":351},"other-size-reduction-tactics",[353],{"type":23,"value":354},"Other size-reduction tactics",{"type":17,"tag":25,"props":356,"children":357},{},[358,371],{"type":17,"tag":142,"props":359,"children":360},{},[361,363,369],{"type":23,"value":362},"Use ",{"type":17,"tag":53,"props":364,"children":366},{"className":365},[],[367],{"type":23,"value":368},"--no-cache-dir",{"type":23,"value":370}," with pip.",{"type":23,"value":372}," Without it, pip caches downloaded packages inside the image. These caches serve no purpose in a container.",{"type":17,"tag":25,"props":374,"children":375},{},[376,381,383,388,390,396,398,404,406,412,414,420],{"type":17,"tag":142,"props":377,"children":378},{},[379],{"type":23,"value":380},"Don't install dev dependencies.",{"type":23,"value":382}," If your ",{"type":17,"tag":53,"props":384,"children":386},{"className":385},[],[387],{"type":23,"value":265},{"type":23,"value":389}," includes ",{"type":17,"tag":53,"props":391,"children":393},{"className":392},[],[394],{"type":23,"value":395},"pytest",{"type":23,"value":397},", ",{"type":17,"tag":53,"props":399,"children":401},{"className":400},[],[402],{"type":23,"value":403},"jupyter",{"type":23,"value":405},", or ",{"type":17,"tag":53,"props":407,"children":409},{"className":408},[],[410],{"type":23,"value":411},"matplotlib",{"type":23,"value":413}," but you don't need them at serving time, create a separate ",{"type":17,"tag":53,"props":415,"children":417},{"className":416},[],[418],{"type":23,"value":419},"requirements-serve.txt",{"type":23,"value":421}," with only the serving dependencies.",{"type":17,"tag":25,"props":423,"children":424},{},[425,437,439,444,446,452,454,460,462,468,469,475],{"type":17,"tag":142,"props":426,"children":427},{},[428,429,435],{"type":23,"value":362},{"type":17,"tag":53,"props":430,"children":432},{"className":431},[],[433],{"type":23,"value":434},".dockerignore",{"type":23,"value":436},".",{"type":23,"value":438}," Without a ",{"type":17,"tag":53,"props":440,"children":442},{"className":441},[],[443],{"type":23,"value":434},{"type":23,"value":445}," file, ",{"type":17,"tag":53,"props":447,"children":449},{"className":448},[],[450],{"type":23,"value":451},"COPY . .",{"type":23,"value":453}," copies everything — including your ",{"type":17,"tag":53,"props":455,"children":457},{"className":456},[],[458],{"type":23,"value":459},".git",{"type":23,"value":461}," directory, ",{"type":17,"tag":53,"props":463,"children":465},{"className":464},[],[466],{"type":23,"value":467},"__pycache__",{"type":23,"value":397},{"type":17,"tag":53,"props":470,"children":472},{"className":471},[],[473],{"type":23,"value":474},".venv",{"type":23,"value":476},", and data files. Create one:",{"type":17,"tag":48,"props":478,"children":480},{"code":479},".git\n__pycache__\n*.pyc\n.venv\ndata\u002F\nnotebooks\u002F\n.env\n",[481],{"type":17,"tag":53,"props":482,"children":483},{"__ignoreMap":8},[484],{"type":23,"value":479},{"type":17,"tag":25,"props":486,"children":487},{},[488,493,495,504,506,511],{"type":17,"tag":142,"props":489,"children":490},{},[491],{"type":23,"value":492},"Consider distroless or Alpine.",{"type":23,"value":494}," If you don't need a shell or package manager at runtime, Google's ",{"type":17,"tag":496,"props":497,"children":501},"a",{"href":498,"rel":499},"https:\u002F\u002Fgithub.com\u002FGoogleContainerTools\u002Fdistroless",[500],"nofollow",[502],{"type":23,"value":503},"distroless",{"type":23,"value":505}," Python images are even smaller than ",{"type":17,"tag":53,"props":507,"children":509},{"className":508},[],[510],{"type":23,"value":158},{"type":23,"value":512},". Alpine is another option but can cause issues with Python packages that have C extensions.",{"type":17,"tag":36,"props":514,"children":516},{"id":515},"gpu-images-with-nvidia",[517],{"type":23,"value":518},"GPU images with NVIDIA",{"type":17,"tag":25,"props":520,"children":521},{},[522],{"type":23,"value":523},"If your model needs GPU inference (deep learning, large models), you need NVIDIA's CUDA runtime in the container. This means using a different base image.",{"type":17,"tag":48,"props":525,"children":528},{"code":526,"language":89,"meta":8,"className":527},"FROM nvidia\u002Fcuda:12.1.1-runtime-ubuntu22.04\n\n# Install Python (NVIDIA images don't include it)\nRUN apt-get update && \\\n apt-get install -y --no-install-recommends python3 python3-pip && \\\n rm -rf \u002Fvar\u002Flib\u002Fapt\u002Flists\u002F* && \\\n ln -s \u002Fusr\u002Fbin\u002Fpython3 \u002Fusr\u002Fbin\u002Fpython\n\nWORKDIR \u002Fapp\nCOPY requirements.txt .\nRUN pip install --no-cache-dir -r requirements.txt\n\nCOPY serve.py model\u002F .\u002F\n\nCMD [\"uvicorn\", \"serve:app\", \"--host\", \"0.0.0.0\", \"--port\", \"8000\"]\n",[91],[529],{"type":17,"tag":53,"props":530,"children":531},{"__ignoreMap":8},[532],{"type":23,"value":526},{"type":17,"tag":25,"props":534,"children":535},{},[536],{"type":23,"value":537},"NVIDIA provides several image variants:",{"type":17,"tag":134,"props":539,"children":540},{},[541,555,569],{"type":17,"tag":138,"props":542,"children":543},{},[544,553],{"type":17,"tag":142,"props":545,"children":546},{},[547],{"type":17,"tag":53,"props":548,"children":550},{"className":549},[],[551],{"type":23,"value":552},"base",{"type":23,"value":554}," — CUDA driver libraries only. The smallest option.",{"type":17,"tag":138,"props":556,"children":557},{},[558,567],{"type":17,"tag":142,"props":559,"children":560},{},[561],{"type":17,"tag":53,"props":562,"children":564},{"className":563},[],[565],{"type":23,"value":566},"runtime",{"type":23,"value":568}," — adds CUDA runtime libraries. Use this for inference.",{"type":17,"tag":138,"props":570,"children":571},{},[572,581],{"type":17,"tag":142,"props":573,"children":574},{},[575],{"type":17,"tag":53,"props":576,"children":578},{"className":577},[],[579],{"type":23,"value":580},"devel",{"type":23,"value":582}," — adds compilers and headers. Use this if you need to compile CUDA code (most people don't for serving).",{"type":17,"tag":25,"props":584,"children":585},{},[586,588,593],{"type":23,"value":587},"For inference, ",{"type":17,"tag":53,"props":589,"children":591},{"className":590},[],[592],{"type":23,"value":566},{"type":23,"value":594}," is almost always what you want.",{"type":17,"tag":127,"props":596,"children":598},{"id":597},"running-gpu-containers",[599],{"type":23,"value":600},"Running GPU containers",{"type":17,"tag":25,"props":602,"children":603},{},[604],{"type":23,"value":605},"You need two things on the host:",{"type":17,"tag":607,"props":608,"children":609},"ol",{},[610,620],{"type":17,"tag":138,"props":611,"children":612},{},[613,618],{"type":17,"tag":142,"props":614,"children":615},{},[616],{"type":23,"value":617},"NVIDIA drivers",{"type":23,"value":619}," installed on the host machine (not in the container).",{"type":17,"tag":138,"props":621,"children":622},{},[623,628,630,636],{"type":17,"tag":142,"props":624,"children":625},{},[626],{"type":23,"value":627},"NVIDIA Container Toolkit",{"type":23,"value":629}," (",{"type":17,"tag":53,"props":631,"children":633},{"className":632},[],[634],{"type":23,"value":635},"nvidia-ctk",{"type":23,"value":637},") installed, so Docker can pass GPU devices into the container.",{"type":17,"tag":48,"props":639,"children":642},{"code":640,"language":105,"meta":8,"className":641},"docker run --gpus all -p 8000:8000 my-model-gpu:v1\n",[107],[643],{"type":17,"tag":53,"props":644,"children":645},{"__ignoreMap":8},[646],{"type":23,"value":640},{"type":17,"tag":25,"props":648,"children":649},{},[650,651,657,659,665],{"type":23,"value":332},{"type":17,"tag":53,"props":652,"children":654},{"className":653},[],[655],{"type":23,"value":656},"--gpus all",{"type":23,"value":658}," flag makes all host GPUs available inside the container. You can also specify ",{"type":17,"tag":53,"props":660,"children":662},{"className":661},[],[663],{"type":23,"value":664},"--gpus '\"device=0\"'",{"type":23,"value":666}," for a specific GPU.",{"type":17,"tag":25,"props":668,"children":669},{},[670],{"type":23,"value":671},"On Kubernetes, you'd request GPUs in the Pod spec instead:",{"type":17,"tag":48,"props":673,"children":678},{"code":674,"language":675,"meta":8,"className":676},"resources:\n limits:\n nvidia.com\u002Fgpu: 1\n","yaml",[677],"language-yaml",[679],{"type":17,"tag":53,"props":680,"children":681},{"__ignoreMap":8},[682],{"type":23,"value":674},{"type":17,"tag":25,"props":684,"children":685},{},[686],{"type":23,"value":687},"The NVIDIA device plugin handles the rest.",{"type":17,"tag":127,"props":689,"children":691},{"id":690},"pytorch-specific-tip",[692],{"type":23,"value":693},"PyTorch-specific tip",{"type":17,"tag":25,"props":695,"children":696},{},[697],{"type":23,"value":698},"PyTorch distributes separate packages for CPU and GPU. The GPU version includes CUDA bindings and is much larger. Use the right one:",{"type":17,"tag":48,"props":700,"children":702},{"code":701},"# GPU (includes CUDA)\n--extra-index-url https:\u002F\u002Fdownload.pytorch.org\u002Fwhl\u002Fcu121\ntorch==2.1.0\n\n# CPU only (much smaller)\n--extra-index-url https:\u002F\u002Fdownload.pytorch.org\u002Fwhl\u002Fcpu\ntorch==2.1.0\n",[703],{"type":17,"tag":53,"props":704,"children":705},{"__ignoreMap":8},[706],{"type":23,"value":701},{"type":17,"tag":25,"props":708,"children":709},{},[710],{"type":23,"value":711},"If your serving code runs on CPU, use the CPU wheel. Your image will be gigabytes smaller.",{"type":17,"tag":36,"props":713,"children":715},{"id":714},"pushing-to-a-registry",[716],{"type":23,"value":717},"Pushing to a registry",{"type":17,"tag":25,"props":719,"children":720},{},[721],{"type":23,"value":722},"A Docker image on your laptop isn't useful to anyone else. Push it to a container registry so your deployment pipeline can pull it.",{"type":17,"tag":127,"props":724,"children":726},{"id":725},"docker-hub",[727],{"type":23,"value":728},"Docker Hub",{"type":17,"tag":48,"props":730,"children":733},{"code":731,"language":105,"meta":8,"className":732},"docker tag my-model:v1 yourusername\u002Fmy-model:v1\ndocker push yourusername\u002Fmy-model:v1\n",[107],[734],{"type":17,"tag":53,"props":735,"children":736},{"__ignoreMap":8},[737],{"type":23,"value":731},{"type":17,"tag":25,"props":739,"children":740},{},[741],{"type":23,"value":742},"Docker Hub is public by default (free tier). Fine for open-source, not for proprietary models.",{"type":17,"tag":127,"props":744,"children":746},{"id":745},"private-registry-github-container-registry",[747],{"type":23,"value":748},"Private registry (GitHub Container Registry)",{"type":17,"tag":48,"props":750,"children":753},{"code":751,"language":105,"meta":8,"className":752},"echo $GITHUB_TOKEN | docker login ghcr.io -u USERNAME --password-stdin\ndocker tag my-model:v1 ghcr.io\u002Fyourorg\u002Fmy-model:v1\ndocker push ghcr.io\u002Fyourorg\u002Fmy-model:v1\n",[107],[754],{"type":17,"tag":53,"props":755,"children":756},{"__ignoreMap":8},[757],{"type":23,"value":751},{"type":17,"tag":127,"props":759,"children":761},{"id":760},"private-registry-self-hosted",[762],{"type":23,"value":763},"Private registry (self-hosted)",{"type":17,"tag":25,"props":765,"children":766},{},[767],{"type":23,"value":768},"If you're on-prem, you might run your own registry (Harbor, or the basic Docker registry):",{"type":17,"tag":48,"props":770,"children":773},{"code":771,"language":105,"meta":8,"className":772},"docker tag my-model:v1 registry.internal.company.com\u002Fml\u002Fmy-model:v1\ndocker push registry.internal.company.com\u002Fml\u002Fmy-model:v1\n",[107],[774],{"type":17,"tag":53,"props":775,"children":776},{"__ignoreMap":8},[777],{"type":23,"value":771},{"type":17,"tag":127,"props":779,"children":781},{"id":780},"tagging-strategy",[782],{"type":23,"value":783},"Tagging strategy",{"type":17,"tag":25,"props":785,"children":786},{},[787,789,795],{"type":23,"value":788},"Don't use ",{"type":17,"tag":53,"props":790,"children":792},{"className":791},[],[793],{"type":23,"value":794},":latest",{"type":23,"value":796},". It's mutable — every push overwrites it. You won't know which version is running in production.",{"type":17,"tag":25,"props":798,"children":799},{},[800],{"type":23,"value":801},"Use one of these:",{"type":17,"tag":134,"props":803,"children":804},{},[805,837,854],{"type":17,"tag":138,"props":806,"children":807},{},[808,813,815,821,822,828,829,835],{"type":17,"tag":142,"props":809,"children":810},{},[811],{"type":23,"value":812},"Version tags:",{"type":23,"value":814}," ",{"type":17,"tag":53,"props":816,"children":818},{"className":817},[],[819],{"type":23,"value":820},":v1",{"type":23,"value":397},{"type":17,"tag":53,"props":823,"children":825},{"className":824},[],[826],{"type":23,"value":827},":v2",{"type":23,"value":397},{"type":17,"tag":53,"props":830,"children":832},{"className":831},[],[833],{"type":23,"value":834},":v3",{"type":23,"value":836},". Simple and clear.",{"type":17,"tag":138,"props":838,"children":839},{},[840,845,846,852],{"type":17,"tag":142,"props":841,"children":842},{},[843],{"type":23,"value":844},"Git SHA:",{"type":23,"value":814},{"type":17,"tag":53,"props":847,"children":849},{"className":848},[],[850],{"type":23,"value":851},":sha-a1b2c3d",{"type":23,"value":853},". Ties the image to a specific commit.",{"type":17,"tag":138,"props":855,"children":856},{},[857,862,863,869],{"type":17,"tag":142,"props":858,"children":859},{},[860],{"type":23,"value":861},"Date-based:",{"type":23,"value":814},{"type":17,"tag":53,"props":864,"children":866},{"className":865},[],[867],{"type":23,"value":868},":2025-07-11",{"type":23,"value":870},". Useful for models that retrain on a schedule.",{"type":17,"tag":25,"props":872,"children":873},{},[874],{"type":23,"value":875},"Tag every image with something immutable. When something breaks in production, you need to know exactly which image is running.",{"type":17,"tag":36,"props":877,"children":879},{"id":878},"the-full-example",[880],{"type":23,"value":881},"The full example",{"type":17,"tag":25,"props":883,"children":884},{},[885],{"type":23,"value":886},"Putting it all together — a multi-stage Dockerfile for a model serving endpoint with health checks:",{"type":17,"tag":48,"props":888,"children":891},{"code":889,"language":89,"meta":8,"className":890},"# Stage 1: install dependencies\nFROM python:3.11-slim AS builder\n\nWORKDIR \u002Fbuild\nCOPY requirements.txt .\nRUN pip install --no-cache-dir --prefix=\u002Finstall -r requirements.txt\n\n# Stage 2: runtime\nFROM python:3.11-slim\n\nWORKDIR \u002Fapp\n\n# Copy installed packages from builder\nCOPY --from=builder \u002Finstall \u002Fusr\u002Flocal\n\n# Copy application code\nCOPY serve.py .\nCOPY model\u002F model\u002F\n\n# Non-root user for security\nRUN useradd --create-home appuser\nUSER appuser\n\nEXPOSE 8000\nHEALTHCHECK --interval=30s --timeout=5s --retries=3 \\\n CMD python -c \"import urllib.request; urllib.request.urlopen('http:\u002F\u002Flocalhost:8000\u002Fhealth')\"\n\nCMD [\"uvicorn\", \"serve:app\", \"--host\", \"0.0.0.0\", \"--port\", \"8000\"]\n",[91],[892],{"type":17,"tag":53,"props":893,"children":894},{"__ignoreMap":8},[895],{"type":23,"value":889},{"type":17,"tag":25,"props":897,"children":898},{},[899],{"type":23,"value":900},"Build, tag, push:",{"type":17,"tag":48,"props":902,"children":905},{"code":903,"language":105,"meta":8,"className":904},"docker build -t registry.example.com\u002Fml\u002Fmy-model:v1 .\ndocker push registry.example.com\u002Fml\u002Fmy-model:v1\n",[107],[906],{"type":17,"tag":53,"props":907,"children":908},{"__ignoreMap":8},[909],{"type":23,"value":903},{"type":17,"tag":25,"props":911,"children":912},{},[913],{"type":23,"value":914},"Deploy to Kubernetes:",{"type":17,"tag":48,"props":916,"children":919},{"code":917,"language":675,"meta":8,"className":918},"apiVersion: apps\u002Fv1\nkind: Deployment\nmetadata:\n name: my-model\nspec:\n replicas: 2\n selector:\n matchLabels:\n app: my-model\n template:\n metadata:\n labels:\n app: my-model\n spec:\n containers:\n - name: serve\n image: registry.example.com\u002Fml\u002Fmy-model:v1\n ports:\n - containerPort: 8000\n livenessProbe:\n httpGet:\n path: \u002Fhealth\n port: 8000\n readinessProbe:\n httpGet:\n path: \u002Fhealth\n port: 8000\n resources:\n requests:\n memory: \"512Mi\"\n cpu: \"250m\"\n limits:\n memory: \"1Gi\"\n cpu: \"500m\"\n",[677],[920],{"type":17,"tag":53,"props":921,"children":922},{"__ignoreMap":8},[923],{"type":23,"value":917},{"type":17,"tag":25,"props":925,"children":926},{},[927,929,935],{"type":23,"value":928},"The model is now running in a container, behind a health check, with resource limits, and can be scaled horizontally by changing ",{"type":17,"tag":53,"props":930,"children":932},{"className":931},[],[933],{"type":23,"value":934},"replicas",{"type":23,"value":936},". Anyone with access to the registry can deploy it without knowing anything about Python versions, pip, or your machine's environment.",{"type":17,"tag":938,"props":939,"children":940},"hr",{},[],{"type":17,"tag":25,"props":942,"children":943},{},[944,946,953,955,961],{"type":23,"value":945},"Containerizing ML workloads is a core part of what we set up at ",{"type":17,"tag":496,"props":947,"children":950},{"href":948,"rel":949},"https:\u002F\u002Fdeploying.ai",[500],[951],{"type":23,"value":952},"deploying.ai",{"type":23,"value":954},". If you need help building reliable ML deployment pipelines, ",{"type":17,"tag":496,"props":956,"children":958},{"href":957},"mailto:vlad@deploying.ai",[959],{"type":23,"value":960},"reach out",{"type":23,"value":436},{"title":8,"searchDepth":963,"depth":963,"links":964},2,[965,970,974,978,984],{"id":38,"depth":963,"text":41,"children":966},[967,969],{"id":129,"depth":968,"text":132},3,{"id":230,"depth":968,"text":233},{"id":299,"depth":963,"text":302,"children":971},[972,973],{"id":310,"depth":968,"text":313},{"id":351,"depth":968,"text":354},{"id":515,"depth":963,"text":518,"children":975},[976,977],{"id":597,"depth":968,"text":600},{"id":690,"depth":968,"text":693},{"id":714,"depth":963,"text":717,"children":979},[980,981,982,983],{"id":725,"depth":968,"text":728},{"id":745,"depth":968,"text":748},{"id":760,"depth":968,"text":763},{"id":780,"depth":968,"text":783},{"id":878,"depth":963,"text":881},"markdown","content:articles:docker-for-ml-engineers.md","content","articles\u002Fdocker-for-ml-engineers.md","articles\u002Fdocker-for-ml-engineers","md",{"_path":992,"_dir":6,"_draft":7,"_partial":7,"_locale":8,"title":993,"description":994,"date":995,"aiGenerated":12,"body":996,"_type":985,"_id":1641,"_source":987,"_file":1642,"_stem":1643,"_extension":990},"\u002Farticles\u002Fend-to-end-ml-model","End-to-End ML Model Walkthrough: From model.fit() to Production","Most ML tutorials stop at training. This walkthrough covers the full loop — training a toy model, tracking experiments with MLflow, deploying to a serving endpoint, and automating retraining with scheduled GPU jobs.","2025-07-28",{"type":14,"children":997,"toc":1631},[998,1003,1016,1021,1026,1032,1037,1046,1051,1057,1062,1075,1133,1138,1144,1149,1158,1163,1206,1211,1217,1222,1231,1236,1279,1284,1290,1295,1300,1343,1348,1354,1359,1381,1386,1419,1424,1433,1445,1450,1456,1461,1544,1549,1554,1560,1565,1575,1585,1595,1605,1610,1613],{"type":17,"tag":18,"props":999,"children":1001},{"id":1000},"end-to-end-ml-model-walkthrough-from-modelfit-to-production",[1002],{"type":23,"value":993},{"type":17,"tag":25,"props":1004,"children":1005},{},[1006,1008,1014],{"type":23,"value":1007},"Most machine learning tutorials end at ",{"type":17,"tag":53,"props":1009,"children":1011},{"className":1010},[],[1012],{"type":23,"value":1013},"model.fit()",{"type":23,"value":1015},". You train a model, print the accuracy, and the tutorial is done. But that's maybe 10% of the real work.",{"type":17,"tag":25,"props":1017,"children":1018},{},[1019],{"type":23,"value":1020},"What happens next? How does the model get into production? How do you retrain it when the data changes? How do you know if the new version is better than the old one? These are the questions that separate a notebook experiment from a working ML system.",{"type":17,"tag":25,"props":1022,"children":1023},{},[1024],{"type":23,"value":1025},"This article walks through the full lifecycle of a deliberately simple model. The model itself is trivial — that's the point. The infrastructure around it is what matters.",{"type":17,"tag":36,"props":1027,"children":1029},{"id":1028},"the-toy-model",[1030],{"type":23,"value":1031},"The toy model",{"type":17,"tag":25,"props":1033,"children":1034},{},[1035],{"type":23,"value":1036},"We'll use a scikit-learn classifier on the Iris dataset. Three classes, four features, 150 samples. You can't get simpler than this.",{"type":17,"tag":48,"props":1038,"children":1041},{"className":1039,"code":1040,"language":73,"meta":8},[75],"from sklearn.datasets import load_iris\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\n\nX, y = load_iris(return_X_y=True)\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n\nmodel = RandomForestClassifier(n_estimators=100, max_depth=5)\nmodel.fit(X_train, y_train)\n\npredictions = model.predict(X_test)\nprint(f\"Accuracy: {accuracy_score(y_test, predictions):.3f}\")\n",[1042],{"type":17,"tag":53,"props":1043,"children":1044},{"__ignoreMap":8},[1045],{"type":23,"value":1040},{"type":17,"tag":25,"props":1047,"children":1048},{},[1049],{"type":23,"value":1050},"This is where most tutorials stop. You have a model. It works. Now what?",{"type":17,"tag":36,"props":1052,"children":1054},{"id":1053},"the-first-training-run",[1055],{"type":23,"value":1056},"The first training run",{"type":17,"tag":25,"props":1058,"children":1059},{},[1060],{"type":23,"value":1061},"Before adding any infrastructure, let's think about what this training script actually produces and what questions come up the moment you try to use the result.",{"type":17,"tag":25,"props":1063,"children":1064},{},[1065,1067,1073],{"type":23,"value":1066},"The output is a fitted model object sitting in memory. If you want to keep it, you ",{"type":17,"tag":53,"props":1068,"children":1070},{"className":1069},[],[1071],{"type":23,"value":1072},"pickle.dump()",{"type":23,"value":1074}," it to a file. And immediately you have problems:",{"type":17,"tag":134,"props":1076,"children":1077},{},[1078,1088,1113,1123],{"type":17,"tag":138,"props":1079,"children":1080},{},[1081,1086],{"type":17,"tag":142,"props":1082,"children":1083},{},[1084],{"type":23,"value":1085},"Where does the file go?",{"type":23,"value":1087}," A local directory? A shared drive? An object store?",{"type":17,"tag":138,"props":1089,"children":1090},{},[1091,1096,1097,1103,1105,1111],{"type":17,"tag":142,"props":1092,"children":1093},{},[1094],{"type":23,"value":1095},"What do you name it?",{"type":23,"value":814},{"type":17,"tag":53,"props":1098,"children":1100},{"className":1099},[],[1101],{"type":23,"value":1102},"model.pkl",{"type":23,"value":1104},"? ",{"type":17,"tag":53,"props":1106,"children":1108},{"className":1107},[],[1109],{"type":23,"value":1110},"model_v2_final_FINAL.pkl",{"type":23,"value":1112},"?",{"type":17,"tag":138,"props":1114,"children":1115},{},[1116,1121],{"type":17,"tag":142,"props":1117,"children":1118},{},[1119],{"type":23,"value":1120},"What produced it?",{"type":23,"value":1122}," Which version of the code? What hyperparameters? What data?",{"type":17,"tag":138,"props":1124,"children":1125},{},[1126,1131],{"type":17,"tag":142,"props":1127,"children":1128},{},[1129],{"type":23,"value":1130},"Is it any good?",{"type":23,"value":1132}," What was the accuracy? On what test set?",{"type":17,"tag":25,"props":1134,"children":1135},{},[1136],{"type":23,"value":1137},"You can solve each of these with discipline and naming conventions. Or you can use a tool that was built for exactly this.",{"type":17,"tag":36,"props":1139,"children":1141},{"id":1140},"mlflow-enters-the-picture",[1142],{"type":23,"value":1143},"MLflow enters the picture",{"type":17,"tag":25,"props":1145,"children":1146},{},[1147],{"type":23,"value":1148},"MLflow is an experiment tracker. At its core, it answers: \"what did I run, with what settings, and what happened?\" Here's the same training script with MLflow:",{"type":17,"tag":48,"props":1150,"children":1153},{"className":1151,"code":1152,"language":73,"meta":8},[75],"import mlflow\nfrom sklearn.datasets import load_iris\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\n\nmlflow.set_tracking_uri(\"http:\u002F\u002Fmlflow.internal:5000\")\nmlflow.set_experiment(\"iris-classifier\")\n\nwith mlflow.start_run():\n X, y = load_iris(return_X_y=True)\n X_train, X_test, y_train, y_test = train_test_split(\n X, y, test_size=0.2, random_state=42\n )\n\n n_estimators = 100\n max_depth = 5\n\n mlflow.log_param(\"n_estimators\", n_estimators)\n mlflow.log_param(\"max_depth\", max_depth)\n mlflow.log_param(\"test_size\", 0.2)\n\n model = RandomForestClassifier(\n n_estimators=n_estimators, max_depth=max_depth\n )\n model.fit(X_train, y_train)\n\n predictions = model.predict(X_test)\n acc = accuracy_score(y_test, predictions)\n mlflow.log_metric(\"accuracy\", acc)\n\n mlflow.sklearn.log_model(model, \"model\")\n",[1154],{"type":17,"tag":53,"props":1155,"children":1156},{"__ignoreMap":8},[1157],{"type":23,"value":1152},{"type":17,"tag":25,"props":1159,"children":1160},{},[1161],{"type":23,"value":1162},"Not much more code, but now you get:",{"type":17,"tag":134,"props":1164,"children":1165},{},[1166,1176,1186,1196],{"type":17,"tag":138,"props":1167,"children":1168},{},[1169,1174],{"type":17,"tag":142,"props":1170,"children":1171},{},[1172],{"type":23,"value":1173},"Experiment history.",{"type":23,"value":1175}," Every run is recorded with its parameters and metrics. You can compare ten different hyperparameter combinations side by side.",{"type":17,"tag":138,"props":1177,"children":1178},{},[1179,1184],{"type":17,"tag":142,"props":1180,"children":1181},{},[1182],{"type":23,"value":1183},"Artifact storage.",{"type":23,"value":1185}," The model is serialized and stored in a known location — not a random pickle file on someone's laptop.",{"type":17,"tag":138,"props":1187,"children":1188},{},[1189,1194],{"type":17,"tag":142,"props":1190,"children":1191},{},[1192],{"type":23,"value":1193},"Reproducibility metadata.",{"type":23,"value":1195}," MLflow logs the git commit, the Python environment, and the exact parameters. Six months from now, you can see exactly what produced this model.",{"type":17,"tag":138,"props":1197,"children":1198},{},[1199,1204],{"type":17,"tag":142,"props":1200,"children":1201},{},[1202],{"type":23,"value":1203},"A model registry.",{"type":23,"value":1205}," You can promote a run's model to \"Staging\" or \"Production\" in the registry, giving you a single source of truth for which model is currently live.",{"type":17,"tag":25,"props":1207,"children":1208},{},[1209],{"type":23,"value":1210},"The difference between MLflow and a pickle file is the same as the difference between a version control system and a folder of zip files. You can survive without it, but you shouldn't.",{"type":17,"tag":36,"props":1212,"children":1214},{"id":1213},"deploying-the-model",[1215],{"type":23,"value":1216},"Deploying the model",{"type":17,"tag":25,"props":1218,"children":1219},{},[1220],{"type":23,"value":1221},"A model in a registry is still just a file. To be useful, it needs to serve predictions. There are many ways to do this — here's one straightforward approach using a Flask endpoint:",{"type":17,"tag":48,"props":1223,"children":1226},{"className":1224,"code":1225,"language":73,"meta":8},[75],"import mlflow\nfrom flask import Flask, request, jsonify\n\napp = Flask(__name__)\n\n# Load the production model from the registry\nmodel = mlflow.sklearn.load_model(\"models:\u002Firis-classifier\u002FProduction\")\n\n@app.route(\"\u002Fpredict\", methods=[\"POST\"])\ndef predict():\n data = request.json[\"features\"]\n prediction = model.predict([data])\n return jsonify({\"prediction\": int(prediction[0])})\n",[1227],{"type":17,"tag":53,"props":1228,"children":1229},{"__ignoreMap":8},[1230],{"type":23,"value":1225},{"type":17,"tag":25,"props":1232,"children":1233},{},[1234],{"type":23,"value":1235},"This is a minimal serving layer. In production, you'd want to think about:",{"type":17,"tag":134,"props":1237,"children":1238},{},[1239,1249,1259,1269],{"type":17,"tag":138,"props":1240,"children":1241},{},[1242,1247],{"type":17,"tag":142,"props":1243,"children":1244},{},[1245],{"type":23,"value":1246},"Containerization.",{"type":23,"value":1248}," Package the serving code and its dependencies into a Docker image so it runs the same everywhere.",{"type":17,"tag":138,"props":1250,"children":1251},{},[1252,1257],{"type":17,"tag":142,"props":1253,"children":1254},{},[1255],{"type":23,"value":1256},"Health checks and readiness probes.",{"type":23,"value":1258}," Kubernetes needs to know when the service is ready to accept traffic.",{"type":17,"tag":138,"props":1260,"children":1261},{},[1262,1267],{"type":17,"tag":142,"props":1263,"children":1264},{},[1265],{"type":23,"value":1266},"Input validation.",{"type":23,"value":1268}," Don't trust the incoming data — validate shapes, types, and ranges before feeding them to the model.",{"type":17,"tag":138,"props":1270,"children":1271},{},[1272,1277],{"type":17,"tag":142,"props":1273,"children":1274},{},[1275],{"type":23,"value":1276},"Logging predictions.",{"type":23,"value":1278}," Store what the model predicted and what it received. You'll need this for monitoring and debugging.",{"type":17,"tag":25,"props":1280,"children":1281},{},[1282],{"type":23,"value":1283},"The important architectural point: the serving layer pulls the model from the registry. It doesn't know or care how the model was trained. This decouples deployment from training — you can update the model without touching the serving code.",{"type":17,"tag":36,"props":1285,"children":1287},{"id":1286},"why-models-need-retraining",[1288],{"type":23,"value":1289},"Why models need retraining",{"type":17,"tag":25,"props":1291,"children":1292},{},[1293],{"type":23,"value":1294},"Deploy a model and it starts degrading. This isn't a bug — it's a fundamental property of ML systems. The world changes, and the model's training data becomes a less accurate picture of reality.",{"type":17,"tag":25,"props":1296,"children":1297},{},[1298],{"type":23,"value":1299},"Common reasons for retraining:",{"type":17,"tag":134,"props":1301,"children":1302},{},[1303,1313,1323,1333],{"type":17,"tag":138,"props":1304,"children":1305},{},[1306,1311],{"type":17,"tag":142,"props":1307,"children":1308},{},[1309],{"type":23,"value":1310},"Data drift.",{"type":23,"value":1312}," The distribution of incoming data shifts away from the training distribution. A fraud detection model trained on pre-pandemic transaction patterns will struggle with post-pandemic shopping behavior.",{"type":17,"tag":138,"props":1314,"children":1315},{},[1316,1321],{"type":17,"tag":142,"props":1317,"children":1318},{},[1319],{"type":23,"value":1320},"Concept drift.",{"type":23,"value":1322}," The relationship between inputs and outputs changes. What counted as spam email five years ago looks different from spam today.",{"type":17,"tag":138,"props":1324,"children":1325},{},[1326,1331],{"type":17,"tag":142,"props":1327,"children":1328},{},[1329],{"type":23,"value":1330},"New data availability.",{"type":23,"value":1332}," You've collected more labeled data since the last training run. The model should benefit from it.",{"type":17,"tag":138,"props":1334,"children":1335},{},[1336,1341],{"type":17,"tag":142,"props":1337,"children":1338},{},[1339],{"type":23,"value":1340},"Feature changes.",{"type":23,"value":1342}," Upstream data pipelines evolve — columns get added, formats change, sources get replaced.",{"type":17,"tag":25,"props":1344,"children":1345},{},[1346],{"type":23,"value":1347},"The question isn't whether to retrain. It's how to retrain reliably, automatically, and without a human babysitting the process.",{"type":17,"tag":36,"props":1349,"children":1351},{"id":1350},"scheduled-gpu-jobs-with-kueue",[1352],{"type":23,"value":1353},"Scheduled GPU jobs with Kueue",{"type":17,"tag":25,"props":1355,"children":1356},{},[1357],{"type":23,"value":1358},"Retraining needs compute. For deep learning models, that means GPUs. Even for our simple scikit-learn example, the pattern matters because it scales to real workloads.",{"type":17,"tag":25,"props":1360,"children":1361},{},[1362,1364,1370,1372,1379],{"type":23,"value":1363},"On Kubernetes, the standard way to run a one-off job is a ",{"type":17,"tag":53,"props":1365,"children":1367},{"className":1366},[],[1368],{"type":23,"value":1369},"Job",{"type":23,"value":1371}," resource. But GPUs are expensive and scarce — you can't just let everyone submit jobs whenever they want. This is where ",{"type":17,"tag":496,"props":1373,"children":1376},{"href":1374,"rel":1375},"https:\u002F\u002Fkueue.sigs.k8s.io\u002F",[500],[1377],{"type":23,"value":1378},"Kueue",{"type":23,"value":1380}," comes in.",{"type":17,"tag":25,"props":1382,"children":1383},{},[1384],{"type":23,"value":1385},"Kueue is a Kubernetes-native job queueing system. It manages:",{"type":17,"tag":134,"props":1387,"children":1388},{},[1389,1399,1409],{"type":17,"tag":138,"props":1390,"children":1391},{},[1392,1397],{"type":17,"tag":142,"props":1393,"children":1394},{},[1395],{"type":23,"value":1396},"Quotas.",{"type":23,"value":1398}," Team A gets 4 GPUs, Team B gets 8. No one can starve out another team.",{"type":17,"tag":138,"props":1400,"children":1401},{},[1402,1407],{"type":17,"tag":142,"props":1403,"children":1404},{},[1405],{"type":23,"value":1406},"Priorities.",{"type":23,"value":1408}," Production retraining jobs preempt experimental runs.",{"type":17,"tag":138,"props":1410,"children":1411},{},[1412,1417],{"type":17,"tag":142,"props":1413,"children":1414},{},[1415],{"type":23,"value":1416},"Fair sharing.",{"type":23,"value":1418}," Idle resources get redistributed rather than sitting unused.",{"type":17,"tag":25,"props":1420,"children":1421},{},[1422],{"type":23,"value":1423},"A scheduled retraining job looks something like this:",{"type":17,"tag":48,"props":1425,"children":1428},{"className":1426,"code":1427,"language":675,"meta":8},[677],"apiVersion: batch\u002Fv1\nkind: CronJob\nmetadata:\n name: iris-retrain\n namespace: ml-jobs\nspec:\n schedule: \"0 2 * * 0\" # Every Sunday at 2am\n jobTemplate:\n spec:\n template:\n metadata:\n labels:\n kueue.x-k8s.io\u002Fqueue-name: ml-training-queue\n spec:\n containers:\n - name: train\n image: ml-training:latest\n command: [\"python\", \"train.py\"]\n resources:\n limits:\n nvidia.com\u002Fgpu: 1\n restartPolicy: Never\n",[1429],{"type":17,"tag":53,"props":1430,"children":1431},{"__ignoreMap":8},[1432],{"type":23,"value":1427},{"type":17,"tag":25,"props":1434,"children":1435},{},[1436,1437,1443],{"type":23,"value":332},{"type":17,"tag":53,"props":1438,"children":1440},{"className":1439},[],[1441],{"type":23,"value":1442},"CronJob",{"type":23,"value":1444}," handles scheduling. Kueue handles resource allocation. The training script inside logs everything to MLflow. If the new model performs better than the current production model, it gets promoted in the registry. The serving layer picks it up.",{"type":17,"tag":25,"props":1446,"children":1447},{},[1448],{"type":23,"value":1449},"No human intervention required.",{"type":17,"tag":36,"props":1451,"children":1453},{"id":1452},"the-full-loop",[1454],{"type":23,"value":1455},"The full loop",{"type":17,"tag":25,"props":1457,"children":1458},{},[1459],{"type":23,"value":1460},"Zoom out and you can see the complete lifecycle:",{"type":17,"tag":607,"props":1462,"children":1463},{},[1464,1474,1484,1494,1504,1514,1524,1534],{"type":17,"tag":138,"props":1465,"children":1466},{},[1467,1472],{"type":17,"tag":142,"props":1468,"children":1469},{},[1470],{"type":23,"value":1471},"Data",{"type":23,"value":1473}," arrives — new samples, updated labels, corrected features.",{"type":17,"tag":138,"props":1475,"children":1476},{},[1477,1482],{"type":17,"tag":142,"props":1478,"children":1479},{},[1480],{"type":23,"value":1481},"Training",{"type":23,"value":1483}," runs on a schedule (or is triggered by data changes). It uses GPU resources managed by Kueue.",{"type":17,"tag":138,"props":1485,"children":1486},{},[1487,1492],{"type":17,"tag":142,"props":1488,"children":1489},{},[1490],{"type":23,"value":1491},"Experiment tracking",{"type":23,"value":1493}," via MLflow records every run's parameters, metrics, and artifacts.",{"type":17,"tag":138,"props":1495,"children":1496},{},[1497,1502],{"type":17,"tag":142,"props":1498,"children":1499},{},[1500],{"type":23,"value":1501},"Evaluation",{"type":23,"value":1503}," compares the new model against the current production model on a held-out test set.",{"type":17,"tag":138,"props":1505,"children":1506},{},[1507,1512],{"type":17,"tag":142,"props":1508,"children":1509},{},[1510],{"type":23,"value":1511},"Registration",{"type":23,"value":1513}," promotes the new model in the MLflow registry if it passes evaluation.",{"type":17,"tag":138,"props":1515,"children":1516},{},[1517,1522],{"type":17,"tag":142,"props":1518,"children":1519},{},[1520],{"type":23,"value":1521},"Deployment",{"type":23,"value":1523}," picks up the new model version — either automatically or via a promotion step.",{"type":17,"tag":138,"props":1525,"children":1526},{},[1527,1532],{"type":17,"tag":142,"props":1528,"children":1529},{},[1530],{"type":23,"value":1531},"Monitoring",{"type":23,"value":1533}," watches prediction quality, latency, and data drift in production.",{"type":17,"tag":138,"props":1535,"children":1536},{},[1537,1542],{"type":17,"tag":142,"props":1538,"children":1539},{},[1540],{"type":23,"value":1541},"Retraining",{"type":23,"value":1543}," is triggered again when monitoring detects degradation, or simply on schedule.",{"type":17,"tag":25,"props":1545,"children":1546},{},[1547],{"type":23,"value":1548},"Each of these steps involves infrastructure decisions. Where does the data live? How are GPUs allocated? Who can promote a model? How do you roll back? What counts as \"better\"?",{"type":17,"tag":25,"props":1550,"children":1551},{},[1552],{"type":23,"value":1553},"The model itself — our little random forest — is almost an afterthought. The value is in the system that surrounds it.",{"type":17,"tag":36,"props":1555,"children":1557},{"id":1556},"what-this-looks-like-in-practice",[1558],{"type":23,"value":1559},"What this looks like in practice",{"type":17,"tag":25,"props":1561,"children":1562},{},[1563],{"type":23,"value":1564},"For a team getting started with MLOps, you don't need all of this on day one. A reasonable progression:",{"type":17,"tag":25,"props":1566,"children":1567},{},[1568,1573],{"type":17,"tag":142,"props":1569,"children":1570},{},[1571],{"type":23,"value":1572},"Week one:",{"type":23,"value":1574}," Get MLflow running. Start logging experiments instead of losing them. This alone is a major improvement — you can compare runs and reproduce results.",{"type":17,"tag":25,"props":1576,"children":1577},{},[1578,1583],{"type":17,"tag":142,"props":1579,"children":1580},{},[1581],{"type":23,"value":1582},"Next:",{"type":23,"value":1584}," Containerize your training script. Run it as a Kubernetes Job. You've now decoupled \"where training runs\" from \"who runs it.\"",{"type":17,"tag":25,"props":1586,"children":1587},{},[1588,1593],{"type":17,"tag":142,"props":1589,"children":1590},{},[1591],{"type":23,"value":1592},"Then:",{"type":23,"value":1594}," Add Kueue for resource management. Set up a CronJob for scheduled retraining. Your model stays fresh without manual effort.",{"type":17,"tag":25,"props":1596,"children":1597},{},[1598,1603],{"type":17,"tag":142,"props":1599,"children":1600},{},[1601],{"type":23,"value":1602},"Finally:",{"type":23,"value":1604}," Build the promotion pipeline. Automated evaluation, registry promotion, serving layer updates. The full loop.",{"type":17,"tag":25,"props":1606,"children":1607},{},[1608],{"type":23,"value":1609},"Each step is independently valuable. You don't need to build the whole thing before any of it is useful.",{"type":17,"tag":938,"props":1611,"children":1612},{},[],{"type":17,"tag":25,"props":1614,"children":1615},{},[1616,1618,1623,1625,1630],{"type":23,"value":1617},"This is the kind of infrastructure work we help teams build at ",{"type":17,"tag":496,"props":1619,"children":1621},{"href":948,"rel":1620},[500],[1622],{"type":23,"value":952},{"type":23,"value":1624},". If you're past the notebook stage and need your models running reliably in production, ",{"type":17,"tag":496,"props":1626,"children":1627},{"href":957},[1628],{"type":23,"value":1629},"let's talk",{"type":23,"value":436},{"title":8,"searchDepth":963,"depth":963,"links":1632},[1633,1634,1635,1636,1637,1638,1639,1640],{"id":1028,"depth":963,"text":1031},{"id":1053,"depth":963,"text":1056},{"id":1140,"depth":963,"text":1143},{"id":1213,"depth":963,"text":1216},{"id":1286,"depth":963,"text":1289},{"id":1350,"depth":963,"text":1353},{"id":1452,"depth":963,"text":1455},{"id":1556,"depth":963,"text":1559},"content:articles:end-to-end-ml-model.md","articles\u002Fend-to-end-ml-model.md","articles\u002Fend-to-end-ml-model",{"_path":1645,"_dir":6,"_draft":7,"_partial":7,"_locale":8,"title":1646,"description":1647,"date":1648,"aiGenerated":12,"body":1649,"_type":985,"_id":2346,"_source":987,"_file":2347,"_stem":2348,"_extension":990},"\u002Farticles\u002Ffive-things-break-notebook-to-production","Five Things That Break When You Move a Model From Notebook to Production","The gap between a working notebook and a working production model is filled with subtle failures. Here are the five most common ones and how to fix them.","2025-07-14",{"type":14,"children":1650,"toc":2328},[1651,1656,1661,1666,1672,1677,1682,1690,1703,1709,1737,1746,1766,1775,1794,1800,1805,1814,1827,1832,1873,1878,1891,1900,1921,1926,1932,1937,1942,1993,1998,2003,2012,2021,2026,2031,2037,2042,2047,2090,2095,2100,2109,2114,2123,2128,2134,2155,2160,2165,2178,2187,2192,2263,2268,2273,2282,2287,2293,2298,2303,2308,2311],{"type":17,"tag":18,"props":1652,"children":1654},{"id":1653},"five-things-that-break-when-you-move-a-model-from-notebook-to-production",[1655],{"type":23,"value":1646},{"type":17,"tag":25,"props":1657,"children":1658},{},[1659],{"type":23,"value":1660},"Your model works in a notebook. The accuracy is good, the plots look right, your colleagues are impressed. You decide to deploy it.",{"type":17,"tag":25,"props":1662,"children":1663},{},[1664],{"type":23,"value":1665},"Then things break — not dramatically, but in quiet, frustrating ways. The predictions are wrong but not obviously wrong. The service crashes under load. The model works on your machine but not in the container. These are the most common failure modes, and they're all avoidable.",{"type":17,"tag":36,"props":1667,"children":1669},{"id":1668},"_1-dependency-mismatches",[1670],{"type":23,"value":1671},"1. Dependency mismatches",{"type":17,"tag":25,"props":1673,"children":1674},{},[1675],{"type":23,"value":1676},"The notebook runs on your machine with whatever packages you've installed over the past year. The production environment is different — different Python version, different library versions, different OS.",{"type":17,"tag":25,"props":1678,"children":1679},{},[1680],{"type":23,"value":1681},"The classic symptom: the model loads fine but produces different predictions. Or it throws a cryptic deserialization error.",{"type":17,"tag":48,"props":1683,"children":1685},{"code":1684},"ModuleNotFoundError: No module named 'sklearn.ensemble._forest'\n",[1686],{"type":17,"tag":53,"props":1687,"children":1688},{"__ignoreMap":8},[1689],{"type":23,"value":1684},{"type":17,"tag":25,"props":1691,"children":1692},{},[1693,1695,1701],{"type":23,"value":1694},"This happens when you train with scikit-learn 1.3 and try to load the pickle with scikit-learn 1.2. The internal module paths changed between versions, and ",{"type":17,"tag":53,"props":1696,"children":1698},{"className":1697},[],[1699],{"type":23,"value":1700},"pickle",{"type":23,"value":1702}," serializes the full path.",{"type":17,"tag":127,"props":1704,"children":1706},{"id":1705},"the-fix",[1707],{"type":23,"value":1708},"The fix",{"type":17,"tag":25,"props":1710,"children":1711},{},[1712,1714,1720,1722,1728,1730,1736],{"type":23,"value":1713},"Pin every dependency with exact versions. Not ",{"type":17,"tag":53,"props":1715,"children":1717},{"className":1716},[],[1718],{"type":23,"value":1719},"scikit-learn>=1.0",{"type":23,"value":1721},", not ",{"type":17,"tag":53,"props":1723,"children":1725},{"className":1724},[],[1726],{"type":23,"value":1727},"scikit-learn~=1.3",{"type":23,"value":1729},". Use ",{"type":17,"tag":53,"props":1731,"children":1733},{"className":1732},[],[1734],{"type":23,"value":1735},"scikit-learn==1.3.2",{"type":23,"value":436},{"type":17,"tag":48,"props":1738,"children":1741},{"code":1739,"language":105,"meta":8,"className":1740},"pip freeze > requirements.txt\n",[107],[1742],{"type":17,"tag":53,"props":1743,"children":1744},{"__ignoreMap":8},[1745],{"type":23,"value":1739},{"type":17,"tag":25,"props":1747,"children":1748},{},[1749,1751,1757,1759,1764],{"type":23,"value":1750},"Better yet, use the same Docker image for training and serving. If the model was trained in ",{"type":17,"tag":53,"props":1752,"children":1754},{"className":1753},[],[1755],{"type":23,"value":1756},"python:3.11-slim",{"type":23,"value":1758}," with ",{"type":17,"tag":53,"props":1760,"children":1762},{"className":1761},[],[1763],{"type":23,"value":1735},{"type":23,"value":1765},", serve it from the same image. This eliminates version mismatches entirely.",{"type":17,"tag":48,"props":1767,"children":1770},{"code":1768,"language":89,"meta":8,"className":1769},"FROM python:3.11-slim\nCOPY requirements.txt .\nRUN pip install --no-cache-dir -r requirements.txt\nCOPY serve.py model\u002F\nCMD [\"python\", \"serve.py\"]\n",[91],[1771],{"type":17,"tag":53,"props":1772,"children":1773},{"__ignoreMap":8},[1774],{"type":23,"value":1768},{"type":17,"tag":25,"props":1776,"children":1777},{},[1778,1780,1786,1787,1792],{"type":23,"value":1779},"MLflow helps here too — it logs the ",{"type":17,"tag":53,"props":1781,"children":1783},{"className":1782},[],[1784],{"type":23,"value":1785},"conda.yaml",{"type":23,"value":246},{"type":17,"tag":53,"props":1788,"children":1790},{"className":1789},[],[1791],{"type":23,"value":265},{"type":23,"value":1793}," alongside the model artifact, so you can always reconstruct the exact environment that produced a model.",{"type":17,"tag":36,"props":1795,"children":1797},{"id":1796},"_2-missing-preprocessing-steps",[1798],{"type":23,"value":1799},"2. Missing preprocessing steps",{"type":17,"tag":25,"props":1801,"children":1802},{},[1803],{"type":23,"value":1804},"This one is insidious. Your notebook has a cell that normalizes the data:",{"type":17,"tag":48,"props":1806,"children":1809},{"code":1807,"language":73,"meta":8,"className":1808},"from sklearn.preprocessing import StandardScaler\n\nscaler = StandardScaler()\nX_train_scaled = scaler.fit_transform(X_train)\nX_test_scaled = scaler.transform(X_test)\n\nmodel.fit(X_train_scaled, y_train)\n",[75],[1810],{"type":17,"tag":53,"props":1811,"children":1812},{"__ignoreMap":8},[1813],{"type":23,"value":1807},{"type":17,"tag":25,"props":1815,"children":1816},{},[1817,1819,1825],{"type":23,"value":1818},"The model is trained on scaled data. It expects scaled inputs. But the serving code receives raw features and passes them directly to ",{"type":17,"tag":53,"props":1820,"children":1822},{"className":1821},[],[1823],{"type":23,"value":1824},"model.predict()",{"type":23,"value":1826},". The predictions are garbage, but the model doesn't throw an error — it just returns wrong answers confidently.",{"type":17,"tag":25,"props":1828,"children":1829},{},[1830],{"type":23,"value":1831},"Variations of this problem:",{"type":17,"tag":134,"props":1833,"children":1834},{},[1835,1845,1863],{"type":17,"tag":138,"props":1836,"children":1837},{},[1838,1843],{"type":17,"tag":142,"props":1839,"children":1840},{},[1841],{"type":23,"value":1842},"Feature encoding",{"type":23,"value":1844}," trained in the notebook but missing in production. Categorical variables that were one-hot encoded during training arrive as raw strings at serving time.",{"type":17,"tag":138,"props":1846,"children":1847},{},[1848,1853,1855,1861],{"type":17,"tag":142,"props":1849,"children":1850},{},[1851],{"type":23,"value":1852},"Feature engineering",{"type":23,"value":1854}," done in a notebook cell that nobody copied to the serving code. The model expects a ",{"type":17,"tag":53,"props":1856,"children":1858},{"className":1857},[],[1859],{"type":23,"value":1860},"price_per_sqft",{"type":23,"value":1862}," feature that only exists in the notebook.",{"type":17,"tag":138,"props":1864,"children":1865},{},[1866,1871],{"type":17,"tag":142,"props":1867,"children":1868},{},[1869],{"type":23,"value":1870},"Missing imputation.",{"type":23,"value":1872}," The notebook filled NaN values with the column median. The serving code passes NaN values through, and the model does something unpredictable.",{"type":17,"tag":127,"props":1874,"children":1876},{"id":1875},"the-fix-1",[1877],{"type":23,"value":1708},{"type":17,"tag":25,"props":1879,"children":1880},{},[1881,1883,1889],{"type":23,"value":1882},"Bundle preprocessing with the model. Use a scikit-learn ",{"type":17,"tag":53,"props":1884,"children":1886},{"className":1885},[],[1887],{"type":23,"value":1888},"Pipeline",{"type":23,"value":1890},":",{"type":17,"tag":48,"props":1892,"children":1895},{"code":1893,"language":73,"meta":8,"className":1894},"from sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.ensemble import RandomForestClassifier\n\npipeline = Pipeline([\n (\"scaler\", StandardScaler()),\n (\"classifier\", RandomForestClassifier(n_estimators=100)),\n])\n\npipeline.fit(X_train, y_train)\n",[75],[1896],{"type":17,"tag":53,"props":1897,"children":1898},{"__ignoreMap":8},[1899],{"type":23,"value":1893},{"type":17,"tag":25,"props":1901,"children":1902},{},[1903,1905,1911,1913,1919],{"type":23,"value":1904},"Now ",{"type":17,"tag":53,"props":1906,"children":1908},{"className":1907},[],[1909],{"type":23,"value":1910},"pipeline.predict(X_raw)",{"type":23,"value":1912}," handles scaling internally. The serving code doesn't need to know about preprocessing — it just calls ",{"type":17,"tag":53,"props":1914,"children":1916},{"className":1915},[],[1917],{"type":23,"value":1918},"predict()",{"type":23,"value":1920}," on whatever comes in.",{"type":17,"tag":25,"props":1922,"children":1923},{},[1924],{"type":23,"value":1925},"For more complex preprocessing (text tokenization, image resizing, feature engineering), the same principle applies: serialize the entire transformation chain, not just the model. If preprocessing can't be part of the model object, make it a separate artifact that gets deployed alongside the model with an explicit contract between the two.",{"type":17,"tag":36,"props":1927,"children":1929},{"id":1928},"_3-no-input-validation",[1930],{"type":23,"value":1931},"3. No input validation",{"type":17,"tag":25,"props":1933,"children":1934},{},[1935],{"type":23,"value":1936},"In the notebook, you control the input. You loaded the dataset, cleaned it, and shaped it correctly. In production, input comes from users, APIs, and upstream services — and it will be wrong in ways you didn't anticipate.",{"type":17,"tag":25,"props":1938,"children":1939},{},[1940],{"type":23,"value":1941},"Common failures:",{"type":17,"tag":134,"props":1943,"children":1944},{},[1945,1955,1965,1975],{"type":17,"tag":138,"props":1946,"children":1947},{},[1948,1953],{"type":17,"tag":142,"props":1949,"children":1950},{},[1951],{"type":23,"value":1952},"Wrong number of features.",{"type":23,"value":1954}," The model expects 10 features, the request sends 9. NumPy reshapes silently and the prediction is meaningless.",{"type":17,"tag":138,"props":1956,"children":1957},{},[1958,1963],{"type":17,"tag":142,"props":1959,"children":1960},{},[1961],{"type":23,"value":1962},"Wrong data types.",{"type":23,"value":1964}," A feature that should be float arrives as a string. Or an integer arrives as a float with a decimal point.",{"type":17,"tag":138,"props":1966,"children":1967},{},[1968,1973],{"type":17,"tag":142,"props":1969,"children":1970},{},[1971],{"type":23,"value":1972},"Out-of-range values.",{"type":23,"value":1974}," The model was trained on ages between 18 and 90. A request arrives with age = -1 or age = 500.",{"type":17,"tag":138,"props":1976,"children":1977},{},[1978,1983,1985,1991],{"type":17,"tag":142,"props":1979,"children":1980},{},[1981],{"type":23,"value":1982},"Missing fields.",{"type":23,"value":1984}," An optional field in the API is omitted and arrives as ",{"type":17,"tag":53,"props":1986,"children":1988},{"className":1987},[],[1989],{"type":23,"value":1990},"None",{"type":23,"value":1992},", which propagates through the model as NaN.",{"type":17,"tag":127,"props":1994,"children":1996},{"id":1995},"the-fix-2",[1997],{"type":23,"value":1708},{"type":17,"tag":25,"props":1999,"children":2000},{},[2001],{"type":23,"value":2002},"Validate at the API boundary. Define the expected schema and reject bad input before it reaches the model.",{"type":17,"tag":48,"props":2004,"children":2007},{"code":2005,"language":73,"meta":8,"className":2006},"from pydantic import BaseModel, validator\nfrom typing import List\n\nclass PredictionRequest(BaseModel):\n features: List[float]\n\n @validator(\"features\")\n def check_feature_count(cls, v):\n if len(v) != 4:\n raise ValueError(f\"Expected 4 features, got {len(v)}\")\n return v\n",[75],[2008],{"type":17,"tag":53,"props":2009,"children":2010},{"__ignoreMap":8},[2011],{"type":23,"value":2005},{"type":17,"tag":48,"props":2013,"children":2016},{"code":2014,"language":73,"meta":8,"className":2015},"from fastapi import FastAPI, HTTPException\n\napp = FastAPI()\n\n@app.post(\"\u002Fpredict\")\ndef predict(request: PredictionRequest):\n try:\n result = model.predict([request.features])\n return {\"prediction\": int(result[0])}\n except Exception as e:\n raise HTTPException(status_code=500, detail=str(e))\n",[75],[2017],{"type":17,"tag":53,"props":2018,"children":2019},{"__ignoreMap":8},[2020],{"type":23,"value":2014},{"type":17,"tag":25,"props":2022,"children":2023},{},[2024],{"type":23,"value":2025},"This returns a clear 422 error when input is wrong, instead of returning a wrong prediction that's indistinguishable from a correct one.",{"type":17,"tag":25,"props":2027,"children":2028},{},[2029],{"type":23,"value":2030},"Validation isn't glamorous, but it's the difference between \"the model is broken\" (it isn't — the input is wrong) and \"bad request, here's what you need to fix.\"",{"type":17,"tag":36,"props":2032,"children":2034},{"id":2033},"_4-no-logging-or-monitoring",[2035],{"type":23,"value":2036},"4. No logging or monitoring",{"type":17,"tag":25,"props":2038,"children":2039},{},[2040],{"type":23,"value":2041},"In the notebook, you see the predictions right there in the output cell. In production, the model runs inside a service, receives requests, and returns responses — and nobody is watching.",{"type":17,"tag":25,"props":2043,"children":2044},{},[2045],{"type":23,"value":2046},"Without logging, you can't answer basic questions:",{"type":17,"tag":134,"props":2048,"children":2049},{},[2050,2060,2070,2080],{"type":17,"tag":138,"props":2051,"children":2052},{},[2053,2058],{"type":17,"tag":142,"props":2054,"children":2055},{},[2056],{"type":23,"value":2057},"Is the model being used?",{"type":23,"value":2059}," How many predictions per day? Is traffic increasing or decreasing?",{"type":17,"tag":138,"props":2061,"children":2062},{},[2063,2068],{"type":17,"tag":142,"props":2064,"children":2065},{},[2066],{"type":23,"value":2067},"What is it predicting?",{"type":23,"value":2069}," If the model starts predicting the same class for every input, that's a problem — but you won't know without logging.",{"type":17,"tag":138,"props":2071,"children":2072},{},[2073,2078],{"type":17,"tag":142,"props":2074,"children":2075},{},[2076],{"type":23,"value":2077},"How fast is it?",{"type":23,"value":2079}," If prediction latency spikes from 10ms to 500ms, users notice before you do.",{"type":17,"tag":138,"props":2081,"children":2082},{},[2083,2088],{"type":17,"tag":142,"props":2084,"children":2085},{},[2086],{"type":23,"value":2087},"Is the input distribution changing?",{"type":23,"value":2089}," If the average feature values shift significantly from training data, the model's predictions become unreliable. This is data drift.",{"type":17,"tag":127,"props":2091,"children":2093},{"id":2092},"the-fix-3",[2094],{"type":23,"value":1708},{"type":17,"tag":25,"props":2096,"children":2097},{},[2098],{"type":23,"value":2099},"Log predictions, inputs (or summaries of inputs), and latency from day one.",{"type":17,"tag":48,"props":2101,"children":2104},{"code":2102,"language":73,"meta":8,"className":2103},"import time\nimport logging\n\nlogger = logging.getLogger(\"model-serving\")\n\n@app.post(\"\u002Fpredict\")\ndef predict(request: PredictionRequest):\n start = time.time()\n\n result = model.predict([request.features])\n prediction = int(result[0])\n\n latency = time.time() - start\n logger.info(\n \"prediction\",\n extra={\n \"prediction\": prediction,\n \"feature_count\": len(request.features),\n \"latency_ms\": round(latency * 1000, 2),\n },\n )\n\n return {\"prediction\": prediction}\n",[75],[2105],{"type":17,"tag":53,"props":2106,"children":2107},{"__ignoreMap":8},[2108],{"type":23,"value":2102},{"type":17,"tag":25,"props":2110,"children":2111},{},[2112],{"type":23,"value":2113},"For metrics, expose a Prometheus endpoint:",{"type":17,"tag":48,"props":2115,"children":2118},{"code":2116,"language":73,"meta":8,"className":2117},"from prometheus_client import Histogram, Counter\n\nPREDICTION_LATENCY = Histogram(\n \"prediction_latency_seconds\",\n \"Time spent processing prediction\",\n)\nPREDICTION_COUNT = Counter(\n \"predictions_total\",\n \"Total predictions\",\n [\"predicted_class\"],\n)\n\n@app.post(\"\u002Fpredict\")\ndef predict(request: PredictionRequest):\n with PREDICTION_LATENCY.time():\n result = model.predict([request.features])\n prediction = int(result[0])\n\n PREDICTION_COUNT.labels(predicted_class=str(prediction)).inc()\n return {\"prediction\": prediction}\n",[75],[2119],{"type":17,"tag":53,"props":2120,"children":2121},{"__ignoreMap":8},[2122],{"type":23,"value":2116},{"type":17,"tag":25,"props":2124,"children":2125},{},[2126],{"type":23,"value":2127},"You don't need a full observability platform on day one. Structured logs and a few Prometheus metrics get you surprisingly far. The key is having them from the start — retrofitting monitoring after a production incident is stressful.",{"type":17,"tag":36,"props":2129,"children":2131},{"id":2130},"_5-works-on-my-machine-deployment",[2132],{"type":23,"value":2133},"5. \"Works on my machine\" deployment",{"type":17,"tag":25,"props":2135,"children":2136},{},[2137,2139,2145,2147,2153],{"type":23,"value":2138},"The model works locally. You ",{"type":17,"tag":53,"props":2140,"children":2142},{"className":2141},[],[2143],{"type":23,"value":2144},"scp",{"type":23,"value":2146}," the model file and the serving script to a VM, run ",{"type":17,"tag":53,"props":2148,"children":2150},{"className":2149},[],[2151],{"type":23,"value":2152},"python serve.py",{"type":23,"value":2154},", and it works. Three weeks later the VM is rebooted, Python is updated by a system package upgrade, and the service doesn't come back up.",{"type":17,"tag":25,"props":2156,"children":2157},{},[2158],{"type":23,"value":2159},"Or: you hand the model to the platform team and say \"deploy this.\" They ask: \"How? What Python version? What dependencies? Does it need a GPU? How much memory? What port? What's the health check endpoint?\" You don't know the answers to half of these.",{"type":17,"tag":127,"props":2161,"children":2163},{"id":2162},"the-fix-4",[2164],{"type":23,"value":1708},{"type":17,"tag":25,"props":2166,"children":2167},{},[2168,2170,2176],{"type":23,"value":2169},"Containerize the entire serving stack. A ",{"type":17,"tag":53,"props":2171,"children":2173},{"className":2172},[],[2174],{"type":23,"value":2175},"Dockerfile",{"type":23,"value":2177}," is a deployment specification — it captures everything needed to run the model.",{"type":17,"tag":48,"props":2179,"children":2182},{"code":2180,"language":89,"meta":8,"className":2181},"FROM python:3.11-slim\n\nWORKDIR \u002Fapp\nCOPY requirements.txt .\nRUN pip install --no-cache-dir -r requirements.txt\n\nCOPY serve.py .\nCOPY model\u002F model\u002F\n\nEXPOSE 8000\nHEALTHCHECK CMD curl -f http:\u002F\u002Flocalhost:8000\u002Fhealth || exit 1\n\nCMD [\"uvicorn\", \"serve:app\", \"--host\", \"0.0.0.0\", \"--port\", \"8000\"]\n",[91],[2183],{"type":17,"tag":53,"props":2184,"children":2185},{"__ignoreMap":8},[2186],{"type":23,"value":2180},{"type":17,"tag":25,"props":2188,"children":2189},{},[2190],{"type":23,"value":2191},"This answers every question the platform team had:",{"type":17,"tag":134,"props":2193,"children":2194},{},[2195,2205,2220,2238,2248],{"type":17,"tag":138,"props":2196,"children":2197},{},[2198,2203],{"type":17,"tag":142,"props":2199,"children":2200},{},[2201],{"type":23,"value":2202},"Python version:",{"type":23,"value":2204}," 3.11",{"type":17,"tag":138,"props":2206,"children":2207},{},[2208,2213,2215],{"type":17,"tag":142,"props":2209,"children":2210},{},[2211],{"type":23,"value":2212},"Dependencies:",{"type":23,"value":2214}," pinned in ",{"type":17,"tag":53,"props":2216,"children":2218},{"className":2217},[],[2219],{"type":23,"value":265},{"type":17,"tag":138,"props":2221,"children":2222},{},[2223,2228,2230,2236],{"type":17,"tag":142,"props":2224,"children":2225},{},[2226],{"type":23,"value":2227},"GPU:",{"type":23,"value":2229}," not needed (if it were, use ",{"type":17,"tag":53,"props":2231,"children":2233},{"className":2232},[],[2234],{"type":23,"value":2235},"nvidia\u002Fcuda",{"type":23,"value":2237}," base image)",{"type":17,"tag":138,"props":2239,"children":2240},{},[2241,2246],{"type":17,"tag":142,"props":2242,"children":2243},{},[2244],{"type":23,"value":2245},"Port:",{"type":23,"value":2247}," 8000",{"type":17,"tag":138,"props":2249,"children":2250},{},[2251,2256,2257],{"type":17,"tag":142,"props":2252,"children":2253},{},[2254],{"type":23,"value":2255},"Health check:",{"type":23,"value":814},{"type":17,"tag":53,"props":2258,"children":2260},{"className":2259},[],[2261],{"type":23,"value":2262},"GET \u002Fhealth",{"type":17,"tag":25,"props":2264,"children":2265},{},[2266],{"type":23,"value":2267},"The container runs the same way everywhere — on the developer's laptop, in CI, in staging, in production. The \"works on my machine\" problem is eliminated.",{"type":17,"tag":25,"props":2269,"children":2270},{},[2271],{"type":23,"value":2272},"Add a health check endpoint in the serving code:",{"type":17,"tag":48,"props":2274,"children":2277},{"code":2275,"language":73,"meta":8,"className":2276},"@app.get(\"\u002Fhealth\")\ndef health():\n return {\"status\": \"ok\"}\n",[75],[2278],{"type":17,"tag":53,"props":2279,"children":2280},{"__ignoreMap":8},[2281],{"type":23,"value":2275},{"type":17,"tag":25,"props":2283,"children":2284},{},[2285],{"type":23,"value":2286},"Kubernetes, ECS, or whatever orchestrator you use can hit this endpoint to know if the service is alive and ready.",{"type":17,"tag":36,"props":2288,"children":2290},{"id":2289},"the-pattern",[2291],{"type":23,"value":2292},"The pattern",{"type":17,"tag":25,"props":2294,"children":2295},{},[2296],{"type":23,"value":2297},"All five of these problems have the same root cause: the notebook environment is forgiving in ways that production isn't. In a notebook, you're the only user, you control the input, you can see the output, and the environment is whatever happens to be installed.",{"type":17,"tag":25,"props":2299,"children":2300},{},[2301],{"type":23,"value":2302},"Production is the opposite. Multiple users, arbitrary input, invisible output, and a fixed environment that must be explicitly defined.",{"type":17,"tag":25,"props":2304,"children":2305},{},[2306],{"type":23,"value":2307},"The fixes aren't complicated. Pin dependencies, bundle preprocessing, validate input, add logging, and containerize. None of these require advanced tools or infrastructure — they're habits that pay off immediately.",{"type":17,"tag":938,"props":2309,"children":2310},{},[],{"type":17,"tag":25,"props":2312,"children":2313},{},[2314,2316,2321,2323,2327],{"type":23,"value":2315},"Moving models from notebook to production is exactly what ",{"type":17,"tag":496,"props":2317,"children":2319},{"href":948,"rel":2318},[500],[2320],{"type":23,"value":952},{"type":23,"value":2322}," helps teams do. If you're hitting these problems (or want to avoid them), ",{"type":17,"tag":496,"props":2324,"children":2325},{"href":957},[2326],{"type":23,"value":1629},{"type":23,"value":436},{"title":8,"searchDepth":963,"depth":963,"links":2329},[2330,2333,2336,2339,2342,2345],{"id":1668,"depth":963,"text":1671,"children":2331},[2332],{"id":1705,"depth":968,"text":1708},{"id":1796,"depth":963,"text":1799,"children":2334},[2335],{"id":1875,"depth":968,"text":1708},{"id":1928,"depth":963,"text":1931,"children":2337},[2338],{"id":1995,"depth":968,"text":1708},{"id":2033,"depth":963,"text":2036,"children":2340},[2341],{"id":2092,"depth":968,"text":1708},{"id":2130,"depth":963,"text":2133,"children":2343},[2344],{"id":2162,"depth":968,"text":1708},{"id":2289,"depth":963,"text":2292},"content:articles:five-things-break-notebook-to-production.md","articles\u002Ffive-things-break-notebook-to-production.md","articles\u002Ffive-things-break-notebook-to-production",{"_path":2350,"_dir":6,"_draft":7,"_partial":7,"_locale":8,"title":2351,"description":2352,"date":2353,"aiGenerated":12,"body":2354,"_type":985,"_id":3049,"_source":987,"_file":3050,"_stem":3051,"_extension":990},"\u002Farticles\u002Fgpu-scheduling-kubernetes-kueue","GPU Scheduling on Kubernetes with Kueue","GPUs are expensive and shared. Kueue is a Kubernetes-native job queueing system that manages quotas, priorities, and fair sharing so teams can share a GPU cluster without stepping on each other.","2025-07-18",{"type":14,"children":2355,"toc":3033},[2356,2361,2366,2371,2381,2387,2392,2415,2420,2448,2472,2478,2483,2488,2497,2518,2523,2528,2537,2546,2559,2564,2569,2578,2587,2608,2614,2619,2628,2633,2653,2658,2663,2669,2680,2689,2701,2706,2739,2744,2750,2755,2764,2769,2778,2783,2789,2794,2803,2834,2839,2845,2850,2859,2879,2885,2890,2926,2945,2951,2959,2982,2990,3008,3013,3016],{"type":17,"tag":18,"props":2357,"children":2359},{"id":2358},"gpu-scheduling-on-kubernetes-with-kueue",[2360],{"type":23,"value":2351},{"type":17,"tag":25,"props":2362,"children":2363},{},[2364],{"type":23,"value":2365},"You have a Kubernetes cluster with GPUs. Multiple teams want to use them. What happens next is predictable: someone submits a large training job that grabs all the GPUs, and everyone else waits. Or worse, teams learn to hoard resources by keeping idle jobs running \"just in case.\"",{"type":17,"tag":25,"props":2367,"children":2368},{},[2369],{"type":23,"value":2370},"This is a scheduling problem, and Kubernetes alone doesn't solve it well. The default scheduler is first-come, first-served. It doesn't understand quotas, priorities, or fairness. It just assigns Pods to nodes that have the requested resources available.",{"type":17,"tag":25,"props":2372,"children":2373},{},[2374,2379],{"type":17,"tag":496,"props":2375,"children":2377},{"href":1374,"rel":2376},[500],[2378],{"type":23,"value":1378},{"type":23,"value":2380}," fills this gap. It's a Kubernetes-native job queueing system built by the Kubernetes community specifically for batch workloads like ML training.",{"type":17,"tag":36,"props":2382,"children":2384},{"id":2383},"the-problem-in-concrete-terms",[2385],{"type":23,"value":2386},"The problem in concrete terms",{"type":17,"tag":25,"props":2388,"children":2389},{},[2390],{"type":23,"value":2391},"Say you have a cluster with 8 GPUs and two teams:",{"type":17,"tag":134,"props":2393,"children":2394},{},[2395,2405],{"type":17,"tag":138,"props":2396,"children":2397},{},[2398,2403],{"type":17,"tag":142,"props":2399,"children":2400},{},[2401],{"type":23,"value":2402},"Team A",{"type":23,"value":2404}," (ML research) runs lots of experimental training jobs. They'd happily use all 8 GPUs if they could.",{"type":17,"tag":138,"props":2406,"children":2407},{},[2408,2413],{"type":17,"tag":142,"props":2409,"children":2410},{},[2411],{"type":23,"value":2412},"Team B",{"type":23,"value":2414}," (production ML) runs weekly retraining jobs. These are critical — if they don't run, production models go stale.",{"type":17,"tag":25,"props":2416,"children":2417},{},[2418],{"type":23,"value":2419},"Without Kueue, the scenario plays out like this:",{"type":17,"tag":607,"props":2421,"children":2422},{},[2423,2428,2433,2438,2443],{"type":17,"tag":138,"props":2424,"children":2425},{},[2426],{"type":23,"value":2427},"Team A submits 8 single-GPU training jobs on Monday morning.",{"type":17,"tag":138,"props":2429,"children":2430},{},[2431],{"type":23,"value":2432},"All 8 GPUs are allocated.",{"type":17,"tag":138,"props":2434,"children":2435},{},[2436],{"type":23,"value":2437},"Team B's production retraining job arrives Tuesday. It can't be scheduled — all GPUs are busy.",{"type":17,"tag":138,"props":2439,"children":2440},{},[2441],{"type":23,"value":2442},"Team B's job sits in Pending state for hours until one of Team A's experiments finishes.",{"type":17,"tag":138,"props":2444,"children":2445},{},[2446],{"type":23,"value":2447},"Team B complains. A policy meeting is scheduled. Nothing gets resolved.",{"type":17,"tag":25,"props":2449,"children":2450},{},[2451,2453,2458,2459,2464,2466,2471],{"type":23,"value":2452},"Kueue solves this with three concepts: ",{"type":17,"tag":142,"props":2454,"children":2455},{},[2456],{"type":23,"value":2457},"ResourceFlavors",{"type":23,"value":397},{"type":17,"tag":142,"props":2460,"children":2461},{},[2462],{"type":23,"value":2463},"ClusterQueues",{"type":23,"value":2465},", and ",{"type":17,"tag":142,"props":2467,"children":2468},{},[2469],{"type":23,"value":2470},"LocalQueues",{"type":23,"value":436},{"type":17,"tag":36,"props":2473,"children":2475},{"id":2474},"how-kueue-works",[2476],{"type":23,"value":2477},"How Kueue works",{"type":17,"tag":127,"props":2479,"children":2481},{"id":2480},"resourceflavors",[2482],{"type":23,"value":2457},{"type":17,"tag":25,"props":2484,"children":2485},{},[2486],{"type":23,"value":2487},"A ResourceFlavor describes a type of resource in your cluster. For GPUs, this maps to the actual hardware:",{"type":17,"tag":48,"props":2489,"children":2492},{"className":2490,"code":2491,"language":675,"meta":8},[677],"apiVersion: kueue.x-k8s.io\u002Fv1beta1\nkind: ResourceFlavor\nmetadata:\n name: gpu-a100\nspec:\n nodeLabels:\n gpu-type: a100\n",[2493],{"type":17,"tag":53,"props":2494,"children":2495},{"__ignoreMap":8},[2496],{"type":23,"value":2491},{"type":17,"tag":25,"props":2498,"children":2499},{},[2500,2502,2508,2510,2516],{"type":23,"value":2501},"This tells Kueue: \"there's a class of resources called ",{"type":17,"tag":53,"props":2503,"children":2505},{"className":2504},[],[2506],{"type":23,"value":2507},"gpu-a100",{"type":23,"value":2509}," that lives on nodes labeled ",{"type":17,"tag":53,"props":2511,"children":2513},{"className":2512},[],[2514],{"type":23,"value":2515},"gpu-type: a100",{"type":23,"value":2517},".\" If you have mixed GPU types (some A100s, some T4s), you'd create separate flavors for each.",{"type":17,"tag":127,"props":2519,"children":2521},{"id":2520},"clusterqueues",[2522],{"type":23,"value":2463},{"type":17,"tag":25,"props":2524,"children":2525},{},[2526],{"type":23,"value":2527},"A ClusterQueue defines a resource budget. This is where quotas live:",{"type":17,"tag":48,"props":2529,"children":2532},{"className":2530,"code":2531,"language":675,"meta":8},[677],"apiVersion: kueue.x-k8s.io\u002Fv1beta1\nkind: ClusterQueue\nmetadata:\n name: team-a-queue\nspec:\n cohort: gpu-cluster\n resourceGroups:\n - coveredResources: [\"cpu\", \"memory\", \"nvidia.com\u002Fgpu\"]\n flavors:\n - name: gpu-a100\n resources:\n - name: \"nvidia.com\u002Fgpu\"\n nominalQuota: 4\n - name: \"cpu\"\n nominalQuota: 32\n - name: \"memory\"\n nominalQuota: 128Gi\n",[2533],{"type":17,"tag":53,"props":2534,"children":2535},{"__ignoreMap":8},[2536],{"type":23,"value":2531},{"type":17,"tag":48,"props":2538,"children":2541},{"className":2539,"code":2540,"language":675,"meta":8},[677],"apiVersion: kueue.x-k8s.io\u002Fv1beta1\nkind: ClusterQueue\nmetadata:\n name: team-b-queue\nspec:\n cohort: gpu-cluster\n resourceGroups:\n - coveredResources: [\"cpu\", \"memory\", \"nvidia.com\u002Fgpu\"]\n flavors:\n - name: gpu-a100\n resources:\n - name: \"nvidia.com\u002Fgpu\"\n nominalQuota: 4\n - name: \"cpu\"\n nominalQuota: 32\n - name: \"memory\"\n nominalQuota: 128Gi\n",[2542],{"type":17,"tag":53,"props":2543,"children":2544},{"__ignoreMap":8},[2545],{"type":23,"value":2540},{"type":17,"tag":25,"props":2547,"children":2548},{},[2549,2551,2557],{"type":23,"value":2550},"Each team gets a nominal quota of 4 GPUs. The ",{"type":17,"tag":53,"props":2552,"children":2554},{"className":2553},[],[2555],{"type":23,"value":2556},"cohort",{"type":23,"value":2558}," field groups them together — more on that shortly.",{"type":17,"tag":127,"props":2560,"children":2562},{"id":2561},"localqueues",[2563],{"type":23,"value":2470},{"type":17,"tag":25,"props":2565,"children":2566},{},[2567],{"type":23,"value":2568},"A LocalQueue is a namespaced queue that points to a ClusterQueue. It's how users actually submit jobs:",{"type":17,"tag":48,"props":2570,"children":2573},{"className":2571,"code":2572,"language":675,"meta":8},[677],"apiVersion: kueue.x-k8s.io\u002Fv1beta1\nkind: LocalQueue\nmetadata:\n name: training-queue\n namespace: team-a\nspec:\n clusterQueue: team-a-queue\n",[2574],{"type":17,"tag":53,"props":2575,"children":2576},{"__ignoreMap":8},[2577],{"type":23,"value":2572},{"type":17,"tag":48,"props":2579,"children":2582},{"className":2580,"code":2581,"language":675,"meta":8},[677],"apiVersion: kueue.x-k8s.io\u002Fv1beta1\nkind: LocalQueue\nmetadata:\n name: training-queue\n namespace: team-b\nspec:\n clusterQueue: team-b-queue\n",[2583],{"type":17,"tag":53,"props":2584,"children":2585},{"__ignoreMap":8},[2586],{"type":23,"value":2581},{"type":17,"tag":25,"props":2588,"children":2589},{},[2590,2592,2598,2600,2606],{"type":23,"value":2591},"Team A submits jobs to ",{"type":17,"tag":53,"props":2593,"children":2595},{"className":2594},[],[2596],{"type":23,"value":2597},"training-queue",{"type":23,"value":2599}," in their namespace. Kueue routes them to ",{"type":17,"tag":53,"props":2601,"children":2603},{"className":2602},[],[2604],{"type":23,"value":2605},"team-a-queue",{"type":23,"value":2607}," and enforces the quota.",{"type":17,"tag":36,"props":2609,"children":2611},{"id":2610},"submitting-jobs",[2612],{"type":23,"value":2613},"Submitting jobs",{"type":17,"tag":25,"props":2615,"children":2616},{},[2617],{"type":23,"value":2618},"Jobs opt into Kueue by adding a label:",{"type":17,"tag":48,"props":2620,"children":2623},{"className":2621,"code":2622,"language":675,"meta":8},[677],"apiVersion: batch\u002Fv1\nkind: Job\nmetadata:\n name: experiment-42\n namespace: team-a\n labels:\n kueue.x-k8s.io\u002Fqueue-name: training-queue\nspec:\n template:\n spec:\n containers:\n - name: train\n image: training:latest\n resources:\n requests:\n nvidia.com\u002Fgpu: 1\n restartPolicy: Never\n",[2624],{"type":17,"tag":53,"props":2625,"children":2626},{"__ignoreMap":8},[2627],{"type":23,"value":2622},{"type":17,"tag":25,"props":2629,"children":2630},{},[2631],{"type":23,"value":2632},"When this Job is created, Kueue intercepts it. Instead of being scheduled immediately, it's suspended and placed in the queue. Kueue checks:",{"type":17,"tag":607,"props":2634,"children":2635},{},[2636,2648],{"type":17,"tag":138,"props":2637,"children":2638},{},[2639,2641,2646],{"type":23,"value":2640},"Does ",{"type":17,"tag":53,"props":2642,"children":2644},{"className":2643},[],[2645],{"type":23,"value":2605},{"type":23,"value":2647}," have available quota for 1 GPU?",{"type":17,"tag":138,"props":2649,"children":2650},{},[2651],{"type":23,"value":2652},"Is there an actual GPU available in the cluster?",{"type":17,"tag":25,"props":2654,"children":2655},{},[2656],{"type":23,"value":2657},"If both are true, Kueue unsuspends the Job and it runs. If not, it waits.",{"type":17,"tag":25,"props":2659,"children":2660},{},[2661],{"type":23,"value":2662},"This is the key difference from vanilla Kubernetes. Without Kueue, the Job would be created and the Pod would sit in Pending state, occupying a \"slot\" that other Jobs can't use. With Kueue, the Job is explicitly queued and managed.",{"type":17,"tag":36,"props":2664,"children":2666},{"id":2665},"fair-sharing-with-cohorts",[2667],{"type":23,"value":2668},"Fair sharing with cohorts",{"type":17,"tag":25,"props":2670,"children":2671},{},[2672,2673,2678],{"type":23,"value":332},{"type":17,"tag":53,"props":2674,"children":2676},{"className":2675},[],[2677],{"type":23,"value":2556},{"type":23,"value":2679}," field on ClusterQueues enables borrowing. When Team B isn't using their 4 GPUs, Team A can borrow them — and vice versa.",{"type":17,"tag":48,"props":2681,"children":2684},{"className":2682,"code":2683,"language":675,"meta":8},[677],"spec:\n cohort: gpu-cluster\n resourceGroups:\n - coveredResources: [\"cpu\", \"memory\", \"nvidia.com\u002Fgpu\"]\n flavors:\n - name: gpu-a100\n resources:\n - name: \"nvidia.com\u002Fgpu\"\n nominalQuota: 4\n borrowingLimit: 4\n",[2685],{"type":17,"tag":53,"props":2686,"children":2687},{"__ignoreMap":8},[2688],{"type":23,"value":2683},{"type":17,"tag":25,"props":2690,"children":2691},{},[2692,2693,2699],{"type":23,"value":332},{"type":17,"tag":53,"props":2694,"children":2696},{"className":2695},[],[2697],{"type":23,"value":2698},"borrowingLimit: 4",{"type":23,"value":2700}," means Team A can borrow up to 4 additional GPUs from the cohort (the full cluster's worth) when others aren't using them.",{"type":17,"tag":25,"props":2702,"children":2703},{},[2704],{"type":23,"value":2705},"The behavior:",{"type":17,"tag":134,"props":2707,"children":2708},{},[2709,2719,2729],{"type":17,"tag":138,"props":2710,"children":2711},{},[2712,2717],{"type":17,"tag":142,"props":2713,"children":2714},{},[2715],{"type":23,"value":2716},"Both teams idle:",{"type":23,"value":2718}," 8 GPUs available for whoever submits first.",{"type":17,"tag":138,"props":2720,"children":2721},{},[2722,2727],{"type":17,"tag":142,"props":2723,"children":2724},{},[2725],{"type":23,"value":2726},"Team A using 6 GPUs, Team B submits:",{"type":23,"value":2728}," Team A's jobs beyond their nominal quota (4) get preempted to make room for Team B's quota.",{"type":17,"tag":138,"props":2730,"children":2731},{},[2732,2737],{"type":17,"tag":142,"props":2733,"children":2734},{},[2735],{"type":23,"value":2736},"Team A using 4 GPUs, Team B using 2:",{"type":23,"value":2738}," Team A can borrow Team B's 2 unused GPUs, using 6 total.",{"type":17,"tag":25,"props":2740,"children":2741},{},[2742],{"type":23,"value":2743},"This is fair sharing. Resources don't sit idle just because their \"owner\" isn't using them, but guaranteed quotas are respected when demand appears.",{"type":17,"tag":36,"props":2745,"children":2747},{"id":2746},"priorities",[2748],{"type":23,"value":2749},"Priorities",{"type":17,"tag":25,"props":2751,"children":2752},{},[2753],{"type":23,"value":2754},"Not all jobs are equal. Production retraining should preempt experimental runs. Kueue supports this with WorkloadPriorityClasses:",{"type":17,"tag":48,"props":2756,"children":2759},{"className":2757,"code":2758,"language":675,"meta":8},[677],"apiVersion: kueue.x-k8s.io\u002Fv1beta1\nkind: WorkloadPriorityClass\nmetadata:\n name: production\nvalue: 1000\ndescription: \"Production retraining jobs\"\n---\napiVersion: kueue.x-k8s.io\u002Fv1beta1\nkind: WorkloadPriorityClass\nmetadata:\n name: experiment\nvalue: 100\ndescription: \"Experimental training runs\"\n",[2760],{"type":17,"tag":53,"props":2761,"children":2762},{"__ignoreMap":8},[2763],{"type":23,"value":2758},{"type":17,"tag":25,"props":2765,"children":2766},{},[2767],{"type":23,"value":2768},"Jobs reference the priority:",{"type":17,"tag":48,"props":2770,"children":2773},{"className":2771,"code":2772,"language":675,"meta":8},[677],"apiVersion: batch\u002Fv1\nkind: Job\nmetadata:\n name: weekly-retrain\n namespace: team-b\n labels:\n kueue.x-k8s.io\u002Fqueue-name: training-queue\n kueue.x-k8s.io\u002Fpriority-class: production\nspec:\n template:\n spec:\n containers:\n - name: train\n image: training:latest\n resources:\n requests:\n nvidia.com\u002Fgpu: 2\n restartPolicy: Never\n",[2774],{"type":17,"tag":53,"props":2775,"children":2776},{"__ignoreMap":8},[2777],{"type":23,"value":2772},{"type":17,"tag":25,"props":2779,"children":2780},{},[2781],{"type":23,"value":2782},"When a high-priority job arrives and there aren't enough resources, Kueue can preempt lower-priority jobs to make room. The preempted jobs go back into the queue and run when resources free up.",{"type":17,"tag":127,"props":2784,"children":2786},{"id":2785},"preemption-policies",[2787],{"type":23,"value":2788},"Preemption policies",{"type":17,"tag":25,"props":2790,"children":2791},{},[2792],{"type":23,"value":2793},"Kueue's preemption is configurable per ClusterQueue:",{"type":17,"tag":48,"props":2795,"children":2798},{"className":2796,"code":2797,"language":675,"meta":8},[677],"spec:\n preemption:\n reclaimWithinCohort: Any\n withinClusterQueue: LowerPriority\n",[2799],{"type":17,"tag":53,"props":2800,"children":2801},{"__ignoreMap":8},[2802],{"type":23,"value":2797},{"type":17,"tag":134,"props":2804,"children":2805},{},[2806,2820],{"type":17,"tag":138,"props":2807,"children":2808},{},[2809,2818],{"type":17,"tag":142,"props":2810,"children":2811},{},[2812],{"type":17,"tag":53,"props":2813,"children":2815},{"className":2814},[],[2816],{"type":23,"value":2817},"reclaimWithinCohort: Any",{"type":23,"value":2819}," — reclaim borrowed resources from other queues in the cohort.",{"type":17,"tag":138,"props":2821,"children":2822},{},[2823,2832],{"type":17,"tag":142,"props":2824,"children":2825},{},[2826],{"type":17,"tag":53,"props":2827,"children":2829},{"className":2828},[],[2830],{"type":23,"value":2831},"withinClusterQueue: LowerPriority",{"type":23,"value":2833}," — within the same queue, preempt lower-priority jobs.",{"type":17,"tag":25,"props":2835,"children":2836},{},[2837],{"type":23,"value":2838},"This gives you fine-grained control. You might want production jobs to preempt experiments, but not to preempt other production jobs.",{"type":17,"tag":36,"props":2840,"children":2842},{"id":2841},"putting-it-all-together",[2843],{"type":23,"value":2844},"Putting it all together",{"type":17,"tag":25,"props":2846,"children":2847},{},[2848],{"type":23,"value":2849},"Here's the full setup for our two-team scenario:",{"type":17,"tag":48,"props":2851,"children":2854},{"className":2852,"code":2853,"language":675,"meta":8},[677],"# The GPU flavor\napiVersion: kueue.x-k8s.io\u002Fv1beta1\nkind: ResourceFlavor\nmetadata:\n name: gpu-a100\nspec:\n nodeLabels:\n gpu-type: a100\n---\n# Team A: 4 GPU quota, can borrow 4 more\napiVersion: kueue.x-k8s.io\u002Fv1beta1\nkind: ClusterQueue\nmetadata:\n name: team-a-queue\nspec:\n cohort: gpu-cluster\n preemption:\n reclaimWithinCohort: Any\n withinClusterQueue: LowerPriority\n resourceGroups:\n - coveredResources: [\"cpu\", \"memory\", \"nvidia.com\u002Fgpu\"]\n flavors:\n - name: gpu-a100\n resources:\n - name: \"nvidia.com\u002Fgpu\"\n nominalQuota: 4\n borrowingLimit: 4\n - name: \"cpu\"\n nominalQuota: 32\n - name: \"memory\"\n nominalQuota: 128Gi\n---\n# Team B: 4 GPU quota, can borrow 4 more\napiVersion: kueue.x-k8s.io\u002Fv1beta1\nkind: ClusterQueue\nmetadata:\n name: team-b-queue\nspec:\n cohort: gpu-cluster\n preemption:\n reclaimWithinCohort: Any\n withinClusterQueue: LowerPriority\n resourceGroups:\n - coveredResources: [\"cpu\", \"memory\", \"nvidia.com\u002Fgpu\"]\n flavors:\n - name: gpu-a100\n resources:\n - name: \"nvidia.com\u002Fgpu\"\n nominalQuota: 4\n borrowingLimit: 4\n - name: \"cpu\"\n nominalQuota: 32\n - name: \"memory\"\n nominalQuota: 128Gi\n---\n# LocalQueues in each team's namespace\napiVersion: kueue.x-k8s.io\u002Fv1beta1\nkind: LocalQueue\nmetadata:\n name: training-queue\n namespace: team-a\nspec:\n clusterQueue: team-a-queue\n---\napiVersion: kueue.x-k8s.io\u002Fv1beta1\nkind: LocalQueue\nmetadata:\n name: training-queue\n namespace: team-b\nspec:\n clusterQueue: team-b-queue\n",[2855],{"type":17,"tag":53,"props":2856,"children":2857},{"__ignoreMap":8},[2858],{"type":23,"value":2853},{"type":17,"tag":25,"props":2860,"children":2861},{},[2862,2864,2870,2872,2877],{"type":23,"value":2863},"Apply with ",{"type":17,"tag":53,"props":2865,"children":2867},{"className":2866},[],[2868],{"type":23,"value":2869},"kubectl apply -f kueue-setup.yaml",{"type":23,"value":2871},", and both teams can submit jobs to their respective ",{"type":17,"tag":53,"props":2873,"children":2875},{"className":2874},[],[2876],{"type":23,"value":2597},{"type":23,"value":2878}," LocalQueues. Kueue handles the rest.",{"type":17,"tag":36,"props":2880,"children":2882},{"id":2881},"monitoring-kueue",[2883],{"type":23,"value":2884},"Monitoring Kueue",{"type":17,"tag":25,"props":2886,"children":2887},{},[2888],{"type":23,"value":2889},"Kueue exposes Prometheus metrics out of the box:",{"type":17,"tag":134,"props":2891,"children":2892},{},[2893,2904,2915],{"type":17,"tag":138,"props":2894,"children":2895},{},[2896,2902],{"type":17,"tag":53,"props":2897,"children":2899},{"className":2898},[],[2900],{"type":23,"value":2901},"kueue_pending_workloads",{"type":23,"value":2903}," — how many jobs are waiting in each queue",{"type":17,"tag":138,"props":2905,"children":2906},{},[2907,2913],{"type":17,"tag":53,"props":2908,"children":2910},{"className":2909},[],[2911],{"type":23,"value":2912},"kueue_admitted_active_workloads",{"type":23,"value":2914}," — how many jobs are currently running",{"type":17,"tag":138,"props":2916,"children":2917},{},[2918,2924],{"type":17,"tag":53,"props":2919,"children":2921},{"className":2920},[],[2922],{"type":23,"value":2923},"kueue_cluster_queue_resource_usage",{"type":23,"value":2925}," — current resource consumption per queue",{"type":17,"tag":25,"props":2927,"children":2928},{},[2929,2931,2936,2938,2943],{"type":23,"value":2930},"These tell you whether your quotas are right-sized. If ",{"type":17,"tag":53,"props":2932,"children":2934},{"className":2933},[],[2935],{"type":23,"value":2901},{"type":23,"value":2937}," is consistently high for one queue, that team needs more quota (or the cluster needs more GPUs). If ",{"type":17,"tag":53,"props":2939,"children":2941},{"className":2940},[],[2942],{"type":23,"value":2923},{"type":23,"value":2944}," is consistently low, resources are being wasted.",{"type":17,"tag":36,"props":2946,"children":2948},{"id":2947},"when-to-use-kueue-vs-simpler-approaches",[2949],{"type":23,"value":2950},"When to use Kueue vs. simpler approaches",{"type":17,"tag":25,"props":2952,"children":2953},{},[2954],{"type":17,"tag":142,"props":2955,"children":2956},{},[2957],{"type":23,"value":2958},"Use Kueue when:",{"type":17,"tag":134,"props":2960,"children":2961},{},[2962,2967,2972,2977],{"type":17,"tag":138,"props":2963,"children":2964},{},[2965],{"type":23,"value":2966},"Multiple teams or users share GPU resources.",{"type":17,"tag":138,"props":2968,"children":2969},{},[2970],{"type":23,"value":2971},"You need quotas to prevent one team from monopolizing the cluster.",{"type":17,"tag":138,"props":2973,"children":2974},{},[2975],{"type":23,"value":2976},"Production jobs need to preempt experimental work.",{"type":17,"tag":138,"props":2978,"children":2979},{},[2980],{"type":23,"value":2981},"You want fair sharing — idle resources should be usable by anyone, but guaranteed quotas should be respected.",{"type":17,"tag":25,"props":2983,"children":2984},{},[2985],{"type":17,"tag":142,"props":2986,"children":2987},{},[2988],{"type":23,"value":2989},"You might not need Kueue when:",{"type":17,"tag":134,"props":2991,"children":2992},{},[2993,2998,3003],{"type":17,"tag":138,"props":2994,"children":2995},{},[2996],{"type":23,"value":2997},"One team, one GPU. Just submit Jobs directly.",{"type":17,"tag":138,"props":2999,"children":3000},{},[3001],{"type":23,"value":3002},"All jobs have the same priority and there's no contention. First-come, first-served works fine.",{"type":17,"tag":138,"props":3004,"children":3005},{},[3006],{"type":23,"value":3007},"You're using a managed service (GKE Autopilot, AWS Batch) that handles scheduling for you.",{"type":17,"tag":25,"props":3009,"children":3010},{},[3011],{"type":23,"value":3012},"Kueue adds operational complexity. It's worth it when contention for GPUs is a real problem. If you're not fighting over resources, plain Kubernetes Jobs are simpler.",{"type":17,"tag":938,"props":3014,"children":3015},{},[],{"type":17,"tag":25,"props":3017,"children":3018},{},[3019,3021,3026,3028,3032],{"type":23,"value":3020},"GPU scheduling is one of the trickier parts of on-prem ML infrastructure. If you're setting up shared GPU clusters and need help with Kueue, quotas, or cluster architecture, ",{"type":17,"tag":496,"props":3022,"children":3024},{"href":948,"rel":3023},[500],[3025],{"type":23,"value":952},{"type":23,"value":3027}," does this work — ",{"type":17,"tag":496,"props":3029,"children":3030},{"href":957},[3031],{"type":23,"value":1629},{"type":23,"value":436},{"title":8,"searchDepth":963,"depth":963,"links":3034},[3035,3036,3041,3042,3043,3046,3047,3048],{"id":2383,"depth":963,"text":2386},{"id":2474,"depth":963,"text":2477,"children":3037},[3038,3039,3040],{"id":2480,"depth":968,"text":2457},{"id":2520,"depth":968,"text":2463},{"id":2561,"depth":968,"text":2470},{"id":2610,"depth":963,"text":2613},{"id":2665,"depth":963,"text":2668},{"id":2746,"depth":963,"text":2749,"children":3044},[3045],{"id":2785,"depth":968,"text":2788},{"id":2841,"depth":963,"text":2844},{"id":2881,"depth":963,"text":2884},{"id":2947,"depth":963,"text":2950},"content:articles:gpu-scheduling-kubernetes-kueue.md","articles\u002Fgpu-scheduling-kubernetes-kueue.md","articles\u002Fgpu-scheduling-kubernetes-kueue",{"_path":3053,"_dir":6,"_draft":7,"_partial":7,"_locale":8,"title":3054,"description":3055,"date":3056,"aiGenerated":12,"body":3057,"_type":985,"_id":3854,"_source":987,"_file":3855,"_stem":3856,"_extension":990},"\u002Farticles\u002Fkubernetes-cronjob-model-retraining","Your First Kubernetes CronJob for Model Retraining","A practical walkthrough of moving from manual model retraining on your laptop to an automated Kubernetes CronJob — covering container builds, secrets, and failure alerting.","2025-07-21",{"type":14,"children":3058,"toc":3841},[3059,3064,3069,3074,3102,3107,3113,3118,3147,3153,3158,3163,3172,3177,3210,3215,3221,3226,3235,3246,3254,3259,3268,3273,3282,3287,3293,3298,3307,3316,3321,3362,3368,3373,3378,3387,3392,3401,3420,3426,3431,3440,3445,3518,3523,3532,3537,3546,3551,3557,3562,3567,3580,3605,3610,3619,3624,3630,3635,3644,3649,3655,3660,3711,3716,3722,3740,3765,3783,3821,3824],{"type":17,"tag":18,"props":3060,"children":3062},{"id":3061},"your-first-kubernetes-cronjob-for-model-retraining",[3063],{"type":23,"value":3054},{"type":17,"tag":25,"props":3065,"children":3066},{},[3067],{"type":23,"value":3068},"You've got a model in production. It was trained once, deployed, and it's serving predictions. Eventually someone asks: \"shouldn't we retrain this on newer data?\" The answer is yes, and the question is how.",{"type":17,"tag":25,"props":3070,"children":3071},{},[3072],{"type":23,"value":3073},"The typical progression looks like this:",{"type":17,"tag":607,"props":3075,"children":3076},{},[3077,3082,3087,3092,3097],{"type":17,"tag":138,"props":3078,"children":3079},{},[3080],{"type":23,"value":3081},"Someone runs the training script manually on their laptop.",{"type":17,"tag":138,"props":3083,"children":3084},{},[3085],{"type":23,"value":3086},"This works until that person goes on vacation.",{"type":17,"tag":138,"props":3088,"children":3089},{},[3090],{"type":23,"value":3091},"The team sets up a shared VM where someone SSHes in and runs the script.",{"type":17,"tag":138,"props":3093,"children":3094},{},[3095],{"type":23,"value":3096},"This works until the VM gets restarted or reconfigured.",{"type":17,"tag":138,"props":3098,"children":3099},{},[3100],{"type":23,"value":3101},"The team automates it properly.",{"type":17,"tag":25,"props":3103,"children":3104},{},[3105],{"type":23,"value":3106},"This article is about step 5 — specifically, using Kubernetes CronJobs to automate retraining. No fancy orchestration frameworks, no complex DAGs. Just a cron schedule and a container.",{"type":17,"tag":36,"props":3108,"children":3110},{"id":3109},"prerequisites",[3111],{"type":23,"value":3112},"Prerequisites",{"type":17,"tag":25,"props":3114,"children":3115},{},[3116],{"type":23,"value":3117},"You'll need:",{"type":17,"tag":134,"props":3119,"children":3120},{},[3121,3126,3131,3142],{"type":17,"tag":138,"props":3122,"children":3123},{},[3124],{"type":23,"value":3125},"A Kubernetes cluster (even a small one — retraining doesn't need a large cluster unless your model does)",{"type":17,"tag":138,"props":3127,"children":3128},{},[3129],{"type":23,"value":3130},"A container registry (Docker Hub, GitHub Container Registry, or a private registry)",{"type":17,"tag":138,"props":3132,"children":3133},{},[3134,3140],{"type":17,"tag":53,"props":3135,"children":3137},{"className":3136},[],[3138],{"type":23,"value":3139},"kubectl",{"type":23,"value":3141}," configured to talk to your cluster",{"type":17,"tag":138,"props":3143,"children":3144},{},[3145],{"type":23,"value":3146},"A training script that works locally",{"type":17,"tag":36,"props":3148,"children":3150},{"id":3149},"step-1-make-the-training-script-self-contained",[3151],{"type":23,"value":3152},"Step 1: Make the training script self-contained",{"type":17,"tag":25,"props":3154,"children":3155},{},[3156],{"type":23,"value":3157},"Before containerizing anything, the training script needs to work without human intervention. That means no interactive prompts, no hardcoded local paths, and no assumptions about the machine it runs on.",{"type":17,"tag":25,"props":3159,"children":3160},{},[3161],{"type":23,"value":3162},"Here's a training script that pulls data, trains a model, and pushes the result to MLflow:",{"type":17,"tag":48,"props":3164,"children":3167},{"code":3165,"language":73,"meta":8,"className":3166},"#!\u002Fusr\u002Fbin\u002Fenv python3\n\"\"\"train.py — automated retraining script.\"\"\"\n\nimport os\nimport sys\nimport mlflow\nimport mlflow.sklearn\nfrom sklearn.datasets import load_iris\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\n\ndef main():\n tracking_uri = os.environ.get(\"MLFLOW_TRACKING_URI\")\n if not tracking_uri:\n print(\"ERROR: MLFLOW_TRACKING_URI not set\", file=sys.stderr)\n sys.exit(1)\n\n mlflow.set_tracking_uri(tracking_uri)\n mlflow.set_experiment(\"iris-classifier\")\n\n with mlflow.start_run():\n # In a real system, this would pull fresh data from a database or object store\n X, y = load_iris(return_X_y=True)\n X_train, X_test, y_train, y_test = train_test_split(\n X, y, test_size=0.2, random_state=42\n )\n\n n_estimators = 100\n max_depth = 5\n\n mlflow.log_param(\"n_estimators\", n_estimators)\n mlflow.log_param(\"max_depth\", max_depth)\n\n model = RandomForestClassifier(\n n_estimators=n_estimators, max_depth=max_depth\n )\n model.fit(X_train, y_train)\n\n predictions = model.predict(X_test)\n acc = accuracy_score(y_test, predictions)\n mlflow.log_metric(\"accuracy\", acc)\n print(f\"Accuracy: {acc:.4f}\")\n\n mlflow.sklearn.log_model(model, \"model\")\n print(\"Model logged to MLflow\")\n\nif __name__ == \"__main__\":\n main()\n",[75],[3168],{"type":17,"tag":53,"props":3169,"children":3170},{"__ignoreMap":8},[3171],{"type":23,"value":3165},{"type":17,"tag":25,"props":3173,"children":3174},{},[3175],{"type":23,"value":3176},"Key points:",{"type":17,"tag":134,"props":3178,"children":3179},{},[3180,3190,3200],{"type":17,"tag":138,"props":3181,"children":3182},{},[3183,3188],{"type":17,"tag":142,"props":3184,"children":3185},{},[3186],{"type":23,"value":3187},"Configuration comes from environment variables.",{"type":23,"value":3189}," The script doesn't know where MLflow lives — that's injected at runtime.",{"type":17,"tag":138,"props":3191,"children":3192},{},[3193,3198],{"type":17,"tag":142,"props":3194,"children":3195},{},[3196],{"type":23,"value":3197},"Errors produce nonzero exit codes.",{"type":23,"value":3199}," Kubernetes uses exit codes to determine if a Job succeeded or failed.",{"type":17,"tag":138,"props":3201,"children":3202},{},[3203,3208],{"type":17,"tag":142,"props":3204,"children":3205},{},[3206],{"type":23,"value":3207},"No interactive input.",{"type":23,"value":3209}," Everything is parameterized or has defaults.",{"type":17,"tag":25,"props":3211,"children":3212},{},[3213],{"type":23,"value":3214},"Test this locally before containerizing it. If it doesn't work on your machine with just environment variables set, it won't work in a container.",{"type":17,"tag":36,"props":3216,"children":3218},{"id":3217},"step-2-build-the-container",[3219],{"type":23,"value":3220},"Step 2: Build the container",{"type":17,"tag":25,"props":3222,"children":3223},{},[3224],{"type":23,"value":3225},"The Dockerfile is straightforward:",{"type":17,"tag":48,"props":3227,"children":3230},{"code":3228,"language":89,"meta":8,"className":3229},"FROM python:3.11-slim\n\nWORKDIR \u002Fapp\n\nCOPY requirements.txt .\nRUN pip install --no-cache-dir -r requirements.txt\n\nCOPY train.py .\n\nCMD [\"python\", \"train.py\"]\n",[91],[3231],{"type":17,"tag":53,"props":3232,"children":3233},{"__ignoreMap":8},[3234],{"type":23,"value":3228},{"type":17,"tag":25,"props":3236,"children":3237},{},[3238,3240,3245],{"type":23,"value":3239},"And the ",{"type":17,"tag":53,"props":3241,"children":3243},{"className":3242},[],[3244],{"type":23,"value":265},{"type":23,"value":1890},{"type":17,"tag":48,"props":3247,"children":3249},{"code":3248},"scikit-learn==1.3.2\nmlflow==2.9.2\n",[3250],{"type":17,"tag":53,"props":3251,"children":3252},{"__ignoreMap":8},[3253],{"type":23,"value":3248},{"type":17,"tag":25,"props":3255,"children":3256},{},[3257],{"type":23,"value":3258},"Build and push:",{"type":17,"tag":48,"props":3260,"children":3263},{"code":3261,"language":105,"meta":8,"className":3262},"docker build -t registry.example.com\u002Fml-training\u002Firis:v1 .\ndocker push registry.example.com\u002Fml-training\u002Firis:v1\n",[107],[3264],{"type":17,"tag":53,"props":3265,"children":3266},{"__ignoreMap":8},[3267],{"type":23,"value":3261},{"type":17,"tag":25,"props":3269,"children":3270},{},[3271],{"type":23,"value":3272},"Test the container locally before deploying to Kubernetes:",{"type":17,"tag":48,"props":3274,"children":3277},{"code":3275,"language":105,"meta":8,"className":3276},"docker run --rm \\\n -e MLFLOW_TRACKING_URI=http:\u002F\u002Fhost.docker.internal:5000 \\\n registry.example.com\u002Fml-training\u002Firis:v1\n",[107],[3278],{"type":17,"tag":53,"props":3279,"children":3280},{"__ignoreMap":8},[3281],{"type":23,"value":3275},{"type":17,"tag":25,"props":3283,"children":3284},{},[3285],{"type":23,"value":3286},"If it runs and logs to MLflow, you're ready for Kubernetes.",{"type":17,"tag":36,"props":3288,"children":3290},{"id":3289},"step-3-create-a-kubernetes-job",[3291],{"type":23,"value":3292},"Step 3: Create a Kubernetes Job",{"type":17,"tag":25,"props":3294,"children":3295},{},[3296],{"type":23,"value":3297},"Before setting up the cron schedule, run the training as a one-off Job to verify it works in the cluster:",{"type":17,"tag":48,"props":3299,"children":3302},{"code":3300,"language":675,"meta":8,"className":3301},"apiVersion: batch\u002Fv1\nkind: Job\nmetadata:\n name: iris-retrain-test\n namespace: ml-jobs\nspec:\n backoffLimit: 2\n template:\n spec:\n containers:\n - name: train\n image: registry.example.com\u002Fml-training\u002Firis:v1\n env:\n - name: MLFLOW_TRACKING_URI\n value: \"http:\u002F\u002Fmlflow.mlflow.svc.cluster.local:5000\"\n resources:\n requests:\n memory: \"512Mi\"\n cpu: \"500m\"\n limits:\n memory: \"1Gi\"\n cpu: \"1\"\n restartPolicy: Never\n",[677],[3303],{"type":17,"tag":53,"props":3304,"children":3305},{"__ignoreMap":8},[3306],{"type":23,"value":3300},{"type":17,"tag":48,"props":3308,"children":3311},{"code":3309,"language":105,"meta":8,"className":3310},"kubectl apply -f job.yaml\nkubectl -n ml-jobs logs -f job\u002Firis-retrain-test\n",[107],[3312],{"type":17,"tag":53,"props":3313,"children":3314},{"__ignoreMap":8},[3315],{"type":23,"value":3309},{"type":17,"tag":25,"props":3317,"children":3318},{},[3319],{"type":23,"value":3320},"A few things to note:",{"type":17,"tag":134,"props":3322,"children":3323},{},[3324,3338,3352],{"type":17,"tag":138,"props":3325,"children":3326},{},[3327,3336],{"type":17,"tag":142,"props":3328,"children":3329},{},[3330],{"type":17,"tag":53,"props":3331,"children":3333},{"className":3332},[],[3334],{"type":23,"value":3335},"backoffLimit: 2",{"type":23,"value":3337}," means Kubernetes will retry twice on failure, then mark the Job as failed. Without this, it retries indefinitely.",{"type":17,"tag":138,"props":3339,"children":3340},{},[3341,3350],{"type":17,"tag":142,"props":3342,"children":3343},{},[3344],{"type":17,"tag":53,"props":3345,"children":3347},{"className":3346},[],[3348],{"type":23,"value":3349},"restartPolicy: Never",{"type":23,"value":3351}," is required for Jobs. Kubernetes manages retries at the Job level, not the Pod level.",{"type":17,"tag":138,"props":3353,"children":3354},{},[3355,3360],{"type":17,"tag":142,"props":3356,"children":3357},{},[3358],{"type":23,"value":3359},"Resource requests and limits",{"type":23,"value":3361}," prevent the training job from consuming the entire node. Set these based on your actual training requirements.",{"type":17,"tag":36,"props":3363,"children":3365},{"id":3364},"step-4-handle-secrets",[3366],{"type":23,"value":3367},"Step 4: Handle secrets",{"type":17,"tag":25,"props":3369,"children":3370},{},[3371],{"type":23,"value":3372},"Your training script probably needs credentials — for the model registry, the data source, or both. Don't put these in the YAML file. Use Kubernetes Secrets.",{"type":17,"tag":25,"props":3374,"children":3375},{},[3376],{"type":23,"value":3377},"Create the secret:",{"type":17,"tag":48,"props":3379,"children":3382},{"code":3380,"language":105,"meta":8,"className":3381},"kubectl -n ml-jobs create secret generic mlflow-credentials \\\n --from-literal=MLFLOW_TRACKING_URI=http:\u002F\u002Fmlflow.mlflow.svc.cluster.local:5000 \\\n --from-literal=AWS_ACCESS_KEY_ID=your-key \\\n --from-literal=AWS_SECRET_ACCESS_KEY=your-secret\n",[107],[3383],{"type":17,"tag":53,"props":3384,"children":3385},{"__ignoreMap":8},[3386],{"type":23,"value":3380},{"type":17,"tag":25,"props":3388,"children":3389},{},[3390],{"type":23,"value":3391},"Reference it in the Job:",{"type":17,"tag":48,"props":3393,"children":3396},{"code":3394,"language":675,"meta":8,"className":3395},"containers:\n- name: train\n image: registry.example.com\u002Fml-training\u002Firis:v1\n envFrom:\n - secretRef:\n name: mlflow-credentials\n resources:\n requests:\n memory: \"512Mi\"\n cpu: \"500m\"\n limits:\n memory: \"1Gi\"\n cpu: \"1\"\n",[677],[3397],{"type":17,"tag":53,"props":3398,"children":3399},{"__ignoreMap":8},[3400],{"type":23,"value":3394},{"type":17,"tag":25,"props":3402,"children":3403},{},[3404,3410,3412,3418],{"type":17,"tag":53,"props":3405,"children":3407},{"className":3406},[],[3408],{"type":23,"value":3409},"envFrom",{"type":23,"value":3411}," injects every key in the Secret as an environment variable. The training script reads them via ",{"type":17,"tag":53,"props":3413,"children":3415},{"className":3414},[],[3416],{"type":23,"value":3417},"os.environ",{"type":23,"value":3419}," without knowing they came from a Secret.",{"type":17,"tag":36,"props":3421,"children":3423},{"id":3422},"step-5-set-up-the-cronjob",[3424],{"type":23,"value":3425},"Step 5: Set up the CronJob",{"type":17,"tag":25,"props":3427,"children":3428},{},[3429],{"type":23,"value":3430},"Once the one-off Job works, wrap it in a CronJob:",{"type":17,"tag":48,"props":3432,"children":3435},{"code":3433,"language":675,"meta":8,"className":3434},"apiVersion: batch\u002Fv1\nkind: CronJob\nmetadata:\n name: iris-retrain\n namespace: ml-jobs\nspec:\n schedule: \"0 3 * * 0\"\n concurrencyPolicy: Forbid\n successfulJobsHistoryLimit: 3\n failedJobsHistoryLimit: 3\n jobTemplate:\n spec:\n backoffLimit: 2\n activeDeadlineSeconds: 3600\n template:\n spec:\n containers:\n - name: train\n image: registry.example.com\u002Fml-training\u002Firis:v1\n envFrom:\n - secretRef:\n name: mlflow-credentials\n resources:\n requests:\n memory: \"512Mi\"\n cpu: \"500m\"\n limits:\n memory: \"1Gi\"\n cpu: \"1\"\n restartPolicy: Never\n",[677],[3436],{"type":17,"tag":53,"props":3437,"children":3438},{"__ignoreMap":8},[3439],{"type":23,"value":3433},{"type":17,"tag":25,"props":3441,"children":3442},{},[3443],{"type":23,"value":3444},"Breaking down the CronJob-specific fields:",{"type":17,"tag":134,"props":3446,"children":3447},{},[3448,3462,3476,3490,3504],{"type":17,"tag":138,"props":3449,"children":3450},{},[3451,3460],{"type":17,"tag":142,"props":3452,"children":3453},{},[3454],{"type":17,"tag":53,"props":3455,"children":3457},{"className":3456},[],[3458],{"type":23,"value":3459},"schedule: \"0 3 * * 0\"",{"type":23,"value":3461}," — runs every Sunday at 3 AM. Standard cron syntax. Pick a time when the cluster is underutilized.",{"type":17,"tag":138,"props":3463,"children":3464},{},[3465,3474],{"type":17,"tag":142,"props":3466,"children":3467},{},[3468],{"type":17,"tag":53,"props":3469,"children":3471},{"className":3470},[],[3472],{"type":23,"value":3473},"concurrencyPolicy: Forbid",{"type":23,"value":3475}," — if the previous run is still going when the next one is scheduled, skip it. This prevents pile-ups when training takes longer than expected.",{"type":17,"tag":138,"props":3477,"children":3478},{},[3479,3488],{"type":17,"tag":142,"props":3480,"children":3481},{},[3482],{"type":17,"tag":53,"props":3483,"children":3485},{"className":3484},[],[3486],{"type":23,"value":3487},"successfulJobsHistoryLimit: 3",{"type":23,"value":3489}," — keep the last 3 completed Job objects around for debugging. Older ones are garbage collected.",{"type":17,"tag":138,"props":3491,"children":3492},{},[3493,3502],{"type":17,"tag":142,"props":3494,"children":3495},{},[3496],{"type":17,"tag":53,"props":3497,"children":3499},{"className":3498},[],[3500],{"type":23,"value":3501},"failedJobsHistoryLimit: 3",{"type":23,"value":3503}," — same for failed Jobs.",{"type":17,"tag":138,"props":3505,"children":3506},{},[3507,3516],{"type":17,"tag":142,"props":3508,"children":3509},{},[3510],{"type":17,"tag":53,"props":3511,"children":3513},{"className":3512},[],[3514],{"type":23,"value":3515},"activeDeadlineSeconds: 3600",{"type":23,"value":3517}," — kill the Job if it runs longer than an hour. This is your safety net against hanging training runs.",{"type":17,"tag":25,"props":3519,"children":3520},{},[3521],{"type":23,"value":3522},"Apply it:",{"type":17,"tag":48,"props":3524,"children":3527},{"code":3525,"language":105,"meta":8,"className":3526},"kubectl apply -f cronjob.yaml\n",[107],[3528],{"type":17,"tag":53,"props":3529,"children":3530},{"__ignoreMap":8},[3531],{"type":23,"value":3525},{"type":17,"tag":25,"props":3533,"children":3534},{},[3535],{"type":23,"value":3536},"Verify it's scheduled:",{"type":17,"tag":48,"props":3538,"children":3541},{"code":3539,"language":105,"meta":8,"className":3540},"kubectl -n ml-jobs get cronjob iris-retrain\n",[107],[3542],{"type":17,"tag":53,"props":3543,"children":3544},{"__ignoreMap":8},[3545],{"type":23,"value":3539},{"type":17,"tag":25,"props":3547,"children":3548},{},[3549],{"type":23,"value":3550},"You should see the schedule and the time of the next run.",{"type":17,"tag":36,"props":3552,"children":3554},{"id":3553},"step-6-know-when-it-fails",[3555],{"type":23,"value":3556},"Step 6: Know when it fails",{"type":17,"tag":25,"props":3558,"children":3559},{},[3560],{"type":23,"value":3561},"A CronJob that fails silently is worse than no CronJob at all — you think retraining is happening when it isn't. You need alerting.",{"type":17,"tag":25,"props":3563,"children":3564},{},[3565],{"type":23,"value":3566},"The simplest approach is a Kubernetes Event watcher. But for most teams, the practical answer is to check Job status from your existing monitoring stack.",{"type":17,"tag":25,"props":3568,"children":3569},{},[3570,3572,3578],{"type":23,"value":3571},"If you're running Prometheus (and on Kubernetes, you probably are), the ",{"type":17,"tag":53,"props":3573,"children":3575},{"className":3574},[],[3576],{"type":23,"value":3577},"kube-state-metrics",{"type":23,"value":3579}," exporter already exposes Job metrics:",{"type":17,"tag":134,"props":3581,"children":3582},{},[3583,3594],{"type":17,"tag":138,"props":3584,"children":3585},{},[3586,3592],{"type":17,"tag":53,"props":3587,"children":3589},{"className":3588},[],[3590],{"type":23,"value":3591},"kube_job_status_failed",{"type":23,"value":3593}," — number of failed Jobs",{"type":17,"tag":138,"props":3595,"children":3596},{},[3597,3603],{"type":17,"tag":53,"props":3598,"children":3600},{"className":3599},[],[3601],{"type":23,"value":3602},"kube_job_status_succeeded",{"type":23,"value":3604}," — number of succeeded Jobs",{"type":17,"tag":25,"props":3606,"children":3607},{},[3608],{"type":23,"value":3609},"A basic Prometheus alert rule:",{"type":17,"tag":48,"props":3611,"children":3614},{"code":3612,"language":675,"meta":8,"className":3613},"groups:\n- name: ml-retraining\n rules:\n - alert: RetrainingJobFailed\n expr: |\n kube_job_status_failed{namespace=\"ml-jobs\", job_name=~\"iris-retrain.*\"} > 0\n for: 5m\n labels:\n severity: warning\n annotations:\n summary: \"ML retraining job failed\"\n description: \"The iris-retrain CronJob has a failed run. Check logs with: kubectl -n ml-jobs logs job\u002F{{ $labels.job_name }}\"\n",[677],[3615],{"type":17,"tag":53,"props":3616,"children":3617},{"__ignoreMap":8},[3618],{"type":23,"value":3612},{"type":17,"tag":25,"props":3620,"children":3621},{},[3622],{"type":23,"value":3623},"This fires an alert if any retraining Job has been in a failed state for more than 5 minutes. Route it to Slack, PagerDuty, or email — wherever your team already gets alerts.",{"type":17,"tag":127,"props":3625,"children":3627},{"id":3626},"quick-check-script",[3628],{"type":23,"value":3629},"Quick check script",{"type":17,"tag":25,"props":3631,"children":3632},{},[3633],{"type":23,"value":3634},"For teams without Prometheus, a simple script that runs on its own cron schedule works:",{"type":17,"tag":48,"props":3636,"children":3639},{"code":3637,"language":105,"meta":8,"className":3638},"#!\u002Fbin\u002Fbash\nFAILED=$(kubectl -n ml-jobs get jobs \\\n -l app=iris-retrain \\\n --field-selector=status.successful=0 \\\n -o name 2>\u002Fdev\u002Fnull | wc -l)\n\nif [ \"$FAILED\" -gt 0 ]; then\n echo \"WARNING: $FAILED failed retraining jobs\" | \\\n mail -s \"Retraining failure\" team@example.com\nfi\n",[107],[3640],{"type":17,"tag":53,"props":3641,"children":3642},{"__ignoreMap":8},[3643],{"type":23,"value":3637},{"type":17,"tag":25,"props":3645,"children":3646},{},[3647],{"type":23,"value":3648},"Not elegant, but it works and it's better than silence.",{"type":17,"tag":36,"props":3650,"children":3652},{"id":3651},"what-this-looks-like-day-to-day",[3653],{"type":23,"value":3654},"What this looks like day-to-day",{"type":17,"tag":25,"props":3656,"children":3657},{},[3658],{"type":23,"value":3659},"Once the CronJob is running, the day-to-day workflow changes:",{"type":17,"tag":134,"props":3661,"children":3662},{},[3663,3673,3691,3701],{"type":17,"tag":138,"props":3664,"children":3665},{},[3666,3671],{"type":17,"tag":142,"props":3667,"children":3668},{},[3669],{"type":23,"value":3670},"No more manual retraining.",{"type":23,"value":3672}," The CronJob runs on schedule. New models appear in MLflow.",{"type":17,"tag":138,"props":3674,"children":3675},{},[3676,3681,3683,3689],{"type":17,"tag":142,"props":3677,"children":3678},{},[3679],{"type":23,"value":3680},"Debugging is straightforward.",{"type":23,"value":3682}," If a run fails, check the Job logs: ",{"type":17,"tag":53,"props":3684,"children":3686},{"className":3685},[],[3687],{"type":23,"value":3688},"kubectl -n ml-jobs logs job\u002F\u003Cjob-name>",{"type":23,"value":3690},". The logs are the same output you'd see running locally.",{"type":17,"tag":138,"props":3692,"children":3693},{},[3694,3699],{"type":17,"tag":142,"props":3695,"children":3696},{},[3697],{"type":23,"value":3698},"Updating the training code",{"type":23,"value":3700}," means building a new container image and updating the CronJob's image reference. The schedule and infrastructure stay the same.",{"type":17,"tag":138,"props":3702,"children":3703},{},[3704,3709],{"type":17,"tag":142,"props":3705,"children":3706},{},[3707],{"type":23,"value":3708},"Adjusting the schedule",{"type":23,"value":3710}," is a one-line YAML change.",{"type":17,"tag":25,"props":3712,"children":3713},{},[3714],{"type":23,"value":3715},"The main thing you'll want to add next is automated evaluation — comparing the new model against the current production model before promoting it. But that's a separate concern. The CronJob just ensures training happens reliably.",{"type":17,"tag":36,"props":3717,"children":3719},{"id":3718},"common-gotchas",[3720],{"type":23,"value":3721},"Common gotchas",{"type":17,"tag":25,"props":3723,"children":3724},{},[3725,3730,3732,3738],{"type":17,"tag":142,"props":3726,"children":3727},{},[3728],{"type":23,"value":3729},"Image pull failures.",{"type":23,"value":3731}," If your container registry requires authentication, set up an ",{"type":17,"tag":53,"props":3733,"children":3735},{"className":3734},[],[3736],{"type":23,"value":3737},"imagePullSecret",{"type":23,"value":3739}," on the namespace. Silent image pull failures are a common source of confusion.",{"type":17,"tag":25,"props":3741,"children":3742},{},[3743,3748,3750,3755,3757,3763],{"type":17,"tag":142,"props":3744,"children":3745},{},[3746],{"type":23,"value":3747},"Timezone.",{"type":23,"value":3749}," Kubernetes CronJobs use UTC by default. If you set ",{"type":17,"tag":53,"props":3751,"children":3753},{"className":3752},[],[3754],{"type":23,"value":3459},{"type":23,"value":3756}," expecting 3 AM local time, it might run at a different hour. Kubernetes 1.27+ supports ",{"type":17,"tag":53,"props":3758,"children":3760},{"className":3759},[],[3761],{"type":23,"value":3762},"timeZone",{"type":23,"value":3764}," on CronJobs if this matters.",{"type":17,"tag":25,"props":3766,"children":3767},{},[3768,3773,3775,3781],{"type":17,"tag":142,"props":3769,"children":3770},{},[3771],{"type":23,"value":3772},"Resource starvation.",{"type":23,"value":3774}," If the cluster doesn't have enough resources to schedule the training Pod, it'll sit in Pending state until the ",{"type":17,"tag":53,"props":3776,"children":3778},{"className":3777},[],[3779],{"type":23,"value":3780},"activeDeadlineSeconds",{"type":23,"value":3782}," kills it. Set resource requests realistically and monitor Pod scheduling.",{"type":17,"tag":25,"props":3784,"children":3785},{},[3786,3791,3793,3798,3800,3805,3806,3811,3813,3819],{"type":17,"tag":142,"props":3787,"children":3788},{},[3789],{"type":23,"value":3790},"Stale images.",{"type":23,"value":3792}," Using ",{"type":17,"tag":53,"props":3794,"children":3796},{"className":3795},[],[3797],{"type":23,"value":794},{"type":23,"value":3799}," tags means you're never sure which version of the training code is running. Use explicit version tags (",{"type":17,"tag":53,"props":3801,"children":3803},{"className":3802},[],[3804],{"type":23,"value":820},{"type":23,"value":397},{"type":17,"tag":53,"props":3807,"children":3809},{"className":3808},[],[3810],{"type":23,"value":827},{"type":23,"value":3812},") or content-addressable tags (",{"type":17,"tag":53,"props":3814,"children":3816},{"className":3815},[],[3817],{"type":23,"value":3818},":sha-abc123",{"type":23,"value":3820},").",{"type":17,"tag":938,"props":3822,"children":3823},{},[],{"type":17,"tag":25,"props":3825,"children":3826},{},[3827,3829,3834,3836,3840],{"type":23,"value":3828},"Kubernetes CronJobs are the foundation of automated ML pipelines. If you need help setting up reliable retraining infrastructure, ",{"type":17,"tag":496,"props":3830,"children":3832},{"href":948,"rel":3831},[500],[3833],{"type":23,"value":952},{"type":23,"value":3835}," can help — ",{"type":17,"tag":496,"props":3837,"children":3838},{"href":957},[3839],{"type":23,"value":960},{"type":23,"value":436},{"title":8,"searchDepth":963,"depth":963,"links":3842},[3843,3844,3845,3846,3847,3848,3849,3852,3853],{"id":3109,"depth":963,"text":3112},{"id":3149,"depth":963,"text":3152},{"id":3217,"depth":963,"text":3220},{"id":3289,"depth":963,"text":3292},{"id":3364,"depth":963,"text":3367},{"id":3422,"depth":963,"text":3425},{"id":3553,"depth":963,"text":3556,"children":3850},[3851],{"id":3626,"depth":968,"text":3629},{"id":3651,"depth":963,"text":3654},{"id":3718,"depth":963,"text":3721},"content:articles:kubernetes-cronjob-model-retraining.md","articles\u002Fkubernetes-cronjob-model-retraining.md","articles\u002Fkubernetes-cronjob-model-retraining",{"_path":3858,"_dir":6,"_draft":7,"_partial":7,"_locale":8,"title":3859,"description":3860,"date":3861,"aiGenerated":12,"body":3862,"_type":985,"_id":4456,"_source":987,"_file":4457,"_stem":4458,"_extension":990},"\u002Farticles\u002Fwhat-mlflow-actually-does","What MLflow Actually Does (and What It Doesn't)","MLflow shows up in every MLOps discussion, but what does it actually handle? A clear-eyed look at experiment tracking, the model registry, and artifact storage — plus what you'll still need to solve yourself.","2025-07-25",{"type":14,"children":3863,"toc":4434},[3864,3869,3874,3879,3885,3890,3899,3920,3925,3931,3936,3976,3985,3990,3996,4001,4010,4015,4020,4026,4031,4036,4087,4096,4101,4144,4150,4155,4173,4178,4184,4190,4203,4212,4224,4229,4235,4248,4257,4262,4268,4273,4279,4284,4289,4295,4300,4306,4311,4316,4321,4326,4332,4337,4342,4348,4353,4371,4376,4389,4395,4400,4408,4413,4416],{"type":17,"tag":18,"props":3865,"children":3867},{"id":3866},"what-mlflow-actually-does-and-what-it-doesnt",[3868],{"type":23,"value":3859},{"type":17,"tag":25,"props":3870,"children":3871},{},[3872],{"type":23,"value":3873},"MLflow comes up in every MLOps conversation. If you search for \"how to manage ML experiments,\" it's the first tool you'll find. But the messaging is broad enough that it's hard to know what MLflow actually handles versus what you'll still need to build around it.",{"type":17,"tag":25,"props":3875,"children":3876},{},[3877],{"type":23,"value":3878},"This article breaks it down concretely. What MLflow does well, what it does passably, and what it doesn't do at all.",{"type":17,"tag":36,"props":3880,"children":3882},{"id":3881},"the-core-experiment-tracking",[3883],{"type":23,"value":3884},"The core: experiment tracking",{"type":17,"tag":25,"props":3886,"children":3887},{},[3888],{"type":23,"value":3889},"At its heart, MLflow is a logging system for ML experiments. You call a few functions during training, and MLflow records what happened.",{"type":17,"tag":48,"props":3891,"children":3894},{"code":3892,"language":73,"meta":8,"className":3893},"import mlflow\n\nmlflow.set_experiment(\"fraud-detection\")\n\nwith mlflow.start_run():\n mlflow.log_param(\"learning_rate\", 0.01)\n mlflow.log_param(\"epochs\", 50)\n mlflow.log_param(\"batch_size\", 128)\n\n # ... training happens ...\n\n mlflow.log_metric(\"auc\", 0.934)\n mlflow.log_metric(\"precision\", 0.891)\n mlflow.log_metric(\"recall\", 0.867)\n",[75],[3895],{"type":17,"tag":53,"props":3896,"children":3897},{"__ignoreMap":8},[3898],{"type":23,"value":3892},{"type":17,"tag":25,"props":3900,"children":3901},{},[3902,3904,3910,3912,3918],{"type":23,"value":3903},"That's really it. You call ",{"type":17,"tag":53,"props":3905,"children":3907},{"className":3906},[],[3908],{"type":23,"value":3909},"log_param",{"type":23,"value":3911}," for inputs and ",{"type":17,"tag":53,"props":3913,"children":3915},{"className":3914},[],[3916],{"type":23,"value":3917},"log_metric",{"type":23,"value":3919}," for outputs. MLflow stores them in a database and gives you a UI to browse and compare runs.",{"type":17,"tag":25,"props":3921,"children":3922},{},[3923],{"type":23,"value":3924},"This solves a very real problem. Without it, teams end up with spreadsheets, Slack messages (\"hey, what learning rate did you use for that good run last Tuesday?\"), or notebooks with output cells that may or may not match the code above them.",{"type":17,"tag":127,"props":3926,"children":3928},{"id":3927},"what-you-get-from-the-tracking-server",[3929],{"type":23,"value":3930},"What you get from the tracking server",{"type":17,"tag":25,"props":3932,"children":3933},{},[3934],{"type":23,"value":3935},"The MLflow UI lets you:",{"type":17,"tag":134,"props":3937,"children":3938},{},[3939,3949,3959],{"type":17,"tag":138,"props":3940,"children":3941},{},[3942,3947],{"type":17,"tag":142,"props":3943,"children":3944},{},[3945],{"type":23,"value":3946},"Compare runs side by side.",{"type":23,"value":3948}," Sort by any metric, filter by parameters, see which hyperparameter combinations actually moved the needle.",{"type":17,"tag":138,"props":3950,"children":3951},{},[3952,3957],{"type":17,"tag":142,"props":3953,"children":3954},{},[3955],{"type":23,"value":3956},"View run history.",{"type":23,"value":3958}," See every experiment chronologically. Useful when you need to answer \"what changed between the model that worked and the one that didn't?\"",{"type":17,"tag":138,"props":3960,"children":3961},{},[3962,3967,3968,3974],{"type":17,"tag":142,"props":3963,"children":3964},{},[3965],{"type":23,"value":3966},"Search runs programmatically.",{"type":23,"value":814},{"type":17,"tag":53,"props":3969,"children":3971},{"className":3970},[],[3972],{"type":23,"value":3973},"mlflow.search_runs()",{"type":23,"value":3975}," returns a DataFrame, so you can do your own analysis on experiment history.",{"type":17,"tag":48,"props":3977,"children":3980},{"code":3978,"language":73,"meta":8,"className":3979},"import mlflow\n\n# Find all runs with AUC > 0.9\nruns = mlflow.search_runs(\n experiment_names=[\"fraud-detection\"],\n filter_string=\"metrics.auc > 0.9\",\n order_by=[\"metrics.auc DESC\"],\n)\nprint(runs[[\"params.learning_rate\", \"metrics.auc\"]])\n",[75],[3981],{"type":17,"tag":53,"props":3982,"children":3983},{"__ignoreMap":8},[3984],{"type":23,"value":3978},{"type":17,"tag":25,"props":3986,"children":3987},{},[3988],{"type":23,"value":3989},"This is genuinely useful. The difference between \"I think the model with learning rate 0.01 was better\" and \"here are the metrics for every run, sorted\" is significant.",{"type":17,"tag":36,"props":3991,"children":3993},{"id":3992},"artifact-storage",[3994],{"type":23,"value":3995},"Artifact storage",{"type":17,"tag":25,"props":3997,"children":3998},{},[3999],{"type":23,"value":4000},"Beyond parameters and metrics, MLflow stores files — called artifacts. The most common artifact is the trained model itself, but you can log anything: plots, data samples, configuration files.",{"type":17,"tag":48,"props":4002,"children":4005},{"code":4003,"language":73,"meta":8,"className":4004},"# Log the model\nmlflow.sklearn.log_model(model, \"model\")\n\n# Log a confusion matrix plot\nmlflow.log_artifact(\"confusion_matrix.png\")\n\n# Log a data sample\nmlflow.log_artifact(\"sample_data.csv\")\n",[75],[4006],{"type":17,"tag":53,"props":4007,"children":4008},{"__ignoreMap":8},[4009],{"type":23,"value":4003},{"type":17,"tag":25,"props":4011,"children":4012},{},[4013],{"type":23,"value":4014},"MLflow supports multiple artifact backends: local filesystem, S3, GCS, Azure Blob Storage, or HDFS. You configure this once on the tracking server and every client stores artifacts in the same place.",{"type":17,"tag":25,"props":4016,"children":4017},{},[4018],{"type":23,"value":4019},"This matters because the alternative is ad-hoc. Models end up on local drives, in random S3 buckets, or attached to Slack messages. MLflow gives you a single, organized location with each artifact tied to the run that produced it.",{"type":17,"tag":36,"props":4021,"children":4023},{"id":4022},"the-model-registry",[4024],{"type":23,"value":4025},"The model registry",{"type":17,"tag":25,"props":4027,"children":4028},{},[4029],{"type":23,"value":4030},"The model registry is where MLflow goes from \"experiment tracking tool\" to \"part of your deployment pipeline.\" It's a central catalog of models with versioning and stage transitions.",{"type":17,"tag":25,"props":4032,"children":4033},{},[4034],{"type":23,"value":4035},"The workflow:",{"type":17,"tag":607,"props":4037,"children":4038},{},[4039,4044,4049,4054],{"type":17,"tag":138,"props":4040,"children":4041},{},[4042],{"type":23,"value":4043},"Train a model and log it to a run.",{"type":17,"tag":138,"props":4045,"children":4046},{},[4047],{"type":23,"value":4048},"Register it in the model registry under a name (e.g., \"fraud-detector\").",{"type":17,"tag":138,"props":4050,"children":4051},{},[4052],{"type":23,"value":4053},"Each registration creates a new version (v1, v2, v3...).",{"type":17,"tag":138,"props":4055,"children":4056},{},[4057,4059,4064,4066,4072,4073,4079,4080,4086],{"type":23,"value":4058},"Transition versions between stages: ",{"type":17,"tag":53,"props":4060,"children":4062},{"className":4061},[],[4063],{"type":23,"value":1990},{"type":23,"value":4065}," → ",{"type":17,"tag":53,"props":4067,"children":4069},{"className":4068},[],[4070],{"type":23,"value":4071},"Staging",{"type":23,"value":4065},{"type":17,"tag":53,"props":4074,"children":4076},{"className":4075},[],[4077],{"type":23,"value":4078},"Production",{"type":23,"value":4065},{"type":17,"tag":53,"props":4081,"children":4083},{"className":4082},[],[4084],{"type":23,"value":4085},"Archived",{"type":23,"value":436},{"type":17,"tag":48,"props":4088,"children":4091},{"code":4089,"language":73,"meta":8,"className":4090},"import mlflow\n\n# Register a model from a run\nresult = mlflow.register_model(\n model_uri=\"runs:\u002Fabc123def\u002Fmodel\",\n name=\"fraud-detector\"\n)\n\n# Transition to staging\nclient = mlflow.tracking.MlflowClient()\nclient.transition_model_version_stage(\n name=\"fraud-detector\",\n version=result.version,\n stage=\"Staging\"\n)\n",[75],[4092],{"type":17,"tag":53,"props":4093,"children":4094},{"__ignoreMap":8},[4095],{"type":23,"value":4089},{"type":17,"tag":25,"props":4097,"children":4098},{},[4099],{"type":23,"value":4100},"The registry answers questions that come up constantly in teams:",{"type":17,"tag":134,"props":4102,"children":4103},{},[4104,4114,4124,4134],{"type":17,"tag":138,"props":4105,"children":4106},{},[4107,4112],{"type":17,"tag":142,"props":4108,"children":4109},{},[4110],{"type":23,"value":4111},"Which model is in production?",{"type":23,"value":4113}," Check the registry. The version in the \"Production\" stage is the one serving traffic.",{"type":17,"tag":138,"props":4115,"children":4116},{},[4117,4122],{"type":17,"tag":142,"props":4118,"children":4119},{},[4120],{"type":23,"value":4121},"What changed between versions?",{"type":23,"value":4123}," Each version links back to its run, which has all the parameters and metrics.",{"type":17,"tag":138,"props":4125,"children":4126},{},[4127,4132],{"type":17,"tag":142,"props":4128,"children":4129},{},[4130],{"type":23,"value":4131},"Who promoted this model?",{"type":23,"value":4133}," The registry logs stage transitions.",{"type":17,"tag":138,"props":4135,"children":4136},{},[4137,4142],{"type":17,"tag":142,"props":4138,"children":4139},{},[4140],{"type":23,"value":4141},"Can we roll back?",{"type":23,"value":4143}," The previous version is still in the registry. Transition it back to \"Production.\"",{"type":17,"tag":127,"props":4145,"children":4147},{"id":4146},"the-registrys-limitations",[4148],{"type":23,"value":4149},"The registry's limitations",{"type":17,"tag":25,"props":4151,"children":4152},{},[4153],{"type":23,"value":4154},"The registry manages metadata and stage labels. It does not:",{"type":17,"tag":134,"props":4156,"children":4157},{},[4158,4163,4168],{"type":17,"tag":138,"props":4159,"children":4160},{},[4161],{"type":23,"value":4162},"Actually deploy models. Transitioning to \"Production\" stage is just a label change. You need something else to notice that label and update your serving infrastructure.",{"type":17,"tag":138,"props":4164,"children":4165},{},[4166],{"type":23,"value":4167},"Run validation tests. There's no built-in gate that says \"only promote to Production if accuracy > X.\" You have to build that logic yourself.",{"type":17,"tag":138,"props":4169,"children":4170},{},[4171],{"type":23,"value":4172},"Handle A\u002FB testing or canary deployments. The registry has one \"Production\" slot per model name. Traffic splitting is your problem.",{"type":17,"tag":25,"props":4174,"children":4175},{},[4176],{"type":23,"value":4177},"This is a common source of confusion. People expect the registry to be a deployment tool. It's a catalog.",{"type":17,"tag":36,"props":4179,"children":4181},{"id":4180},"what-mlflow-does-passably",[4182],{"type":23,"value":4183},"What MLflow does passably",{"type":17,"tag":127,"props":4185,"children":4187},{"id":4186},"mlflow-projects",[4188],{"type":23,"value":4189},"MLflow Projects",{"type":17,"tag":25,"props":4191,"children":4192},{},[4193,4195,4201],{"type":23,"value":4194},"MLflow Projects is a packaging format for ML code. You define an ",{"type":17,"tag":53,"props":4196,"children":4198},{"className":4197},[],[4199],{"type":23,"value":4200},"MLproject",{"type":23,"value":4202}," file that specifies the entry point, parameters, and environment:",{"type":17,"tag":48,"props":4204,"children":4207},{"code":4205,"language":675,"meta":8,"className":4206},"name: fraud-detection\nconda_env: conda.yaml\nentry_points:\n main:\n parameters:\n learning_rate: {type: float, default: 0.01}\n epochs: {type: int, default: 50}\n command: \"python train.py --lr {learning_rate} --epochs {epochs}\"\n",[677],[4208],{"type":17,"tag":53,"props":4209,"children":4210},{"__ignoreMap":8},[4211],{"type":23,"value":4205},{"type":17,"tag":25,"props":4213,"children":4214},{},[4215,4217,4222],{"type":23,"value":4216},"The idea is good — make training runs reproducible by pinning the environment and parameterizing the entry point. In practice, most teams outgrow this quickly. Docker containers give you more control, CI\u002FCD systems give you better orchestration, and the ",{"type":17,"tag":53,"props":4218,"children":4220},{"className":4219},[],[4221],{"type":23,"value":4200},{"type":23,"value":4223}," format adds a layer of abstraction that doesn't carry its weight for production workloads.",{"type":17,"tag":25,"props":4225,"children":4226},{},[4227],{"type":23,"value":4228},"It's fine for personal projects and small teams. For anything running on Kubernetes, you'll probably skip it.",{"type":17,"tag":127,"props":4230,"children":4232},{"id":4231},"mlflow-models-the-serving-component",[4233],{"type":23,"value":4234},"MLflow Models (the serving component)",{"type":17,"tag":25,"props":4236,"children":4237},{},[4238,4240,4246],{"type":23,"value":4239},"MLflow can serve models via ",{"type":17,"tag":53,"props":4241,"children":4243},{"className":4242},[],[4244],{"type":23,"value":4245},"mlflow models serve",{"type":23,"value":4247},". It wraps your model in a REST endpoint:",{"type":17,"tag":48,"props":4249,"children":4252},{"code":4250,"language":105,"meta":8,"className":4251},"mlflow models serve -m \"models:\u002Ffraud-detector\u002FProduction\" -p 5001\n",[107],[4253],{"type":17,"tag":53,"props":4254,"children":4255},{"__ignoreMap":8},[4256],{"type":23,"value":4250},{"type":17,"tag":25,"props":4258,"children":4259},{},[4260],{"type":23,"value":4261},"This gives you a working prediction endpoint. It's useful for testing and demos. For production, you'll want more control over the serving infrastructure — custom preprocessing, batching, GPU inference, health checks, autoscaling. Tools like Seldon, KServe, or even a plain Flask\u002FFastAPI app typically replace this.",{"type":17,"tag":36,"props":4263,"children":4265},{"id":4264},"what-mlflow-doesnt-do",[4266],{"type":23,"value":4267},"What MLflow doesn't do",{"type":17,"tag":25,"props":4269,"children":4270},{},[4271],{"type":23,"value":4272},"This is where expectations often diverge from reality.",{"type":17,"tag":127,"props":4274,"children":4276},{"id":4275},"data-versioning",[4277],{"type":23,"value":4278},"Data versioning",{"type":17,"tag":25,"props":4280,"children":4281},{},[4282],{"type":23,"value":4283},"MLflow tracks which parameters and code produced a model, but it does not track which data was used. You can log a hash or a path as a parameter, but MLflow doesn't version datasets, detect data drift, or manage data lineage.",{"type":17,"tag":25,"props":4285,"children":4286},{},[4287],{"type":23,"value":4288},"If you need data versioning, look at DVC, Delta Lake, or LakeFS. This is a separate problem from experiment tracking.",{"type":17,"tag":127,"props":4290,"children":4292},{"id":4291},"feature-stores",[4293],{"type":23,"value":4294},"Feature stores",{"type":17,"tag":25,"props":4296,"children":4297},{},[4298],{"type":23,"value":4299},"MLflow doesn't manage features. It doesn't compute, store, or serve feature values. If you need consistent feature computation between training and serving, you need a feature store (Feast, Tecton, or a custom solution).",{"type":17,"tag":127,"props":4301,"children":4303},{"id":4302},"pipeline-orchestration",[4304],{"type":23,"value":4305},"Pipeline orchestration",{"type":17,"tag":25,"props":4307,"children":4308},{},[4309],{"type":23,"value":4310},"MLflow doesn't schedule or orchestrate multi-step pipelines. It can log the results of each step, but it doesn't manage dependencies between steps, retry failed steps, or schedule runs.",{"type":17,"tag":25,"props":4312,"children":4313},{},[4314],{"type":23,"value":4315},"For orchestration, teams use Airflow, Prefect, Argo Workflows, or Kubernetes CronJobs. MLflow is a logging destination, not an orchestrator.",{"type":17,"tag":127,"props":4317,"children":4319},{"id":4318},"monitoring",[4320],{"type":23,"value":1531},{"type":17,"tag":25,"props":4322,"children":4323},{},[4324],{"type":23,"value":4325},"MLflow doesn't monitor models in production. It doesn't track prediction latency, data drift, or model degradation over time. Monitoring is a separate concern — tools like Evidently, Whylogs, or custom Prometheus metrics handle this.",{"type":17,"tag":127,"props":4327,"children":4329},{"id":4328},"access-control-in-the-open-source-version",[4330],{"type":23,"value":4331},"Access control (in the open-source version)",{"type":17,"tag":25,"props":4333,"children":4334},{},[4335],{"type":23,"value":4336},"The open-source MLflow tracking server has no authentication or authorization. Anyone who can reach the server can see all experiments and modify all models. Databricks' managed MLflow adds access control, but the self-hosted version doesn't have it.",{"type":17,"tag":25,"props":4338,"children":4339},{},[4340],{"type":23,"value":4341},"If you're running MLflow internally, you'll need to put it behind a reverse proxy with authentication, or accept that it's an open system within your network.",{"type":17,"tag":36,"props":4343,"children":4345},{"id":4344},"when-to-adopt-mlflow",[4346],{"type":23,"value":4347},"When to adopt MLflow",{"type":17,"tag":25,"props":4349,"children":4350},{},[4351],{"type":23,"value":4352},"MLflow is worth adopting when:",{"type":17,"tag":134,"props":4354,"children":4355},{},[4356,4361,4366],{"type":17,"tag":138,"props":4357,"children":4358},{},[4359],{"type":23,"value":4360},"You have more than one person training models, or you're training models often enough that you lose track of what you've tried.",{"type":17,"tag":138,"props":4362,"children":4363},{},[4364],{"type":23,"value":4365},"You need a model registry to answer \"what's in production and why.\"",{"type":17,"tag":138,"props":4367,"children":4368},{},[4369],{"type":23,"value":4370},"You want a standard way to log experiments that doesn't depend on a specific cloud vendor.",{"type":17,"tag":25,"props":4372,"children":4373},{},[4374],{"type":23,"value":4375},"MLflow is probably overkill when:",{"type":17,"tag":134,"props":4377,"children":4378},{},[4379,4384],{"type":17,"tag":138,"props":4380,"children":4381},{},[4382],{"type":23,"value":4383},"You're one person training one model occasionally. A notebook with notes is fine.",{"type":17,"tag":138,"props":4385,"children":4386},{},[4387],{"type":23,"value":4388},"You're fully committed to a cloud ML platform (SageMaker, Vertex AI) that has its own tracking built in.",{"type":17,"tag":36,"props":4390,"children":4392},{"id":4391},"the-practical-architecture",[4393],{"type":23,"value":4394},"The practical architecture",{"type":17,"tag":25,"props":4396,"children":4397},{},[4398],{"type":23,"value":4399},"In most setups we see, MLflow occupies a specific slot:",{"type":17,"tag":48,"props":4401,"children":4403},{"code":4402},"Training jobs (K8s \u002F GPU nodes)\n │\n ▼\n MLflow Tracking Server ──▶ Artifact Store (S3 \u002F MinIO)\n │\n ▼\n MLflow Model Registry\n │\n ▼\n Deployment pipeline (CI\u002FCD, Argo, custom)\n │\n ▼\n Serving infrastructure (KServe, Flask, etc.)\n",[4404],{"type":17,"tag":53,"props":4405,"children":4406},{"__ignoreMap":8},[4407],{"type":23,"value":4402},{"type":17,"tag":25,"props":4409,"children":4410},{},[4411],{"type":23,"value":4412},"MLflow handles the middle layers — tracking and registry. Everything above (compute, scheduling) and below (deployment, serving, monitoring) is handled by other tools. Understanding this boundary is key to adopting MLflow without being disappointed by what it can't do.",{"type":17,"tag":938,"props":4414,"children":4415},{},[],{"type":17,"tag":25,"props":4417,"children":4418},{},[4419,4421,4426,4428,4433],{"type":23,"value":4420},"Setting up MLflow as part of a production ML platform is something we do regularly at ",{"type":17,"tag":496,"props":4422,"children":4424},{"href":948,"rel":4423},[500],[4425],{"type":23,"value":952},{"type":23,"value":4427},". If you're figuring out where MLflow fits in your stack, ",{"type":17,"tag":496,"props":4429,"children":4430},{"href":957},[4431],{"type":23,"value":4432},"get in touch",{"type":23,"value":436},{"title":8,"searchDepth":963,"depth":963,"links":4435},[4436,4439,4440,4443,4447,4454,4455],{"id":3881,"depth":963,"text":3884,"children":4437},[4438],{"id":3927,"depth":968,"text":3930},{"id":3992,"depth":963,"text":3995},{"id":4022,"depth":963,"text":4025,"children":4441},[4442],{"id":4146,"depth":968,"text":4149},{"id":4180,"depth":963,"text":4183,"children":4444},[4445,4446],{"id":4186,"depth":968,"text":4189},{"id":4231,"depth":968,"text":4234},{"id":4264,"depth":963,"text":4267,"children":4448},[4449,4450,4451,4452,4453],{"id":4275,"depth":968,"text":4278},{"id":4291,"depth":968,"text":4294},{"id":4302,"depth":968,"text":4305},{"id":4318,"depth":968,"text":1531},{"id":4328,"depth":968,"text":4331},{"id":4344,"depth":963,"text":4347},{"id":4391,"depth":963,"text":4394},"content:articles:what-mlflow-actually-does.md","articles\u002Fwhat-mlflow-actually-does.md","articles\u002Fwhat-mlflow-actually-does",1785575589126]