Practical DevOps Interview Questions

systemctl status nginx sudo systemctl start nginx sudo systemctl restart nginx sudo systemctl enable nginx # start at boot sudo systemctl disable nginx journalctl -u nginx -f # follow service logs sudo systemctl daemon-reload # after editing a unit file Unit files live in /etc/systemd/system. enable creates the symlink for the boot target, while start only runs it now.

find /var/log -type f -size +100M -mtime +7 -exec ls -lh {} + # compress them instead of just listing find /var/log -type f -size +100M -mtime +7 -exec gzip {} + -size +100M matches files above 100 MB, -mtime +7 means modified more than 7 days ago, and -exec with + passes many files to one command call, which is faster than running it once per file.

#!/bin/bash set -u SERVICE="nginx" if ! systemctl is-active --quiet "$SERVICE"; then echo "$(date) $SERVICE down, restarting" >> /var/log/svc-check.log systemctl restart "$SERVICE" if ! systemctl is-active --quiet "$SERVICE"; then echo "$SERVICE failed to start" | mail -s "ALERT" ops@example.com fi fi Run it from cron every few minutes; a real setup would rely on systemd Restart= plus monitoring.

awk '{print $1}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head -10 # total number of unique IPs awk '{print $1}' access.log | sort -u | wc -l The first field in the default log format is the client IP. uniq -c only collapses adjacent lines, so sort must run before it; the final sort -rn shows the busiest IPs first.

#!/bin/bash THRESHOLD=80 df -hP | awk 'NR>1 {gsub("%","",$5); print $5, $6}' | while read -r used mount; do if [ "$used" -ge "$THRESHOLD" ]; then echo "WARNING: $mount is ${used}% full on $(hostname)" fi done df -P keeps each entry on one line, awk strips the percent sign, and the loop compares each value with the threshold.

crontab -e # min hour day month weekday command 30 2 * * * /opt/scripts/backup.sh >> /var/log/backup.log 2>&1 crontab -l # list jobs Use absolute paths because cron runs with a minimal PATH and environment. Redirect output to a log, otherwise errors are mailed or lost. On systemd hosts, timers are an alternative with better logging.

git rebase -i HEAD~3 # in the editor, keep the first line as pick # and change the other two to squash (or s) pick a1b2c3 Add login API squash d4e5f6 Fix typo squash 789abc Add tests # alternative without the editor git reset --soft HEAD~3 git commit -m "Add login API with tests" If the branch was already pushed, update it with git push --force-with-lease.

git reset --soft HEAD~1 # undo commit, keep changes staged git reset HEAD~1 # undo commit, keep changes unstaged # only fix the message or add a missed file git add missed-file.txt git commit --amend Use these only on commits not yet pushed. For a pushed commit on a shared branch, use git revert instead.

git bisect start git bisect bad # current commit has the bug git bisect good v1.4.0 # last known good tag # Git checks out a middle commit; test it, then mark it git bisect good # or: git bisect bad # repeat until Git prints the first bad commit git bisect reset # automate with a test script git bisect run ./run-tests.sh Bisect is a binary search, so 1000 commits need about 10 tests.

git switch -c feature/payment-retry # make changes git add . git commit -m "Add retry for failed payments" git push -u origin feature/payment-retry After -u (set upstream), plain git push and git pull work on this branch without naming the remote. Then open a pull request against main.

git log --oneline --graph --decorate --all # last 10 commits by one author git log --oneline -10 --author="Ravi" # commits touching one file, with diffs git log -p -- src/app.py # who changed each line git blame src/app.py --graph draws branch and merge lines, and --decorate shows branch and tag names next to commits.

git tag -a v1.2.0 -m "Release 1.2.0" git push origin v1.2.0 # push all tags git push origin --tags # list and inspect git tag -l "v1.*" git show v1.2.0 Annotated tags (-a) store author, date and message and are preferred for releases. Many CI pipelines trigger production builds on tags matching v*.

pipeline { agent { label 'linux' } stages { stage('Build') { steps { sh 'mvn -B clean package -DskipTests' } } stage('Test') { steps { sh 'mvn -B test' } post { always { junit 'target/surefire-reports/*.xml' } } } stage('Deploy') { when { branch 'main' } steps { sh './deploy.sh staging' } } } post { failure { echo 'Build failed' } } }

name: CI on: pull_request: branches: [main] jobs: test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: node-version: 20 cache: npm - run: npm ci - run: npm test Add this as .github/workflows/ci.yml and make the job a required status check in branch protection.

jobs: test: runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: [ubuntu-latest, windows-latest] python: ['3.11', '3.12'] steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - run: pip install -r requirements.txt - run: pytest This creates four parallel jobs; fail-fast false lets all finish so you see every failure.

# shell steps in any CI tool IMAGE=registry.example.com/shop/api TAG=$(git rev-parse --short HEAD) echo "$REG_PASS" | docker login registry.example.com -u "$REG_USER" --password-stdin docker build -t $IMAGE:$TAG . docker push $IMAGE:$TAG Tag with the commit SHA so each image is traceable and immutable; avoid relying on latest. Credentials come from the CI secret store, and --password-stdin keeps them out of process lists.

pipeline { agent any environment { SONAR_TOKEN = credentials('sonar-token') } stages { stage('Push') { steps { withCredentials([usernamePassword(credentialsId: 'docker-hub', usernameVariable: 'USER', passwordVariable: 'PASS')]) { sh 'echo $PASS | docker login -u $USER --password-stdin' } } } } } Jenkins masks these values in console output. Use single quotes in sh so Groovy does not interpolate the secret.

stage('Tests') { parallel { stage('Unit') { steps { sh 'mvn test' } } stage('Lint') { steps { sh 'mvn checkstyle:check' } } stage('Security') { agent { docker { image 'aquasec/trivy' } } steps { sh 'trivy fs --exit-code 1 --severity CRITICAL .' } } } } Add failFast true to abort the others when one fails. Each branch can use its own agent.

FROM node:20-alpine WORKDIR /app ENV NODE_ENV=production COPY package.json package-lock.json ./ RUN npm ci --omit=dev COPY . . USER node EXPOSE 3000 CMD ["node", "server.js"] Dependencies are copied and installed before source code so the layer is cached. npm ci gives reproducible installs, USER node avoids running as root, and exec form CMD lets the app receive signals.

FROM maven:3.9-eclipse-temurin-21 AS build WORKDIR /src COPY pom.xml . RUN mvn -B dependency:go-offline COPY src ./src RUN mvn -B package -DskipTests FROM eclipse-temurin:21-jre-alpine WORKDIR /app COPY --from=build /src/target/*.jar app.jar RUN adduser -D appuser USER appuser EXPOSE 8080 ENTRYPOINT ["java", "-jar", "app.jar"] The final image has only the JRE and the JAR, not Maven or source.

services: web: build: . ports: - "8000:8000" environment: DATABASE_URL: postgres://app:secret@db:5432/appdb depends_on: db: condition: service_healthy db: image: postgres:16 environment: POSTGRES_USER: app POSTGRES_PASSWORD: secret POSTGRES_DB: appdb volumes: - pgdata:/var/lib/postgresql/data healthcheck: test: ["CMD", "pg_isready", "-U", "app"] interval: 5s volumes: pgdata:

docker ps # running containers docker logs -f --tail 100 api # follow logs docker exec -it api sh # shell inside docker inspect api # config, IP, mounts, env docker stats # live CPU and memory docker top api # processes inside docker cp api:/app/log.txt . # copy a file out docker events # daemon events Use sh for alpine images, where bash may not exist.

docker run -d --name api \ --memory=512m --memory-swap=512m \ --cpus=1.5 \ --restart=unless-stopped \ myapp:1.0 # change limits on a running container docker update --memory=1g --memory-swap=1g api If the process exceeds the memory limit it is OOM killed, so set application heap sizes below the limit. --cpus caps CPU time rather than reserving cores.

FROM nginx:1.27-alpine COPY site/ /usr/share/nginx/html/ HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \ CMD wget -qO- http://localhost/ || exit 1 # check status docker inspect --format '{{.State.Health.Status}}' web Docker marks the container healthy or unhealthy. Compose can wait on it with condition: service_healthy. Kubernetes ignores HEALTHCHECK and uses its own probes.

apiVersion: apps/v1 kind: Deployment metadata: name: api spec: replicas: 3 selector: matchLabels: app: api template: metadata: labels: app: api spec: containers: - name: api image: registry.example.com/api:1.4.2 ports: - containerPort: 8080 resources: requests: {cpu: 250m, memory: 256Mi} limits: {cpu: 500m, memory: 512Mi} readinessProbe: httpGet: {path: /health, port: 8080} periodSeconds: 5

kubectl get pods -n shop -o wide kubectl describe pod api-7d9f -n shop kubectl logs api-7d9f -n shop -c api --previous kubectl logs -l app=api -f --tail=50 kubectl exec -it api-7d9f -- sh kubectl get events -n shop --sort-by=.lastTimestamp kubectl top pods -n shop kubectl port-forward svc/api 8080:80 kubectl get endpoints api describe and events usually explain scheduling and startup problems; logs explain application problems.

# update the image kubectl set image deployment/api api=registry.example.com/api:1.5.0 kubectl rollout status deployment/api # history and rollback kubectl rollout history deployment/api kubectl rollout undo deployment/api kubectl rollout undo deployment/api --to-revision=3 # pause or restart kubectl rollout pause deployment/api kubectl rollout restart deployment/api In GitOps setups, you change the image tag in Git instead and let Argo CD apply it.

helm repo add bitnami https://charts.bitnami.com/bitnami helm repo update helm install my-redis bitnami/redis -n cache --create-namespace helm upgrade my-redis bitnami/redis -f values-prod.yaml helm list -A helm rollback my-redis 1 helm uninstall my-redis -n cache helm template ./mychart -f values.yaml # render locally Helm is the package manager for Kubernetes. A chart is a set of templated manifests, and values files customise it per environment.

kubectl create secret generic db-creds \ --from-literal=DB_USER=app --from-literal=DB_PASS='S3cret!' -n shop # in the container spec env: - name: DB_PASS valueFrom: secretKeyRef: name: db-creds key: DB_PASS # or load all keys envFrom: - secretRef: name: db-creds Pods do not pick up changed env values until restarted; mounted secret files update automatically.

terraform { required_providers { aws = { source = "hashicorp/aws", version = "~> 5.0" } } } provider "aws" { region = "ap-south-1" } resource "aws_instance" "web" { ami = var.ami_id instance_type = "t3.micro" subnet_id = var.subnet_id tags = { Name = "web-server" Env = "dev" } } Run terraform init, plan and apply. ap-south-1 is the Mumbai region.

# variables.tf variable "instance_type" { type = string default = "t3.micro" description = "EC2 size" } # outputs.tf output "public_ip" { value = aws_instance.web.public_ip } # use them terraform apply -var="instance_type=t3.small" terraform apply -var-file=prod.tfvars terraform output public_ip Outputs also pass values between modules.

terraform { backend "s3" { bucket = "acme-terraform-state" key = "prod/network/terraform.tfstate" region = "ap-south-1" encrypt = true use_lockfile = true } } Then run terraform init (add -migrate-state if moving local state). Enable versioning on the bucket. Older setups use dynamodb_table = "tf-locks" instead of use_lockfile for locking.

- name: Configure web servers hosts: web become: true tasks: - name: Install nginx ansible.builtin.apt: name: nginx state: present update_cache: true - name: Deploy site config ansible.builtin.template: src: site.conf.j2 dest: /etc/nginx/conf.d/site.conf notify: Reload nginx - name: Ensure nginx is running ansible.builtin.service: name: nginx state: started enabled: true handlers: - name: Reload nginx ansible.builtin.service: name: nginx state: reloaded

# inventory.ini [web] web1 ansible_host=10.0.1.10 web2 ansible_host=10.0.1.11 [db] db1 ansible_host=10.0.2.20 [all:vars] ansible_user=ubuntu # ad-hoc commands ansible all -i inventory.ini -m ping ansible web -i inventory.ini -a "uptime" ansible web -i inventory.ini -m apt -a "name=htop state=present" --become Ad-hoc commands suit quick one-off checks; repeatable work belongs in playbooks.

Handlers are tasks that run only when notified by another task that reported a change, and they run once at the end of the play even if notified many times. tasks: - name: Update app config ansible.builtin.copy: src: app.conf dest: /etc/app/app.conf notify: Restart app handlers: - name: Restart app ansible.builtin.service: name: app state: restarted This avoids restarting services when nothing changed.

aws configure aws sts get-caller-identity aws ec2 describe-instances \ --filters Name=instance-state-name,Values=running \ --query 'Reservations[].Instances[].[InstanceId,PrivateIpAddress]' --output table aws s3 cp build.zip s3://acme-artifacts/releases/ aws s3 sync ./site s3://acme-site --delete aws logs tail /aws/lambda/orders --follow get-caller-identity confirms which account and role you are using before you run anything risky.

{ "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": ["s3:ListBucket"], "Resource": "arn:aws:s3:::acme-reports" }, { "Effect": "Allow", "Action": ["s3:GetObject"], "Resource": "arn:aws:s3:::acme-reports/*" } ] } ListBucket applies to the bucket ARN and GetObject to object ARNs, so both resources are needed.

#!/bin/bash set -e dnf update -y dnf install -y nginx systemctl enable --now nginx TOKEN=$(curl -s -X PUT http://169.254.169.254/latest/api/token \ -H 'X-aws-ec2-metadata-token-ttl-seconds: 60') ID=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" \ http://169.254.169.254/latest/meta-data/instance-id) echo "<h1>Served by $ID</h1>" > /usr/share/nginx/html/index.html User data runs once as root on first boot; logs are in /var/log/cloud-init-output.log. This uses IMDSv2.

# ~/.aws/config [profile prod-admin] role_arn = arn:aws:iam::123456789012:role/DevOpsAdmin source_profile = default region = ap-south-1 aws s3 ls --profile prod-admin # manual way aws sts assume-role \ --role-arn arn:aws:iam::123456789012:role/DevOpsAdmin \ --role-session-name ravi-session The role's trust policy must allow your account. Temporary credentials expire, usually after an hour.

eksctl create cluster \ --name shop-dev \ --region ap-south-1 \ --nodegroup-name workers \ --node-type t3.medium \ --nodes 2 --nodes-min 2 --nodes-max 5 \ --managed aws eks update-kubeconfig --name shop-dev --region ap-south-1 kubectl get nodes eksctl builds CloudFormation stacks for the VPC, control plane and node group. In production, teams usually manage EKS with Terraform modules instead.

# requests per second per service over 5 minutes sum by (service) (rate(http_requests_total[5m])) # error ratio (5xx) sum(rate(http_requests_total{status=~"5.."}[5m])) / sum(rate(http_requests_total[5m])) # p95 latency from a histogram histogram_quantile(0.95, sum by (le) (rate(http_request_duration_seconds_bucket[5m]))) rate() works on counters and handles resets; the le label holds histogram bucket bounds.

groups: - name: api-alerts rules: - alert: HighErrorRate expr: | sum(rate(http_requests_total{job="api",status=~"5.."}[5m])) / sum(rate(http_requests_total{job="api"}[5m])) > 0.05 for: 10m labels: severity: page annotations: summary: "API error rate above 5 percent" runbook: "https://wiki.example.com/runbooks/api-errors" for: 10m ignores short spikes; Alertmanager routes by the severity label.

# prometheus.yml global: scrape_interval: 15s scrape_configs: - job_name: node static_configs: - targets: ['10.0.1.10:9100', '10.0.1.11:9100'] - job_name: api metrics_path: /metrics static_configs: - targets: ['api.internal:8080'] # validate and reload promtool check config prometheus.yml curl -X POST http://localhost:9090/-/reload Port 9100 is the node exporter default. In Kubernetes, use service discovery or ServiceMonitors instead.

Allowed downtime = (1 - SLO) x period 99.9 percent over 30 days: 0.001 x 30 x 24 x 60 = 43.2 minutes per month 99.9 percent over a year = about 8.76 hours 99.95 percent per month = about 21.6 minutes 99.99 percent per month = about 4.3 minutes Each extra nine cuts the budget tenfold and usually needs much more redundancy and automation, so pick an SLO that matches real user needs.

# systemd service journalctl -u api --since "1 hour ago" -p err # plain log files grep -i "error" /var/log/app/app.log | tail -50 grep -c "timeout" /var/log/app/app.log # Kubernetes kubectl logs deploy/api --since=1h | grep -i error kubectl logs -l app=api --since=1h --prefix | grep 500 For many hosts, search the central logging tool instead, filtering by service, level and time.