Module 4: Agent-based Installer - Air-gapped Installation

# Create installation directory following Red Hat best practices
mkdir ~/openshift-agent-install
cd ~/openshift-agent-install

# Verify directory structure
echo "Installation directory created: $(pwd)"
echo "This directory will contain all installation manifests and generated artifacts"

# install-config.yaml - Agent-based Installer Configuration
# Reference: https://docs.redhat.com/en/documentation/openshift_container_platform/4.18/html/installing_an_on-premise_cluster_with_the_agent-based_installer/preparing-to-install-with-agent-based-installer#installation-bare-metal-agent-installer-config-yaml_preparing-to-install-with-agent-based-installer
apiVersion: v1
baseDomain: example.com                    # Base domain for cluster DNS records
compute:
- architecture: amd64                      # Supported: amd64, arm64, ppc64le, s390x
  hyperthreading: Enabled
  name: worker
  replicas: 0                             # Number of worker nodes (0 for single-node)
controlPlane:
  architecture: amd64
  hyperthreading: Enabled
  name: master
  replicas: 1                             # Number of control plane nodes
metadata:
  name: sno-cluster                       # Cluster name (must be DNS-compliant)
networking:
  clusterNetwork:
  - cidr: 10.128.0.0/14                   # Pod network CIDR block
    hostPrefix: 23                        # Subnet prefix per node
  machineNetwork:
  - cidr: 192.168.0.0/16                  # Machine network CIDR
  networkType: OVNKubernetes              # Only supported network plugin
  serviceNetwork:
  - 172.30.0.0/16                         # Service network CIDR
platform:
  none: {}                                # Platform type for bare metal
pullSecret: '<your-pull-secret>'          # Red Hat registry pull secret
sshKey: '<your-ssh-public-key>'           # SSH public key for core user access
fips: false                               # FIPS 140-2 compliance mode

# Platform options for different deployment scenarios
platform:
  # Option 1: Bare metal with load balancer VIPs
  baremetal:
    apiVIPs:
    - 192.168.1.100                       # Kubernetes API load balancer VIP
    ingressVIPs:
    - 192.168.1.101                       # Application ingress load balancer VIP

  # Option 2: vSphere virtualized environment
  vsphere:
    apiVIPs:
    - 192.168.1.100
    ingressVIPs:
    - 192.168.1.101
    vCenter: vcenter.example.com
    username: administrator@vsphere.local
    password: password
    datacenter: Datacenter
    defaultDatastore: datastore1
    folder: /Datacenter/vm/openshift

  # Option 3: None (requires external load balancer infrastructure)
  none: {}                                # See platform "none" requirements

# Validate install-config.yaml following Red Hat validation guidelines
echo "=== Validating install-config.yaml Configuration ==="

# Check YAML syntax and structure
if yq eval '.' install-config.yaml > /dev/null 2>&1; then
    echo "✅ YAML syntax valid"
else
    echo "❌ YAML syntax error - check indentation and structure"
    yq eval '.' install-config.yaml
    exit 1
fi

# Validate required fields per Red Hat documentation
echo "Validating required configuration fields..."
CLUSTER_NAME=$(yq eval '.metadata.name' install-config.yaml)
BASE_DOMAIN=$(yq eval '.baseDomain' install-config.yaml)
PULL_SECRET=$(yq eval '.pullSecret' install-config.yaml)
NETWORK_TYPE=$(yq eval '.networking.networkType' install-config.yaml)

# Check required fields
if [[ "$CLUSTER_NAME" != "null" && "$BASE_DOMAIN" != "null" && "$PULL_SECRET" != "null" ]]; then
    echo "✅ Required fields present"
    echo "   Cluster FQDN: $CLUSTER_NAME.$BASE_DOMAIN"
    echo "   Network Type: $NETWORK_TYPE"
else
    echo "❌ Missing required fields - check cluster name, base domain, and pull secret"
    exit 1
fi

# Validate platform configuration
PLATFORM=$(yq eval '.platform | keys | .[0]' install-config.yaml)
echo "✅ Platform configuration: $PLATFORM"

echo "install-config.yaml validation complete - ready for agent-config.yaml creation"

# agent-config.yaml - Host-specific Configuration
# Reference: https://docs.redhat.com/en/documentation/openshift_container_platform/4.18/html/installing_an_on-premise_cluster_with_the_agent-based_installer/preparing-to-install-with-agent-based-installer#agent-host-config_preparing-to-install-with-agent-based-installer
apiVersion: v1beta1
kind: AgentConfig
metadata:
  name: sno-cluster
rendezvousIP: 192.168.111.80              # Bootstrap host IP address
hosts:
- hostname: master-0                      # Host-specific configuration
  role: master                            # Role: master or worker
  interfaces:
  - name: eno1                            # Network interface name
    macAddress: 00:ef:44:21:e6:a5         # MAC address for host identification
  rootDeviceHints:                        # Storage device selection
    deviceName: /dev/sda                  # Target installation device
  networkConfig:                          # NMState format network configuration
    interfaces:
    - name: eno1
      type: ethernet
      state: up
      mac-address: 00:ef:44:21:e6:a5
      ipv4:
        enabled: true
        address:
        - ip: 192.168.111.80
          prefix-length: 23
        dhcp: false
    dns-resolver:
      config:
        server:
        - 192.168.111.1
    routes:
      config:
      - destination: 0.0.0.0/0
        next-hop-address: 192.168.111.2
        next-hop-interface: eno1
        table-id: 254

# agent-config.yaml - Advanced Networking with Bonds and VLANs
# Reference: https://docs.redhat.com/en/documentation/openshift_container_platform/4.18/html/installing_an_on-premise_cluster_with_the_agent-based_installer/preparing-to-install-with-agent-based-installer#agent-install-sample-config-bonds-vlans_preparing-to-install-with-agent-based-installer
apiVersion: v1alpha1
kind: AgentConfig
rendezvousIP: 10.10.10.14
hosts:
- hostname: master0
  role: master
  interfaces:
  - name: enp0s4
    macAddress: 00:21:50:90:c0:10
  - name: enp0s5
    macAddress: 00:21:50:90:c0:20
  networkConfig:
    interfaces:
    - name: bond0.300                     # VLAN interface on bond
      type: vlan
      state: up
      vlan:
        base-iface: bond0
        id: 300
      ipv4:
        enabled: true
        address:
        - ip: 10.10.10.14
          prefix-length: 24
        dhcp: false
    - name: bond0                         # Bond interface
      type: bond
      state: up
      mac-address: 00:21:50:90:c0:10
      ipv4:
        enabled: false
      ipv6:
        enabled: false
      link-aggregation:
        mode: active-backup              # Bond mode
        options:
          miimon: "150"                  # Link monitoring frequency
        port:
        - enp0s4
        - enp0s5
    dns-resolver:
      config:
        server:
        - 10.10.10.11
        - 10.10.10.12
    routes:
      config:
      - destination: 0.0.0.0/0
        next-hop-address: 10.10.10.10
        next-hop-interface: bond0.300
        table-id: 254

# Install and configure mirror registry
echo "=== Setting Up Mirror Registry ==="

# Option 1: Using mirror-registry tool (recommended)
# Download mirror-registry from Red Hat OpenShift Console Downloads page
# Reference: https://docs.redhat.com/en/documentation/openshift_container_platform/4.18/html/disconnected_environments/mirroring-in-disconnected-environments#mirror-registry-localhost_installing-mirroring-creating-registry
echo "=== Downloading Mirror Registry for Red Hat OpenShift ==="
echo "Primary download location: https://console.redhat.com/openshift/downloads#tool-mirror-registry"
echo "Alternative GitHub releases: https://github.com/quay/mirror-registry/releases"

# Download from GitHub releases (latest version)
curl -L -O https://github.com/quay/mirror-registry/releases/latest/download/mirror-registry.tar.gz

# Verify download
echo "Downloaded mirror-registry.tar.gz - verifying archive..."
tar -tzf mirror-registry.tar.gz | head -5

# Extract and install
tar -xzf mirror-registry.tar.gz
./mirror-registry install \
  --quayHostname mirror.example.com \
  --quayRoot /opt/quay \
  --verbose

# Option 2: Using existing registry (if available)
# Configure existing registry for OpenShift mirroring
echo "Alternative: Using existing registry at mirror.example.com:5000"

# Mirror OpenShift release images
echo "=== Mirroring OpenShift Release Images ==="

# Set environment variables
export OCP_RELEASE="4.18.0"
export LOCAL_REGISTRY="mirror.example.com:5000"
export LOCAL_REPOSITORY="openshift/release"
export PRODUCT_REPO="openshift-release-dev"
export LOCAL_SECRET_JSON="/path/to/pull-secret.json"
export RELEASE_NAME="ocp-release"

# Mirror the release images
oc adm release mirror \
  -a ${LOCAL_SECRET_JSON} \
  --from=quay.io/${PRODUCT_REPO}/${RELEASE_NAME}:${OCP_RELEASE}-x86_64 \
  --to=${LOCAL_REGISTRY}/${LOCAL_REPOSITORY} \
  --to-release-image=${LOCAL_REGISTRY}/${LOCAL_REPOSITORY}:${OCP_RELEASE}-x86_64

# The command will output imageContentSources configuration
# Save this output for use in install-config.yaml

# install-config.yaml - Disconnected Configuration
# Reference: https://docs.redhat.com/en/documentation/openshift_container_platform/4.18/html/installing_an_on-premise_cluster_with_the_agent-based_installer/understanding-disconnected-installation-mirroring#configuring-the-agent-based-installer-to-use-mirrored-images
apiVersion: v1
baseDomain: example.com
metadata:
  name: disconnected-cluster
compute:
- architecture: amd64
  hyperthreading: Enabled
  name: worker
  replicas: 3
controlPlane:
  architecture: amd64
  hyperthreading: Enabled
  name: master
  replicas: 3
networking:
  clusterNetwork:
  - cidr: 10.128.0.0/14
    hostPrefix: 23
  machineNetwork:
  - cidr: 192.168.0.0/16
  networkType: OVNKubernetes
  serviceNetwork:
  - 172.30.0.0/16
platform:
  none: {}
pullSecret: '<your-pull-secret-with-mirror-registry>'
sshKey: '<your-ssh-public-key>'
# Mirror registry configuration
additionalTrustBundle: |                  # Mirror registry certificate
  -----BEGIN CERTIFICATE-----
  <mirror-registry-certificate-content>
  -----END CERTIFICATE-----
imageContentSources:                      # Mirror configuration from oc adm release mirror
- mirrors:
  - mirror.example.com:5000/openshift/release
  source: quay.io/openshift-release-dev/ocp-release
- mirrors:
  - mirror.example.com:5000/openshift/release
  source: quay.io/openshift-release-dev/ocp-v4.0-art-dev

# Using oc-mirror plugin for advanced mirroring
echo "=== Using oc-mirror Plugin ==="

# Install oc-mirror plugin from official OpenShift mirror
# Reference: https://docs.redhat.com/en/documentation/openshift_container_platform/4.18/html-single/disconnected_environments/#installing-oc-mirror-v2
curl -O https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/stable/oc-mirror.tar.gz
tar -xzf oc-mirror.tar.gz
sudo mv oc-mirror /usr/local/bin/
chmod +x /usr/local/bin/oc-mirror

# Create ImageSetConfiguration
cat > imageset-config.yaml << EOF
apiVersion: mirror.openshift.io/v1alpha2
kind: ImageSetConfiguration
storageConfig:
  registry:
    imageURL: mirror.example.com:5000/metadata:latest
mirror:
  platform:
    channels:
    - name: stable-4.18
      type: ocp
  operators:
  - catalog: registry.redhat.io/redhat/redhat-operator-index:v4.18
    packages:
    - name: cluster-logging
    - name: elasticsearch-operator
EOF

# Execute mirroring
oc mirror --config=imageset-config.yaml docker://mirror.example.com:5000

# Generate Agent ISO following Red Hat procedures
echo "=== Generating Agent ISO ==="

# Ensure configuration files are in place
ls -la install-config.yaml agent-config.yaml

# Create the agent image
# Reference: https://docs.redhat.com/en/documentation/openshift_container_platform/4.18/html/installing_an_on-premise_cluster_with_the_agent-based_installer/installing-with-agent-based-installer#installing-ocp-agent-boot_installing-with-agent-based-installer
openshift-install --dir . agent create image

# Verify generated artifacts
echo "Generated files:"
ls -la agent.*.iso auth/

# Architecture-specific ISO generation
echo "=== Architecture-Specific ISO Generation ==="

# Check supported architectures
openshift-install version

# Generated ISO files by architecture:
# - agent.x86_64.iso    (Intel/AMD 64-bit)
# - agent.aarch64.iso   (ARM 64-bit)
# - agent.s390x.iso     (IBM Z)
# - agent.ppc64le.iso   (IBM Power)

echo "ISO ready for deployment to target architecture"

# Deploy Agent ISO to target hardware
echo "=== Deploying Agent ISO to Hardware ==="

# Deployment methods (choose appropriate method):

# Method 1: Physical media deployment
echo "Creating bootable USB drive..."
sudo dd if=agent.x86_64.iso of=/dev/sdX bs=4M status=progress
sync

# Method 2: BMC virtual media (example with ipmitool)
echo "Mounting ISO via BMC virtual media..."
ipmitool -I lanplus -H <bmc-ip> -U <username> -P <password> \
  chassis bootdev cdrom options=persistent
# Upload ISO to BMC virtual media interface

# Method 3: PXE boot configuration
echo "Configuring PXE boot for agent ISO..."
# Configure DHCP and TFTP server to serve agent ISO
# Reference: https://docs.redhat.com/en/documentation/openshift_container_platform/4.18/html/installing_an_on-premise_cluster_with_the_agent-based_installer/prepare-pxe-assets-agent

echo "ISO deployment method configured"

# Boot process monitoring
echo "=== Boot Process Monitoring ==="

# 1. Configure BIOS/UEFI boot order
echo "Ensure nodes are configured to boot from chosen media"

# 2. Boot nodes with agent ISO
echo "Booting nodes with agent ISO..."
echo "Nodes will display agent console application"

# 3. Agent console interaction
echo "Agent console will perform connectivity checks:"
echo "  - Release image URL pull check"
echo "  - Network configuration validation"
echo "  - Hardware discovery and validation"

# 4. Network configuration (if needed)
echo "Use agent console to configure networking if checks fail"
echo "Select 'Configure network' to launch NetworkManager TUI"

# Monitor installation progress
echo "=== Installation Progress Monitoring ==="

# Monitor bootstrap completion
# Reference: https://docs.redhat.com/en/documentation/openshift_container_platform/4.18/html/installing_an_on-premise_cluster_with_the_agent-based_installer/installing-with-agent-based-installer#installing-ocp-agent-verify_installing-with-agent-based-installer
openshift-install --dir . agent wait-for bootstrap-complete --log-level=info

# Expected output:
# INFO Bootstrap configMap status is complete
# INFO cluster bootstrap is complete

# Monitor complete installation
openshift-install --dir . agent wait-for install-complete --log-level=info

# Expected output:
# INFO Cluster is installed
# INFO Install complete!
# INFO To access the cluster as the system:admin user when using 'oc', run
# INFO     export KUBECONFIG=/home/core/installer/auth/kubeconfig
# INFO Access the OpenShift web-console here: https://console-openshift-console.apps.cluster-name.example.com

# Post-installation verification
echo "=== Post-Installation Verification ==="

# Set KUBECONFIG environment variable
export KUBECONFIG=./auth/kubeconfig

# Verify cluster nodes
echo "Checking cluster nodes..."
oc get nodes

# Expected output:
# NAME       STATUS   ROLES                  AGE   VERSION
# master-0   Ready    control-plane,master   10m   v1.31.0+...

# Check cluster operators
echo "Verifying cluster operators..."
oc get clusteroperators

# Verify cluster version
echo "Checking cluster version..."
oc get clusterversion

# Check cluster health
echo "Overall cluster health check..."
oc get co --no-headers | awk '{print $1 " " $3 " " $4 " " $5}' | grep -v "True False False"

echo "✅ Agent-based installation completed successfully!"

# Troubleshooting failed installations
echo "=== Troubleshooting Failed Installations ==="

# Gather logs from rendezvous host
ssh core@<rendezvous-ip> agent-gather -O >agent-gather.tar.xz

# If bootstrap fails, gather bootstrap logs
openshift-install --dir . agent wait-for bootstrap-complete --log-level=debug

# If installation fails after bootstrap
export KUBECONFIG=./auth/kubeconfig
oc adm must-gather
tar cvaf must-gather.tar.gz <must_gather_directory>

echo "Logs collected for Red Hat Support analysis"