Quick Deployment Guide for Kubernetes¶

# config.yaml

############################
# prepare
############################
# Optional offline installation of system software packages (offline|online)
# Default to "offline" in an offline environment
INSTALL_SOURCE: "offline"

# Optional perform system security hardening github.com/dev-sec/ansible-collection-hardening
OS_HARDEN: false


############################
# role:deploy
############################
# Default: CA will expire in 100 years
# Default: Certificates issued by the CA will expire in 50 years
CA_EXPIRY: "876000h"
CERT_EXPIRY: "438000h"

# kubeconfig configuration parameters
CLUSTER_NAME: "cluster1"
CONTEXT_NAME: "context-{{ CLUSTER_NAME }}"

# k8s version
K8S_VER: "__k8s_ver__"

############################
# role:etcd
############################
# Set different WAL directories to avoid disk I/O contention and improve performance 

# It is recommended to modify to a directory with larger disk space, specific to actual conditions
ETCD_DATA_DIR: "/var/lib/etcd"
ETCD_WAL_DIR: ""


############################
# role:runtime [containerd,docker]
############################
# ------------------------------------------- containerd
# [.] Enable container registry mirror
ENABLE_MIRROR_REGISTRY: true

# [containerd] Base container image
SANDBOX_IMAGE: "easzlab.io.local:5000/easzlab/pause:__pause__"

# [containerd] Container persistent storage directory

# It is recommended to modify to a directory with larger disk space, specific to actual conditions
CONTAINERD_STORAGE_DIR: "/var/lib/containerd" 

# ------------------------------------------- docker
# [docker] Container storage directory
# It is recommended to modify to a directory with larger disk space, specific to actual conditions 
# Only the deployment machine has the Docker service
DOCKER_STORAGE_DIR: "/var/lib/docker"

# [docker] Enable Restful API
ENABLE_REMOTE_API: false

# [docker] Trusted HTTP registry
INSECURE_REG: '["http://easzlab.io.local:5000"]'


############################
# role:kube-master
############################
# k8s cluster master node certificate configuration, multiple IPs and domains can be added (such as adding public IP and domain)
MASTER_CERT_HOSTS:
  - "10.1.1.1"
  - "k8s.easzlab.io"
  #- "www.test.com"

# CIDR length for pod subnet mask on node (determines the maximum number of pod IP addresses each node can allocate)
# If flannel uses the --kube-subnet-mgr parameter, it will read this setting to allocate pod subnets for each node
# https://github.com/coreos/flannel/issues/847
NODE_CIDR_LEN: 24


############################
# role:kube-node
############################
# Kubelet root directory
# It is recommended to modify to a directory with larger disk space, specific to actual conditions 
KUBELET_ROOT_DIR: "/var/lib/kubelet"

# Maximum number of pods on a node
MAX_PODS: 110

# Configure resources reserved for kube components (kubelet, kube-proxy, dockerd, etc.)
# Specific values refer to templates/kubelet-config.yaml.j2
KUBE_RESERVED_ENABLED: "no"

# k8s officially does not recommend casually enabling system-reserved unless you have long-term monitoring and understand the system's resource usage;
# And as the system runs over time, appropriate increases in resource reservations are needed, specific values refer to templates/kubelet-config.yaml.j2
# System reservation settings are based on 4c/8g VMs with minimal system services installed, if using high-performance physical machines, you can appropriately increase the reservation
# Additionally, during cluster installation, apiserver and other resources consume a lot of memory temporarily, it is recommended to reserve at least 1G of memory
SYS_RESERVED_ENABLED: "no"


############################
# role:network [flannel,calico,cilium,kube-ovn,kube-router]
############################
# ------------------------------------------- flannel
# [flannel] Set flannel backend "host-gw", "vxlan", etc.
FLANNEL_BACKEND: "vxlan"
DIRECT_ROUTING: false

# [flannel] flanneld_image: "quay.io/coreos/flannel:v0.10.0-amd64"
flannelVer: "__flannel__"
flanneld_image: "easzlab.io.local:5000/easzlab/flannel:{{ flannelVer }}"

# ------------------------------------------- calico
# [calico] Setting CALICO_IPV4POOL_IPIP="off" can improve network performance, see docs/setup/calico.md for restrictions
CALICO_IPV4POOL_IPIP: "Always"

# [calico] Set the host IP used by calico-node, BGP neighbors establish connections through this address, can be manually specified or automatically discovered
IP_AUTODETECTION_METHOD: "can-reach={{ groups['kube_master'][0] }}"

# [calico] Set calico network backend: brid, vxlan, none
CALICO_NETWORKING_BACKEND: "brid"

# [calico] Whether calico uses route reflectors
# If the cluster scale exceeds 50 nodes, it is recommended to enable this feature
CALICO_RR_ENABLED: false

# CALICO_RR_NODES configure route reflector nodes, if not set, default to cluster master nodes
# CALICO_RR_NODES: ["192.168.1.1", "192.168.1.2"]
CALICO_RR_NODES: []

# [calico] Update supported calico version: [v3.3.x] [v3.4.x] [v3.8.x] [v3.15.x]
calico_ver: "__calico__"

# [calico] Calico main version
calico_ver_main: "{{ calico_ver.split('.')[0] }}.{{ calico_ver.split('.')[1] }}"

# ------------------------------------------- cilium
# [cilium] Image version
cilium_ver: "__cilium__"
cilium_connectivity_check: true
cilium_hubble_enabled: false
cilium_hubble_ui_enabled: false

# ------------------------------------------- kube-ovn
# [kube-ovn] Select OVN DB and OVN Control Plane nodes, default to the first master node
OVN_DB_NODE: "{{ groups['kube_master'][0] }}"

# [kube-ovn] Offline image tarball
kube_ovn_ver: "__kube_ovn__"

# ------------------------------------------- kube-router
# [kube-router] Public clouds have restrictions, generally ipinip should always be enabled; self-hosted environments can set to "subnet"
OVERLAY_TYPE: "full"

# [kube-router] NetworkPolicy support switch
FIREWALL_ENABLE: true

# [kube-router] kube-router image version
kube_router_ver: "__kube_router__"
busybox_ver: "1.28.4"


############################
# role:cluster-addon
############################
# Automatic installation of coredns
dns_install: "yes"
corednsVer: "__coredns__"
ENABLE_LOCAL_DNS_CACHE: true
dnsNodeCacheVer: "__dns_node_cache__"
# Set local DNS cache address
LOCAL_DNS_CACHE: "169.254.20.10"

# Automatic installation of metric server
metricsserver_install: "yes"
metricsVer: "__metrics__"

# Automatic installation of dashboard
dashboard_install: "no"
dashboardVer: "__dashboard__"
dashboardMetricsScraperVer: "__dash_metrics__"

# Automatic installation of prometheus
prom_install: "no"
prom_namespace: "monitor"
prom_chart_ver: "__prom_chart__"

# Automatic installation of nfs-provisioner
# If there is an existing nfs, it must be configured as "yes" and correct server information must be configured
nfs_provisioner_install: "no"
nfs_provisioner_namespace: "kube-system"
nfs_provisioner_ver: "__nfs_provisioner__"
nfs_storage_class: "managed-nfs-storage"
nfs_server: "192.168.1.10"
nfs_path: "/data/nfs"

# Automatic installation of network-check
network_check_enabled: false
network_check_schedule: "*/5 * * * *"

############################
# role:harbor
############################
# Harbor version, full version number
HARBOR_VER: "__harbor__"
HARBOR_DOMAIN: "harbor.easzlab.io.local"
HARBOR_TLS_PORT: 8443

# If set to 'false', you need to put certificates named harbor.pem and harbor-key.pem in the 'down' directory
HARBOR_SELF_SIGNED_CERT: true

# Install extra components
HARBOR_WITH_NOTARY: false
HARBOR_WITH_TRIVY: false
HARBOR_WITH_CLAIR: false
HARBOR_WITH_CHARTMUSEUM: true

# ingress-nginx related configurations
ingress_nginx_install: "yes"
ingressnginxVer: v1.4.0
certgenVer: v20220916-gd32f8c343

# 'etcd' cluster should have odd members (1,3,5,...)
# Must replace IPs according to actual planning
[etcd]
192.168.1.1
192.168.1.2
192.168.1.3

# master nodes
# Must replace IPs according to actual planning
[kube_master]
192.168.1.1
192.168.1.2

# work nodes
# Must replace IPs according to actual planning
[kube_node]
192.168.1.3
192.168.1.4

# [optional] Harbor server, a private Docker registry
# 'NEW_INSTALL': 'true' to install a Harbor server; 'false' to integrate with an existing one
[harbor]
#192.168.1.8 NEW_INSTALL=false

# [optional] Load balancer for accessing k8s from outside
[ex_lb]
#192.168.1.6 LB_ROLE=backup EX_APISERVER_VIP=192.168.1.250 EX_APISERVER_PORT=8443
#192.168.1.7 LB_ROLE=master EX_APISERVER_VIP=192.168.1.250 EX_APISERVER_PORT=8443

# [optional] NTP server for the cluster
[chrony]
#192.168.1.1

[all:vars]
# --------- Main Variables ---------------
# Secure port for apiservers
SECURE_PORT="6443"

# Cluster container-runtime supported: docker, containerd
# if k8s version >= 1.24, docker is not supported
CONTAINER_RUNTIME="containerd"

# Network plugins supported: calico, flannel, kube-router, cilium, kube-ovn
# It is recommended to use this default value or change to "flannel"
CLUSTER_NETWORK="calico"

# Service proxy mode of kube-proxy: 'iptables' or 'ipvs'
PROXY_MODE="ipvs"

# K8S Service CIDR, must not overlap with node(host) networking
# Cannot conflict with the host IP segment
SERVICE_CIDR="10.68.0.0/16"

# Cluster CIDR (Pod CIDR), must not overlap with node(host) networking
# Cannot conflict with the host IP segment
CLUSTER_CIDR="172.20.0.0/16"

# NodePort Range
NODE_PORT_RANGE="30000-32767"

# Cluster DNS Domain
CLUSTER_DNS_DOMAIN="cluster.local"

# -------- Additional Variables (do not change the default value right now) ---
# Binaries Directory
bin_dir="/opt/kube/bin"

# Deploy Directory (kubeasz workspace)
base_dir="/etc/kubeasz"

# Directory for a specific cluster
cluster_dir="{{ base_dir }}/clusters/_cluster_name_"

# CA and other components cert/key Directory
ca_dir="/etc/kubernetes/ssl"