Quick Start Guide for Kubernetes Deployment¶

# config.yaml

############################
# prepare
############################
# Optional: offline installation of system software packages (offline|online)
# Default for offline environment: "offline"
INSTALL_SOURCE: "offline"

# Optional: apply system security hardening (github.com/dev-sec/ansible-collection-hardening)
OS_HARDEN: false


############################
# role:deploy
############################
# default: CA will expire in 100 years
# default: certs issued by the CA will expire in 50 years
CA_EXPIRY: "876000h"
CERT_EXPIRY: "438000h"

# kubeconfig configuration parameters
CLUSTER_NAME: "cluster1"
CONTEXT_NAME: "context-{{ CLUSTER_NAME }}"

# k8s version
K8S_VER: "__k8s_ver__"

############################
# role:etcd
############################
# Setting a different WAL directory can avoid disk I/O contention and improve performance.

# It is recommended to change to a directory with more disk space, based on actual conditions.
ETCD_DATA_DIR: "/var/lib/etcd"
ETCD_WAL_DIR: ""


############################
# role:runtime [containerd,docker]
############################
# ------------------------------------------- containerd
# [.]Enable container registry mirroring
ENABLE_MIRROR_REGISTRY: true

# [containerd]Base container image
SANDBOX_IMAGE: "easzlab.io.local:5000/easzlab/pause:__pause__"

# [containerd]Container persistent storage directory

# It is recommended to change to a directory with more disk space, based on actual conditions.
CONTAINERD_STORAGE_DIR: "/var/lib/containerd" 

# ------------------------------------------- docker
# [docker]Container storage directory
# It is recommended to change to a directory with more disk space, based on actual conditions. 
# Only the deployment machine will have Docker installed.
DOCKER_STORAGE_DIR: "/var/lib/docker"

# [docker]Enable Restful API
ENABLE_REMOTE_API: false

# [docker]Trusted HTTP registry
INSECURE_REG: '["http://easzlab.io.local:5000"]'


############################
# role:kube-master
############################
# k8s cluster master node certificate configuration. You can add multiple IPs and domain names (e.g., public IP and domain).
MASTER_CERT_HOSTS:
  - "10.1.1.1"
  - "k8s.easzlab.io"
  #- "www.test.com"

# Pod CIDR length on worker nodes (determines the maximum number of pod IPs per node).
# If flannel uses the --kube-subnet-mgr parameter, it will read this setting to assign pod CIDRs to each node.
# https://github.com/coreos/flannel/issues/847
NODE_CIDR_LEN: 24


############################
# role:kube-node
############################
# Kubelet root directory
# It is recommended to change to a directory with more disk space, based on actual conditions. 
KUBELET_ROOT_DIR: "/var/lib/kubelet"

# Maximum number of pods per node
MAX_PODS: 110

# Configure the amount of resources reserved for kube components (kubelet, kube-proxy, dockerd, etc.)
# See templates/kubelet-config.yaml.j2 for values.
KUBE_RESERVED_ENABLED: "no"

# It is not recommended to enable system-reserved without thorough consideration, unless you have long-term monitoring and understand the system's resource usage.
# As the system runs, you may need to increase resource reservations. See templates/kubelet-config.yaml.j2 for values.
# System reservation settings are based on a 4c/8g VM with minimal system services. For high-performance physical machines, you can increase reservations appropriately.
# Additionally, during cluster installation, resource usage by components like apiserver can be high temporarily. It is recommended to reserve at least 1G of memory.
SYS_RESERVED_ENABLED: "no"


############################
# role:network [flannel,calico,cilium,kube-ovn,kube-router]
############################
# ------------------------------------------- flannel
# [flannel]Set flannel backend: "host-gw", "vxlan", etc.
FLANNEL_BACKEND: "vxlan"
DIRECT_ROUTING: false

# [flannel] flanneld_image: "quay.io/coreos/flannel:v0.10.0-amd64"
flannelVer: "__flannel__"
flanneld_image: "easzlab.io.local:5000/easzlab/flannel:{{ flannelVer }}"

# ------------------------------------------- calico
# [calico]Setting CALICO_IPV4POOL_IPIP="off" can improve network performance. See docs/setup/calico.md for constraints.
CALICO_IPV4POOL_IPIP: "Always"

# [calico]Set the host IP used by calico-node. BGP neighbors establish connections through this IP. Can be specified manually or discovered automatically.
IP_AUTODETECTION_METHOD: "can-reach={{ groups['kube_master'][0] }}"

# [calico]Set calico network backend: brid, vxlan, none
CALICO_NETWORKING_BACKEND: "brid"

# [calico]Set whether calico uses route reflectors.
# If the cluster size exceeds 50 nodes, it is recommended to enable this feature.
CALICO_RR_ENABLED: false

# CALICO_RR_NODES configures the nodes for route reflectors. If not set, master nodes are used by default.
# CALICO_RR_NODES: ["192.168.1.1", "192.168.1.2"]
CALICO_RR_NODES: []

# [calico]Update to support calico versions: [v3.3.x] [v3.4.x] [v3.8.x] [v3.15.x]
calico_ver: "__calico__"

# [calico]calico major version
calico_ver_main: "{{ calico_ver.split('.')[0] }}.{{ calico_ver.split('.')[1] }}"

# ------------------------------------------- cilium
# [cilium]Image version
cilium_ver: "__cilium__"
cilium_connectivity_check: true
cilium_hubble_enabled: false
cilium_hubble_ui_enabled: false

# ------------------------------------------- kube-ovn
# [kube-ovn]Select OVN DB and OVN Control Plane node. Default: first master node.
OVN_DB_NODE: "{{ groups['kube_master'][0] }}"

# [kube-ovn]Offline image tar package
kube_ovn_ver: "__kube_ovn__"

# ------------------------------------------- kube-router
# [kube-router]Public clouds have restrictions; generally, ipinip must be enabled. For private environments, can be set to "subnet".
OVERLAY_TYPE: "full"

# [kube-router]NetworkPolicy support toggle
FIREWALL_ENABLE: true

# [kube-router]kube-router image version
kube_router_ver: "__kube_router__"
busybox_ver: "1.28.4"


############################
# role:cluster-addon
############################
# CoreDNS auto-install
dns_install: "yes"
corednsVer: "__coredns__"
ENABLE_LOCAL_DNS_CACHE: true
dnsNodeCacheVer: "__dns_node_cache__"
# Set local DNS cache address
LOCAL_DNS_CACHE: "169.254.20.10"

# Metrics Server auto-install
metricsserver_install: "yes"
metricsVer: "__metrics__"

# Dashboard auto-install
dashboard_install: "no"
dashboardVer: "__dashboard__"
dashboardMetricsScraperVer: "__dash_metrics__"

# Prometheus auto-install
prom_install: "no"
prom_namespace: "monitor"
prom_chart_ver: "__prom_chart__"

# NFS Provisioner auto-install
# If an existing NFS is available, must be set to "yes" and configure correct server information.
nfs_provisioner_install: "no"
nfs_provisioner_namespace: "kube-system"
nfs_provisioner_ver: "__nfs_provisioner__"
nfs_storage_class: "managed-nfs-storage"
nfs_server: "192.168.1.10"
nfs_path: "/data/nfs"

# network-check auto-install
network_check_enabled: false
network_check_schedule: "*/5 * * * *"

############################
# role:harbor
############################
# harbor version, full version number
HARBOR_VER: "__harbor__"
HARBOR_DOMAIN: "harbor.easzlab.io.local"
HARBOR_TLS_PORT: 8443

# if set 'false', you need to put certs named harbor.pem and harbor-key.pem in directory 'down'
HARBOR_SELF_SIGNED_CERT: true

# install extra component
HARBOR_WITH_NOTARY: false
HARBOR_WITH_TRIVY: false
HARBOR_WITH_CLAIR: false
HARBOR_WITH_CHARTMUSEUM: true

# ingress-nginx related configuration
ingress_nginx_install: "yes"
ingressnginxVer: v1.4.0
certgenVer: v20220916-gd32f8c343

# 'etcd' cluster should have odd member(s) (1,3,5,...)
# Must replace IPs according to actual planning
[etcd]
192.168.1.1
192.168.1.2
192.168.1.3

# master node(s)
# Must replace IPs according to actual planning
[kube_master]
192.168.1.1
192.168.1.2

# work node(s)
# Must replace IPs according to actual planning
[kube_node]
192.168.1.3
192.168.1.4

# [optional] harbor server, a private docker registry
# 'NEW_INSTALL': 'true' to install a harbor server; 'false' to integrate with existed one
[harbor]
#192.168.1.8 NEW_INSTALL=false

# [optional] loadbalance for accessing k8s from outside
[ex_lb]
#192.168.1.6 LB_ROLE=backup EX_APISERVER_VIP=192.168.1.250 EX_APISERVER_PORT=8443
#192.168.1.7 LB_ROLE=master EX_APISERVER_VIP=192.168.1.250 EX_APISERVER_PORT=8443

# [optional] ntp server for the cluster
[chrony]
#192.168.1.1

[all:vars]
# --------- Main Variables ---------------
# Secure port for apiservers
SECURE_PORT="6443"

# Cluster container-runtime supported: docker, containerd
# if k8s version >= 1.24, docker is not supported
CONTAINER_RUNTIME="containerd"

# Network plugins supported: calico, flannel, kube-router, cilium, kube-ovn
# It is recommended to use the default value, or change to "flannel"
CLUSTER_NETWORK="calico"

# Service proxy mode of kube-proxy: 'iptables' or 'ipvs'
PROXY_MODE="ipvs"

# K8S Service CIDR, not overlap with node(host) networking
# Must not conflict with host IP range
SERVICE_CIDR="10.68.0.0/16"

# Cluster CIDR (Pod CIDR), not overlap with node(host) networking
# Must not conflict with host IP range
CLUSTER_CIDR="172.20.0.0/16"

# NodePort Range
NODE_PORT_RANGE="30000-32767"

# Cluster DNS Domain
CLUSTER_DNS_DOMAIN="cluster.local"

# -------- Additional Variables (don't change the default value right now) ---
# Binaries Directory
bin_dir="/opt/kube/bin"

# Deploy Directory (kubeasz workspace)
base_dir="/etc/kubeasz"

# Directory for a specific cluster
cluster_dir="{{ base_dir }}/clusters/_cluster_name_"

# CA and other components cert/key Directory
ca_dir="/etc/kubernetes/ssl"