环境说明
kubeadm 安装 kubernetes 集群要求最少2核，操作系统采用 CentOS-7.6.1810，建议采用 7.6.1810 及以上的版本
建议实验环境配置如下：4核心，8GB内存，100G硬盘空间(50G系统盘，/var/lib/docker 50G,50G用于 Ceph-RBD )
10.10.1.4 master14c 8G 50G
10.10.1.5 master2
10.10.1.6 master3
10.10.1.7 node1
10.10.1.8 node2
版本说明
镜像地址
https://cr.console.aliyun.com/repository/cn-qingdao/k8s-docker-image-yfq/kubernetes-1.22.6/images
安装之前
此安装文档安装请注意节点
指定所有节点后后面所有都在所有节点执行，直到指定新的安装的节点，例如master/node
基本配置（所有节点）
防火墙
systemctl disable --now firewalld
setenforce 0
sed -i 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config
主机名
# master1
hostnamectl set-hostname master1
# master2
hostnamectl set-hostname master2
# master3
hostnamectl set-hostname master3
# node1
hostnamectl set-hostname node1
# node2
hostnamectl set-hostname node2
时区
timedatectl set-timezone Asia/Shanghai
时间同步
# 安装
yum makecache fast
yum -y install chrony
systemctl enable --now chronyd
#启动chrony服务
systemctl start chronyd
#设置开机自启
systemctl enable chronyd
#查看chrony服务状态
systemctl status chronyd
#修改 /etc/chrony.conf配置文件
vim /etc/chrony.conf
/**
* 注释掉默认的四个ntp服务器，因为该服务器同步时间略慢
**/
#server 0.centos.pool.ntp.org iburst
#server 1.centos.pool.ntp.org iburst
#server 2.centos.pool.ntp.org iburst
#server 3.centos.pool.ntp.org iburst
/**
* 格式为：server 服务器ip地址 iburst
* 添加阿里云的ntp服务器，可以多写几个ntp服务器，防止第一个服务器宕机，备用的其他ntp服务器可以继续进行时间同步
* ip地址为服务器ip地址，iburst代表的是快速同步时间
**/
server ntp1.aliyun.com iburst
server ntp2.aliyun.com iburst
server ntp3.aliyun.com iburst
#重启chrony服务
systemctl restart chronyd
#查看状态
systemctl status chronyd
swap分区
由于kubernetes的想法是将实例紧密包装到尽可能接近100％，所有的部署应与CPU或者内存限制固定在一起，所以如果调度一个pod程序发送到一台机器上时，它不应该使用交换，因为这会减慢其运行速度，所以关闭swap主要是为了性能考虑
#1、临时关闭 swap分区
swapoff -a
#2、永久关闭
sed -ri 's/.*swap.*/#&/' /etc/fstab
#3、查看swap分区是否已经关闭，若swap一行都显示0，则表示关闭成功
free -m
hosts
[root@master1 ~]# cat /etc/hosts
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
10.10.1.4 master1
10.10.1.5 master2
10.10.1.6 master3
10.10.1.7 node1
10.10.1.8 node2
[root@master1 ~]# for i in $(cat /etc/hosts |awk '{print $1}' |tail -n +3);do scp /etc/hosts root@$i:/etc/hosts;done
ssh 免密登录
[root@master1 ~]# ssh-keygen
...
[root@master1 ~]# for i in $(cat /etc/hosts |awk '{print $1}' |tail -n +3);do ssh--id root@$i;done
常用软件安装
yum -y install epel-release
yum -y install net-tools \
nmap-ncat \
sysstat \
git \
ipset \
ipvsadm \
bash-completion \
wget \
unzip \
lrzsz \
lsof \
vim
docker 源配置
wget https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo \
-O /etc/yum.repos.d/docker-ce.repo
kubernetes 源
[root@master1 ~]# cat /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
[root@master1 ~]#
内核升级
CentOS 7.x 系统自带的 3.10.x 内核存在一些 Bugs,导致运行的 Docker， Kubernetes 不稳定。
Linux 内核官网：https://www.kernel.org/
安装elrepo源
elrepo源官网http://elrepo.org/tiki/tiki-index.php
1. 导入验证key
rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
2. 安装 repo文件
rpm -Uvh https://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm
3. 查看可用的版本
yum makecache fast
yum --disablerepo="*" --enablerepo="elrepo-kernel" search all kernel-lt --show-duplicates
echo $?
输出 0
4. 安装最新的lts版本是5.4
yum --enablerepo="elrepo-kernel" -y install kernel-lt-5.4.173-1.el7.elrepo
5. 更改内核默认启动顺序
grep "menuentry " /boot/grub2/grub.cfg
设置5.4为默认启动内核
grub2-set-default 'CentOS Linux (5.4.173-1.el7.elrepo.x86_64) 7 (Core)'
6. 重启系统
reboot
7. 查看内核版本
uname -r
5.4.173-1.el7.elrepo.x86_64
内核升级完成
加载ipvs模块
https://kubernetes.io/zh/blog/2018/07/09/ipvs-based-in-cluster-load-balancing-deep-dive/
开机自动加载
高版本的 centos 内核 nf_conntrack_ipv4 被 nf_conntrack 替换了
tee /etc/modules-load.d/k8s-ipvs.conf <<EOF
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
nf_conntrack
EOF
手动加载
for mod in ip_vs ip_vs_rr ip_vs_wrr ip_vs_sh nf_conntrack; do
modprobe $mod
done
系统参数优化
tee /etc/sysctl.d/k8s.conf <<EOF
net.ipv6.conf.all.disable_ipv6 = 1
net.ipv6.conf.default.disable_ipv6 = 1
net.ipv6.conf.lo.disable_ipv6 = 1
net.ipv4.neigh.default.gc_stale_time = 120
net.ipv4.conf.all.rp_filter = 0
net.ipv4.conf.default.rp_filter = 0
net.ipv4.conf.default.arp_announce = 2
net.ipv4.conf.lo.arp_announce = 2
net.ipv4.conf.all.arp_announce = 2
net.ipv4.ip_forward = 1
net.ipv4.tcp_max_tw_buckets = 5000
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 1024
net.ipv4.tcp_synack_retries = 2
# 要求iptables不对bridge的数据进行处理
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-arptables = 1
net.netfilter.nf_conntrack_max = 2310720
fs.inotify.max_user_watches=89100
fs.may_detach_mounts = 1
fs.file-max = 52706963
fs.nr_open = 52706963
vm.overcommit_memory=1
# 开启OOM
vm.panic_on_oom=0
# 禁止使用 swap 空间，只有当系统 OOM 时才允许使用它
vm.swappiness=0
# ipvs优化
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_intvl = 30
net.ipv4.tcp_keepalive_probes = 10
EOF
sysctl --system
文件最大打开数
tee /etc/security/limits.d/k8s.conf <<EOF
* soft nproc 1048576
* hard nproc 1048576
* soft nofile 1048576
* hard nofile 1048576
root soft nproc 1048576
root hard nproc 1048576
root soft nofile 1048576
root hard nofile 1048576
EOF
优化日志处理
减少磁盘IO
sed -ri 's/^\$ModLoad imjournal/#&/' /etc/rsyslog.conf
sed -ri 's/^\$IMJournalStateFile/#&/' /etc/rsyslog.conf
sed -ri 's/^#(DefaultLimitCORE)=/\1=100000/' /etc/systemd/system.conf
sed -ri 's/^#(DefaultLimitNOFILE)=/\1=100000/' /etc/systemd/system.conf
ssh 连接优化
sed -ri 's/^#(UseDNS )yes/\1no/' /etc/ssh/sshd_config
集群安装
docker-ce
yum list docker-ce --showduplicates
yum -y install docker-ce-20.10.12 docker-ce-cli-20.10.12
配置daemon.json
配置docker镜像加速以及日志相关配置
Kubernetes官方推荐使用cgroup driver为systemd
kubeadm clusters should be using the "systemd" driver
镜像加速推荐使用阿里云的，上https://cr.console.aliyun.com/注册
cat /etc/docker/daemon.json
{
"registry-mirrors": [
"https://hc9oi3yo.mirror.aliyuncs.com",
"https://docker.mirrors.ustc.edu.cn/",
"https://registry.docker-cn.com"
],
"log-driver": "json-file",
"log-opts": {
"max-file": "10",
"max-size": "100m"
},
"storage-driver": "overlay2",
"exec-opts": ["native.cgroupdriver=systemd"]
}
[root@master1 ~]#
加载配置
systemctl daemon-reload
systemctl restart docker
@kubelet/kubeadm/kubectl
安装指定版本，后面带上版本号，不指定默认安装最新版本，master 节点执行
yum -y install kubeadm-1.22.6-0 kubectl-1.22.6-0 kubelet-1.22.6-0
# node 节点一般不需要安装 kubectl
yum -y install kubeadm-1.22.6 kubelet-1.22.6
systemctl enable kubelet
注意：此处直接启动kubelet 是会报错的，因为没有kubeconfig等配置文件，需要在init 之后才会生成
HA（master）
haproxy + keepalived
yum -y install haproxy keepalived
[root@master1 ~]# cat /etc/haproxy/haproxy.cfg
global
log 127.0.0.1 local2
chroot /var/lib/haproxy
pidfile /var/run/haproxy.pid
maxconn 4000
user haproxy
group haproxy
daemon
stats socket /var/lib/haproxy/stats
defaults
mode http
log global
option httplog
option dontlognull
option http-server-close
option redispatch
retries 3
timeout http-request 10s
timeout queue 1m
timeout connect 10s
timeout client 1m
timeout server 1m
timeout http-keep-alive 10s
timeout check 10s
maxconn 3000
listen k8s-apiserver
bind *:8443
mode tcp
timeout client 1h
timeout connect 1h
log global
option tcplog
balance roundrobin
server master1 10.10.1.4:6443 check
server master2 10.10.1.5:6443 check
server master3 10.10.16:6443 check
acl is_websocket hdr(Upgrade) -i WebSocket
acl is_websocket hdr_beg(Host) -i ws
[root@master1 ~]#
[root@master1 ~]# cat /etc/keepalived/keepalived.conf
global_defs {
router_id 100
vrrp_version 2
vrrp_garp_master_delay 1
vrrp_mcast_group4 224.0.0.18
}
vrrp_script chk_haproxy {
script "/usr/bin/nc -nvz -w 2 127.0.0.1 8443"
timeout 1
interval 1 # check every 1 second
fall 2 # require 2 failures for KO
rise 2 # require 2 successes for OK
}
vrrp_instance lb-vips {
state MASTER
interface ens33
virtual_router_id 100
priority 150
advert_int 1
nopreempt
track_script {
chk_haproxy
}
authentication {
auth_type PASS
auth_pass blahblah
}
virtual_ipaddress {
10.10.1.9/24 dev ens33
}
}
[root@master1 ~]#
systemctl enable --now haproxy
systemctl enable --now keepalived
# 检查
ip a
journalctl -fu keepalived
# 可以看大master1 已经出现了 vip 10.10.1.9 其他节点为backend
配置keepalived 的时候注意 priority 权重 ，节点之间是不一样的，权重越高优先级越大
初始化
方法1
提前下载初始化master节点所需镜像
# 查看安装kubernetes v1.23.0需要哪些镜像
kubeadm config images list --kubernetes-version=v1.22.6
# 编写镜像下载脚本并设置可执行权限
vim k8s-image-download.sh
#!/bin/bash
if [ $# -ne 1 ];then
echo "The format is: ./`basename $0` kubernetes-version"
exit 1
fi
version=$1
images=`kubeadm config images list --kubernetes-version=${version} | awk -F '/' '{print $2}'`
for imageName in ${images[@]} ; do
docker pull registry.aliyuncs.com/google_containers/$imageName
docker tag registry.aliyuncs.com/google_containers/$imageName k8s.gcr.io/$imageName
docker rmi registry.aliyuncs.com/google_containers/$imageName
done
bash /k8s-image-download.sh v1.22.6
提示：--apiserver-advertise-address需要改成你自己master节点的ip地址
kubeadm init \
--apiserver-advertise-address=192.168.198.145 \
--image-repository registry.aliyuncs.com/google_containers \
--kubernetes-version=v1.23.0 \
--service-cidr=10.96.0.0/12 \
--pod-network-cidr=10.244.0.0/16
解释：
--apiserver-advertise-address：指定用master节点的ip地址 与 Cluster的其他节点通信。
*--image-repository：Kubenetes默认Registries地址是 k8s.gcr.io，一般在国内并不能访问 gcr.io， *可以将其指定为阿里云镜像地址：registry.aliyuncs.com/google_containers。
--kubernetes-version=v1.23.0：指定要安装kubernetes的版本号。*--service-cidr：指定Service网络的范围，即负载均衡VIP使用的IP地址段。--pod-network-cidr：指定Pod网络的范围，即Pod的IP地址段。
可以看到程序停止在[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'，这是因为它在下载kubernetes相关组件的镜像，所以耐心等待一段时间
方法2
准备kube admin 文件
kubeadm config print init-defaults --component-configs \
KubeProxyConfiguration,KubeletConfiguration > kubeadm-config.yaml
修改
[root@master1 ~]# cat kubeadm-config.yaml
apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
token: abcdef.0123456789abcdef
ttl: 24h0m0s
usages:
- signing
- authentication
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 10.10.1.4 #执行节点
bindPort: 6443
nodeRegistration:
criSocket: /var/run/dockershim.sock
imagePullPolicy: IfNotPresent
name: master1 # 修改为第一台执行节点的hostname
taints: null
---
controlPlaneEndpoint: 10.10.1.9:8443 # 新增控制平台地址
apiServer:
timeoutForControlPlane: 4m0s
extraArgs:
authorization-mode: "Node,RBAC"
# nfs/rbd pvc 需要用到
feature-gates: RemoveSelfLink=false
enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeClaimResize,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,Priority"
etcd-servers: https://master1:2379,https://master2:2379,https://master3:2379 # etcd 节点列表
certSANs:
- 10.10.1.9 # VIP 地址
- 10.96.0.1 # service cidr的第一个ip
- 127.0.0.1 # 多个master的时候负载均衡出问题了能够快速使用localhost调试
- master1
- master2
- master3
- kubernetes
- kubernetes.default
- kubernetes.default.svc
- kubernetes.default.svc.cluster.local
extraVolumes:
- hostPath: /etc/localtime
mountPath: /etc/localtime
name: timezone
readOnly: true
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager:
extraVolumes:
- hostPath: /etc/localtime
mountPath: /etc/localtime
name: timezone
readOnly: true
dns: {}
etcd:
local:
dataDir: /var/lib/etcd
# etcd 高可用，需要配置多个节点
serverCertSANs:
- master1
- master2
- master3
peerCertSANs:
- master1
- master2
- master3
imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers # 仓库地址，访问不到时可以使用国内镜像仓库
kind: ClusterConfiguration
kubernetesVersion: 1.22.6 # 指定安装版本
networking:
dnsDomain: cluster.local
serviceSubnet: 10.96.0.0/12
podSubnet: 10.244.0.0/16 # 指定 pod 网络
scheduler:
extraVolumes:
- hostPath: /etc/localtime
mountPath: /etc/localtime
name: timezone
readOnly: true
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: 0.0.0.0
bindAddressHardFail: false
clientConnection:
acceptContentTypes: ""
burst: 0
contentType: ""
kubeconfig: /var/lib/kube-proxy/kubeconfig.conf
qps: 0
clusterCIDR: ""
configSyncPeriod: 0s
conntrack:
maxPerCore: null
min: null
tcpCloseWaitTimeout: null
tcpEstablishedTimeout: null
detectLocalMode: ""
enableProfiling: false
healthzBindAddress: ""
hostnameOverride: ""
iptables:
masqueradeAll: false
masqueradeBit: null
minSyncPeriod: 0s
syncPeriod: 0s
ipvs:
excludeCIDRs: null
minSyncPeriod: 0s
scheduler: ""
strictARP: false
syncPeriod: 0s
tcpFinTimeout: 0s
tcpTimeout: 0s
udpTimeout: 0s
kind: KubeProxyConfiguration
metricsBindAddress: ""
mode: "ipvs" # ipvs 模式
nodePortAddresses: null
oomScoreAdj: null
portRange: ""
showHiddenMetricsForVersion: ""
udpIdleTimeout: 0s
winkernel:
enableDSR: false
networkName: ""
sourceVip: ""
---
apiVersion: kubelet.config.k8s.io/v1beta1
authentication:
anonymous:
enabled: false
webhook:
cacheTTL: 0s
enabled: true
x509:
clientCAFile: /etc/kubernetes/pki/ca.crt
authorization:
mode: Webhook
webhook:
cacheAuthorizedTTL: 0s
cacheUnauthorizedTTL: 0s
cgroupDriver: systemd
clusterDNS:
- 10.96.0.10
clusterDomain: cluster.local
cpuManagerReconcilePeriod: 0s
evictionPressureTransitionPeriod: 0s
fileCheckFrequency: 0s
healthzBindAddress: 127.0.0.1
healthzPort: 10248
httpCheckFrequency: 0s
imageMinimumGCAge: 0s
kind: KubeletConfiguration
logging: {}
memorySwap: {}
nodeStatusReportFrequency: 0s
nodeStatusUpdateFrequency: 0s
rotateCertificates: true
runtimeRequestTimeout: 0s
shutdownGracePeriod: 0s
shutdownGracePeriodCriticalPods: 0s
staticPodPath: /etc/kubernetes/manifests
streamingConnectionIdleTimeout: 0s
syncFrequency: 0s
volumeStatsAggPeriod: 0s
[root@master1 ~]#
验证语法
kubeadm init --config kubeadm-config.yaml --dry-run
预先拉取镜像
kubeadm config images list --cnfig kubeadm-config.yaml
kubeadm config images pull --config kubeadm-config.yaml
初始化
kubeadm init --config kubeadm-config.yaml --upload-certs
--upload-certs
将证书保存到kube-system名称空间下名为extension-apiserver-authentication的configmap中，这样其他控制平面加入的话只要加上--control-plane --certificate-key` 并带上相应的key就可以拿到证书并下载到本地
收尾
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
节点加入
master 和 node 的节点方式和token在 init 完成后 会显示出来，直接在对应节点上面操作即可
网络插件
网络插件有：flannel、calico、canal、kube-router、weave net等，这里使用calico网络插件
calico 官方文档
https://docs.projectcalico.org/getting-started/kubernetes/self-managed-onprem/onpremises
curl https://docs.projectcalico.org/manifests/calico.yaml -O
# 修改默认pod 网络，如果是默认的192.168.0.0/16，可以忽略
# flannel 默认 pod 网络地址 10.244.0.0/16 calico 默认 pod 网络地址 192.168.0.0/16
···
- name: CALICO_IPV4POOL_CIDR
value: "10.244.0.0/16"
···
要保证 init 的时候和 calico指定的网络是一致的
[root@master1 ~]# cat kubeadm-config.yaml |grep 244
podSubnet: 10.244.0.0/16 # 指定 pod 网络
[root@master1 ~]# grep 244 calico.yaml
value: "10.244.0.0/16"
[root@master1 ~]#
部署
kubectl apply -f calico.yaml
[root@master1 ~]# kubectl apply -f calico.yaml
configmap/calico-config created
customresourcedefinition.apiextensions.k8s.io/bgpconfigurations.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/bgppeers.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/blockaffinities.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/caliconodestatuses.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/clusterinformations.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/felixconfigurations.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/globalnetworkpolicies.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/globalnetworksets.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/hostendpoints.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/ipamblocks.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/ipamconfigs.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/ipamhandles.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/ippools.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/ipreservations.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/kubecontrollersconfigurations.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/networkpolicies.crd.projectcalico.org created
customresourcedefinition.apiextensions.k8s.io/networksets.crd.projectcalico.org created
clusterrole.rbac.authorization.k8s.io/calico-kube-controllers created
clusterrolebinding.rbac.authorization.k8s.io/calico-kube-controllers created
clusterrole.rbac.authorization.k8s.io/calico-node created
clusterrolebinding.rbac.authorization.k8s.io/calico-node created
daemonset.apps/calico-node created
serviceaccount/calico-node created
deployment.apps/calico-kube-controllers created
serviceaccount/calico-kube-controllers created
poddisruptionbudget.policy/calico-kube-controllers created
[root@master1 ~]#
等待一会儿...
[root@master1 ~]# kubectl get no
NAME STATUS ROLES AGE VERSION
master1 Ready control-plane,master 103m v1.22.6
master2 Ready control-plane,master 97m v1.22.6
master3 Ready control-plane,master 96m v1.22.6
node1 Ready <none> 100m v1.22.6
node2 Ready <none> 100m v1.22.6
[root@master1 ~]#
测试集群
创建一个deploy 和一个service
kubectl create deployment nginx --image=nginx:1.20.1-alpine
kubectl expose deployment/nginx --name=nginx --port=80 --type=NodePort
[root@master1 ~]# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 105m
nginx NodePort 10.96.135.205 <none> 80:32622/TCP 41s
[root@master1 ~]# kubectl get po
NAME READY STATUS RESTARTS AGE
nginx-779c9459b8-77zpk 0/1 ContainerCreating 0 50s
[root@master1 ~]#
如果需要给master上面调度（不推荐）
去除 master 标签
kubectl label node master1 node-role.kubernetes.io/master-
kubectl label node master2 node-role.kubernetes.io/master-
kubectl label node master3 node-role.kubernetes.io/master-
去除污点
kubectl taint node master1 node-role.kubernetes.io/master:NoSchedule-
kubectl taint node master2 node-role.kubernetes.io/master:NoSchedule-
kubectl taint node master3 node-role.kubernetes.io/master:NoSchedule-
添加污点
kubectl taint node master1 node-role.kubernetes.io/master="":NoSchedule
kubectl taint node master2 node-role.kubernetes.io/master="":NoSchedule
kubectl taint node master3 node-role.kubernetes.io/master="":NoSchedule
查看 pod 和 service
kubectl get svc -l app=nginx
输出如下
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
nginx NodePort 10.100.168.172 <none> 80:30009/TCP 2m5s
访问测试
curl 10.100.168.172
输出如下
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
body {
width: 35em;
margin: 0 auto;
font-family: Tahoma, Verdana, Arial, sans-serif;
}
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>
<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>
<p><em>Thank you for using nginx.</em></p>
</body>
</html>
查看ipvs
[root@master1 ~]# ipvsadm -L
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP master1:32622 rr
-> 10.244.166.129:http Masq 1 0 0
TCP master1:32622 rr
-> 10.244.166.129:http Masq 1 0 0
TCP master1:https rr
-> master1:sun-sr-https Masq 1 1 0
-> master2:sun-sr-https Masq 1 1 0
-> master3:sun-sr-https Masq 1 2 0
TCP master1:domain rr
-> 10.244.104.2:domain Masq 1 0 0
-> 10.244.104.3:domain Masq 1 0 0
TCP master1:9153 rr
-> 10.244.104.2:9153 Masq 1 0 0
-> 10.244.104.3:9153 Masq 1 0 0
TCP master1:http rr
-> 10.244.166.129:http Masq 1 0 1
TCP master1:32622 rr
-> 10.244.166.129:http Masq 1 0 0
UDP master1:domain rr
-> 10.244.104.2:domain Masq 1 0 0
-> 10.244.104.3:domain Masq 1 0 0
[root@master1 ~]#
存储（可选）
ceph
https://huangzhongde.cn/istio/Chapter2/Chapter2-9.html
核心组件
ingress/metrics/dashboard/EFK/prometheus
ingress
ingress 官方网站
ingress 仓库地址
裸机部署 ingress-nginx 纯软件方案 MetalLB
MetalLB 为不在受支持的云提供程序上运行的 Kubernetes 群集提供了网络负载平衡器实现，从而有效地允许在任何群集中使用 LoadBalancer Services。
MetalLB 仓库地址
MetalLB 安装文档
Kubernetes-v1.22+ 需要使用 ingress-nginx>=1.0，因为networking.k8s.io/v1beta已经移除
Ingress and IngressClass resources have graduated to networking.k8s.io/v1. Ingress and IngressClass types in the extensions/v1beta1 and networking.k8s.io/v1beta1 API versions are deprecated and will no longer be served in 1.22+. Persisted objects can be accessed via the networking.k8s.io/v1 API. Notable changes in v1 Ingress objects (v1beta1 field names are unchanged):
spec.backend->spec.defaultBackendserviceName->service.nameservicePort->service.port.name(for string values)servicePort->service.port.number(for numeric values)pathTypeno longer has a default value in v1; "Exact", "Prefix", or "ImplementationSpecific" must be specified Other Ingress API updates:backends can now be resource or service backends
pathis no longer required to be a valid regular expression (#89778, @cmluciano) [SIG API Machinery, Apps, CLI, Network and Testing]
详情见
CHANGELOG-1.19
部署metallb（可选）
修改 kube-proxy 配置
kubectl get configmap kube-proxy -n kube-system -o yaml | \
sed -e "s/strictARP: false/strictARP: true/" | \
kubectl apply -f - -n kube-system
创建 namespace
[root@master1 ~]# kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.11.0/manifests/namespace.yaml
namespace/metallb-system created
[root@master1 ~]#
执行完会创建一个名为 metallb-system 的名称空间。
部署 metallb
[root@master1 ~]# kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.11.0/manifests/metallb.yaml
Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+
podsecuritypolicy.policy/controller configured
podsecuritypolicy.policy/speaker configured
serviceaccount/controller created
serviceaccount/speaker created
clusterrole.rbac.authorization.k8s.io/metallb-system:controller unchanged
clusterrole.rbac.authorization.k8s.io/metallb-system:speaker unchanged
role.rbac.authorization.k8s.io/config-watcher created
role.rbac.authorization.k8s.io/pod-lister created
role.rbac.authorization.k8s.io/controller created
clusterrolebinding.rbac.authorization.k8s.io/metallb-system:controller unchanged
clusterrolebinding.rbac.authorization.k8s.io/metallb-system:speaker unchanged
rolebinding.rbac.authorization.k8s.io/config-watcher created
rolebinding.rbac.authorization.k8s.io/pod-lister created
rolebinding.rbac.authorization.k8s.io/controller created
daemonset.apps/speaker created
deployment.apps/controller created
[root@master1 ~]#
配置
支持二层，BGP 等方式，这里简单的使用二层配置
二层配置：只需要配置 IP 地址池即可
tee metallb-config.yaml <<EOF
apiVersion: v1
kind: ConfigMap
metadata:
namespace: metallb-system
name: config
data:
config: |
address-pools:
- name: default
protocol: layer2
addresses:
- 10.10.1.150-10.1.1.200 # IP 地址池，根据实际情况分配
EOF
为 matallb 创建 configMap
kubectl apply -f metallb-config.yaml
configmap/config created
查看 pod 运行状态
[root@master1 ~]# kubectl get po -n metallb-system
NAME READY STATUS RESTARTS AGE
controller-7dcc8764f4-2qb8q 1/1 Running 0 111s
speaker-2j2pc 1/1 Running 0 111s
speaker-q8w4w 1/1 Running 0 110s
speaker-rhpxh 1/1 Running 0 110s
speaker-swxtm 1/1 Running 0 111s
speaker-sxbpv 1/1 Running 0 111s
[root@master1 ~]#
部署 ingress-nginx
部署 ingress-nginx
默认这里的镜像使用的是k8s 官方的是k8s 官方的，国内拉取是失败
这里提供两个下载地址下载镜像包，然后到node上面 执行
wget https://download.yutao.co/mirror/controller
wget https://download.yutao.co/mirror/kube-webhook-certgen
docker load -i kube-webhook-certgen
docker load -i controller
[root@master1 ~]# wget https://download.yutao.co/mirror/deploy.yaml
[root@master1 ~]# kubectl apply -f deploy.yaml
namespace/ingress-nginx created
serviceaccount/ingress-nginx created
configmap/ingress-nginx-controller created
clusterrole.rbac.authorization.k8s.io/ingress-nginx created
clusterrolebinding.rbac.authorization.k8s.io/ingress-nginx created
role.rbac.authorization.k8s.io/ingress-nginx created
rolebinding.rbac.authorization.k8s.io/ingress-nginx created
service/ingress-nginx-controller-admission created
service/ingress-nginx-controller created
deployment.apps/ingress-nginx-controller created
ingressclass.networking.k8s.io/nginx created
validatingwebhookconfiguration.admissionregistration.k8s.io/ingress-nginx-admission created
serviceaccount/ingress-nginx-admission created
clusterrole.rbac.authorization.k8s.io/ingress-nginx-admission created
clusterrolebinding.rbac.authorization.k8s.io/ingress-nginx-admission created
role.rbac.authorization.k8s.io/ingress-nginx-admission created
rolebinding.rbac.authorization.k8s.io/ingress-nginx-admission created
job.batch/ingress-nginx-admission-create created
job.batch/ingress-nginx-admission-patch created
[root@master1 ~]# kubectl get po -n ingress-nginx
NAME READY STATUS RESTARTS AGE
ingress-nginx-admission-create--1-8qhvn 0/1 Completed 0 8s
ingress-nginx-admission-patch--1-xjll7 0/1 Completed 1 8s
ingress-nginx-controller-ccb6c766f-rqghb 0/1 Running 0 9s
查看 pod 运行情况
kubectl get po -n ingress-nginx
NAME READY STATUS RESTARTS AGE
ingress-nginx-admission-create--1-ng9ct 0/1 Completed 0 55s
ingress-nginx-admission-patch--1-kkhst 0/1 Completed 0 55s
ingress-nginx-controller-778574f59b-bstjw 0/1 Running 0 56s
如果不是公有云的环境，是不支持LoadBalancer的，所以你需要到deploy.yaml配置文件修改（在第276行），将LoadBalancer改为NodePort（默认）。
部署了 Metallb 的可以将 NodePort 修改为 LoadBalancer (公有云)
一般生产不适用metallb 而是用云提供的lb
[root@master1 ~]# kubectl patch svc -n ingress-nginx ingress-nginx-controller -p '{"spec":{"type": "LoadBalancer"}}'
service/ingress-nginx-controller patched
[root@master1 ~]#
检查安装
[root@master1 ~]# kubectl get po -n ingress-nginx
NAME READY STATUS RESTARTS AGE
ingress-nginx-admission-create--1-jvqnn 0/1 Completed 0 3m13s
ingress-nginx-admission-patch--1-fv9dl 0/1 Completed 0 3m13s
ingress-nginx-controller-ccb6c766f-mgcct 1/1 Running 0 3m13s
[root@master1 ~]# kubectl get svc -n ingress-nginx
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
ingress-nginx-controller LoadBalancer 10.102.115.51 10.10.1.150 80:30114/TCP,443:30503/TCP 3m14s
ingress-nginx-controller-admission ClusterIP 10.111.107.85 <none> 443/TCP 3m15s
[root@master1 ~]#
如果配置metalb 且为loadblancer 会看到看到已经分配了地址池中的一个地址 10.10.1.xxx
检测安装版本信息
POD_NAMESPACE=ingress-nginx
POD_NAME=$(kubectl get pods -n $POD_NAMESPACE -l app.kubernetes.io/name=ingress-nginx,app.kubernetes.io/component=controller -o jsonpath='{.items[0].metadata.name}')
kubectl exec -it $POD_NAME -n $POD_NAMESPACE -- /nginx-ingress-controller --version
-------------------------------------------------------------------------------
NGINX Ingress controller
Release: v1.1.1
Build: a17181e43ec85534a6fea968d95d019c5a4bc8cf
Repository: https://github.com/kubernetes/ingress-nginx
nginx version: nginx/1.19.9
-------------------------------------------------------------------------------
后端 ingress 代理应用示例
创建 myapp 应用
---
kind: Service
apiVersion: v1
metadata:
name: myapp
spec:
selector:
app: myapp
type: ClusterIP
ports:
- name: http
port: 80
targetPort: 80
externalIPs:
- 10.10.1.150
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: myapp
spec:
selector:
matchLabels:
app: myapp
template:
metadata:
labels:
app: myapp
spec:
containers:
- name: myapp
image: nginx:alpine
resources:
limits:
memory: "128Mi"
cpu: "500m"
ports:
- containerPort: 80
service/myapp created
deployment.apps/myapp created
创建 ingress 文件
kubectl apply -f - <<EOF
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: test-myapp
annotations:
# 指定 Ingress Controller 的类型
kubernetes.io/ingress.class: "nginx"
spec:
rules:
- host: myapp.hzde.com
http:
paths:
- path: "/"
pathType: Prefix
backend:
service:
name: myapp
port: number: 80
EOF
ingress.networking.k8s.io/test-myapp created
添加 hosts 并尝试访问
查看 ingress 对应节点的端口
[root@master1 ~]# kubectl get svc -n ingress-nginx
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
ingress-nginx-controller LoadBalancer 10.102.115.51 10.10.1.150 80:30114/TCP,443:30503/TCP 4m21s
ingress-nginx-controller-admission ClusterIP 10.111.107.85 <none> 443/TCP 4m22s
[root@master1 ~]#
echo '10.10.1.150 myapp.hzde.com' >> /etc/hosts
curl myapp.hzde.com
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
html { color-scheme: light dark; }
...
<p><em>Thank you for using nginx.</em></p>
</body>
</html>
不修改hosts访问 myapp.hzde.com
在头部指定 host
[root@master1 ~]# curl -H "Host:myapp.hzde.com" http://10.10.1.150
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
html { color-scheme: light dark; }
body { width: 35em; margin: 0 auto;
font-family: Tahoma, Verdana, Arial, sans-serif; }
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>
<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>
<p><em>Thank you for using nginx.</em></p>
</body>
</html>
[root@master1 ~]#
ingress HTTPS 访问
参考文档TLS/HTTPS
创建自签证书文件
openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout tls.key -out tls.crt -subj "/CN=nginx/O=nginx"
Generating a 2048 bit RSA private key
......................................................+++
..........................+++
writing new private key to 'tls.key'
-----
创建 secret
kubectl create secret tls tls-secret --key tls.key --cert tls.crt
secret/tls-secret created
创建 tls ingress
kubectl apply -f - <<EOF
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: tls-myapp
annotations:
# 指定 Ingress Controller 的类型
kubernetes.io/ingress.class: "nginx"
spec:
tls:
- hosts:
- myapp2.hzde.com
secretName: tls-secret
rules:
- host: myapp2.hzde.com
http:
paths:
- path: "/"
pathType: Prefix
backend:
service:
name: myapp
port:
number: 80
EOF
ingress.networking.k8s.io/tls-myapp created
测试
echo '192.168.122.191 myapp2.hzde.com' >> /etc/hosts
curl -sSk https://myapp2.hzde.com
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
body {
width: 35em;
margin: 0 auto;
font-family: Tahoma, Verdana, Arial, sans-serif;
}
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>
<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>
<p><em>Thank you for using nginx.</em></p>
</body>
</html>
ingress 高级用法请查看官方文档。
metric server
metrics-server 仓库地址
下载 metrics-server 清单文件
wget -O metrics-server-v0.6.0.yaml https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.6.0/components.yaml
如果因为网络下载不了可直接使用下面的文件
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
k8s-app: metrics-server
name: metrics-server
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
k8s-app: metrics-server
rbac.authorization.k8s.io/aggregate-to-admin: "true"
rbac.authorization.k8s.io/aggregate-to-edit: "true"
rbac.authorization.k8s.io/aggregate-to-view: "true"
name: system:aggregated-metrics-reader
rules:
- apiGroups:
- metrics.k8s.io
resources:
- pods
- nodes
verbs:
- get
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
k8s-app: metrics-server
name: system:metrics-server
rules:
- apiGroups:
- ""
resources:
- nodes/metrics
verbs:
- get
- apiGroups:
- ""
resources:
- pods
- nodes
verbs:
- get
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
labels:
k8s-app: metrics-server
name: metrics-server-auth-reader
namespace: kube-system
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: extension-apiserver-authentication-reader
subjects:
- kind: ServiceAccount
name: metrics-server
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
k8s-app: metrics-server
name: metrics-server:system:auth-delegator
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:auth-delegator
subjects:
- kind: ServiceAccount
name: metrics-server
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
k8s-app: metrics-server
name: system:metrics-server
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:metrics-server
subjects:
- kind: ServiceAccount
name: metrics-server
namespace: kube-system
---
apiVersion: v1
kind: Service
metadata:
labels:
k8s-app: metrics-server
name: metrics-server
namespace: kube-system
spec:
ports:
- name: https
port: 443
protocol: TCP
targetPort: https
selector:
k8s-app: metrics-server
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
k8s-app: metrics-server
name: metrics-server
namespace: kube-system
spec:
selector:
matchLabels:
k8s-app: metrics-server
strategy:
rollingUpdate:
maxUnavailable: 0
template:
metadata:
labels:
k8s-app: metrics-server
spec:
containers:
- args:
- --cert-dir=/tmp
- --secure-port=4443
- --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
- --kubelet-use-node-status-port
- --metric-resolution=15s
image: k8s.gcr.io/metrics-server/metrics-server:v0.6.0
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /livez
port: https
scheme: HTTPS
periodSeconds: 10
name: metrics-server
ports:
- containerPort: 4443
name: https
protocol: TCP
readinessProbe:
failureThreshold: 3
httpGet:
path: /readyz
port: https
scheme: HTTPS
initialDelaySeconds: 20
periodSeconds: 10
resources:
requests:
cpu: 100m
memory: 200Mi
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 1000
volumeMounts:
- mountPath: /tmp
name: tmp-dir
nodeSelector:
kubernetes.io/os: linux
priorityClassName: system-cluster-critical
serviceAccountName: metrics-server
volumes:
- emptyDir: {}
name: tmp-dir
---
apiVersion: apiregistration.k8s.io/v1
kind: APIService
metadata:
labels:
k8s-app: metrics-server
name: v1beta1.metrics.k8s.io
spec:
group: metrics.k8s.io
groupPriorityMinimum: 100
insecureSkipTLSVerify: true
service:
name: metrics-server
namespace: kube-system
version: v1beta1
versionPriority: 100
修改清单文件
image: registry.cn-hangzhou.aliyuncs.com/google_containers/metrics-server:v0.6.0
sed -i '/args:/a\ - --kubelet-insecure-tls' metrics-server-v0.6.0.yaml
部署清单文件
[root@master1 ~]# kubectl apply -f metrics-server-v0.6.0.yaml
serviceaccount/metrics-server created
clusterrole.rbac.authorization.k8s.io/system:aggregated-metrics-reader created
clusterrole.rbac.authorization.k8s.io/system:metrics-server created
rolebinding.rbac.authorization.k8s.io/metrics-server-auth-reader created
clusterrolebinding.rbac.authorization.k8s.io/metrics-server:system:auth-delegator created
clusterrolebinding.rbac.authorization.k8s.io/system:metrics-server created
service/metrics-server created
deployment.apps/metrics-server created
apiservice.apiregistration.k8s.io/v1beta1.metrics.k8s.io created
验证 metrics-server
检查 nodes 指标获取情况
需要等待 metrics-server 的 pod 就绪之后再等它收集一会再看。
root@master1 ~]# kubectl get po
NAME READY STATUS RESTARTS AGE
myapp-797b896b4b-vcfjk 1/1 Running 0 123m
nginx-779c9459b8-77zpk 1/1 Running 0 24h
[root@master1 ~]# kubectl top nodes
NAME CPU(cores) CPU% MEMORY(bytes) MEMORY%
master1 308m 15% 1255Mi 67%
master2 288m 14% 1214Mi 65%
master3 269m 13% 1167Mi 62%
node1 182m 9% 995Mi 53%
node2 149m 7% 1014Mi 54%
[root@master1 ~]# kubectl top pods
NAME CPU(cores) MEMORY(bytes)
myapp-797b896b4b-vcfjk 0m 2Mi
nginx-779c9459b8-77zpk 0m 2Mi
[root@master1 ~]#
nodes 和 pods 的 CPU 和内存指标显示正常。
dashbord(可选)
因为后面我会托管到kubespare 所以这里dashboard并没装
如果需要可参照https://huangzhongde.cn/istio/Chapter3/Chapter3-3.html
部署 EFK 日志服务
Kubernetes官方推荐的EFK组合，其实对于我们大多数公司来说并不合适，其中一点原因就是这个组合需要应用日志打印到标准输出。
推荐做法：还是使用传统的ELK组合，ELK可以部署在K8S集群之外，日志写入指定存储目录文件中，并采用filebeat方式采集。
从 github 上下载 kubernetes-v1.22.6 包，下载地址
自带的 es 没有持久化功能,如果需要持久化,可以选择 pvc
下载 1.22.1 源码包
wget https://github.com/kubernetes/kubernetes/archive/refs/tags/v1.22.6.tar.gz \
-O kubernetes-1.22.6.tar.gz
tar xf kubernetes-1.22.6.tar.gz
cd kubernetes-1.22.6/cluster/addons/fluentd-elasticsearch
修改清单文件
修改es的镜像地址和拉取策略
cp es-statefulset.yaml{,.ori}
sed -i '/imagePullPolicy/s@Always@IfNotPresent@' es-statefulset.yaml
修改 kibana 配置
cp kibana-deployment.yaml{,.ori}
vim kibana-deployment.yaml
cpu: 100m
env:
- name: ELASTICSEARCH_HOSTS
value: http://elasticsearch-logging:9200
- name: SERVER_NAME
value: kibana-logging
- name: SERVER_BASEPATH
value: "/api/v1/namespaces/logging/services/kibana-logging/proxy"
- name: SERVER_REWRITEBASEPATH
value: "false"
ports:
部署清单文件
[root@master1 fluentd-elasticsearch]# kubectl apply -f .
namespace/logging created
service/elasticsearch-logging created
serviceaccount/elasticsearch-logging created
clusterrole.rbac.authorization.k8s.io/elasticsearch-logging created
clusterrolebinding.rbac.authorization.k8s.io/elasticsearch-logging created
statefulset.apps/elasticsearch-logging created
configmap/fluentd-es-config-v0.2.1 created
serviceaccount/fluentd-es created
clusterrole.rbac.authorization.k8s.io/fluentd-es created
clusterrolebinding.rbac.authorization.k8s.io/fluentd-es created
daemonset.apps/fluentd-es-v3.1.1 created
deployment.apps/kibana-logging created
service/kibana-logging created
[root@master1 fluentd-elasticsearch]#
暴露服务供外网访问
NodePort 方式
kubectl patch -n logging svc kibana-logging -p '{"spec":{"type":"NodePort"}}'
service/kibana-logging patched
LoadBalancer 方式
kubectl patch -n logging svc kibana-logging -p '{"spec":{"type":"LoadBalancer"}}'
service/kibana-logging patched
kubectl get svc kibana-logging -n logging
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kibana-logging LoadBalancer 10.102.253.67 10.10.1.150 5601:32107/TCP 61s
访问 kibana
打开浏览器，输入 http://10.10.1.150:5601 进行日志查询等操作
/etc/localtime到pod中修改 es
cp es-statefulset.yaml{,.ori}
vim es-statefulset.yaml
查看更改
72c72
< imagePullPolicy: IfNotPresent
---
> imagePullPolicy: Always
101,102d100
< - name: timezone
< mountPath: /etc/localtime
113,115d110
< - name: timezone
< hostPath:
< path: /etc/localtime
kubectl apply -f es-statefulset.yaml
修改 kibana
查看修改
diff kibana-deployment.yaml.ori kibana-deployment.yaml
输出如下
37c37
< value: ""
---
> value: /api/v1/namespaces/logging/services/kibana-logging/proxy
56,63d55
< volumeMounts:
< - name: timezone
< mountPath: /etc/localtime
< volumes:
< - name: timezone
< hostPath:
< path: /etc/localtime
部署kibana
kubectl apply -f kibana-deployment.yaml
持久化存储
这里就 Elasticsearch 属于有状态的，需要给它配置 pvc
vim es-statefulset.yaml
修改后如下
56,59d55
< updateStrategy:
< rollingUpdate:
< partition: 0
< type: RollingUpdate
76c72
< imagePullPolicy: IfNotPresent
---
> imagePullPolicy: Always
105,106d100
< - name: timezone
< mountPath: /etc/localtime
115,117c109,110
< - name: timezone
< hostPath:
< path: /etc/localtime
---
> - name: elasticsearch-logging
> emptyDir: {}
127,135d119
< volumeClaimTemplates:
< - metadata:
< name: elasticsearch-logging
< spec:
< accessModes:
< - ReadWriteOnce
< resources:
< requests:
< storage: 1Gi # 生产环境根据实际使用情况配置,做好日志清理策略
重新创建
kubectl delete -f es-statefulset.yaml
kubectl apply -f es-statefulset.yaml
查看 pvc 创建
kubectl get pvc -n logging
输出如下
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
elasticsearch-logging-elasticsearch-logging-0 Bound pvc-e7f5ceef-d1e2-49f4-a859-7ac1b4dbcc04 1Gi
监控
prometheus 仓库地址
prometheus 官方网站
安装方式支持 kube-prometheus 安装和 helm，这里介绍 kube-prometheus 安装
注：使用
kube-prometheus安装的话，前面的prometheus+grafana+metrics-server都不用安装，已经集成在里面
github 仓库地址
安装 kube-prometheus
下载代码
wget https://github.com/prometheus-operator/kube-prometheus/archive/refs/tags/v0.10.0.tar.gz \
-O kube-prometheus_v0.10.0.tar.gz
tar xf kube-prometheus_v0.10.0.tar.gz
cd kube-prometheus-0.10.0
修改 yaml 清单文件
root@master1 kube-prometheus-0.10.0]# cp manifests/grafana-service.yaml manifests/grafana-service.yaml.bak
[root@master1 kube-prometheus-0.10.0]# sed -i '/spec:/a\ type: LoadBalancer' manifests/grafana-service.yaml
[root@master1 kube-prometheus-0.10.0]# cp manifests/prometheus-service.yaml manifests/prometheus-service.yaml.bak
[root@master1 kube-prometheus-0.10.0]# sed -i '/spec:/a\ type: LoadBalancer' manifests/prometheus-service.yaml
[root@master1 kube-prometheus-0.10.0]#
查看修改
vim manifests/grafana-service.yaml
apiVersion: v1
kind: Service
metadata:
labels:
app.kubernetes.io/component: grafana
app.kubernetes.io/name: grafana
app.kubernetes.io/part-of: kube-prometheus
app.kubernetes.io/version: 8.3.3
name: grafana
namespace: monitoring
spec:
type: LoadBalancer #*
ports:
- name: http
port: 3000
targetPort: http
selector:
app.kubernetes.io/component: grafana
app.kubernetes.io/name: grafana
app.kubernetes.io/part-of: kube-prometheus
[root@master1 kube-prometheus-0.10.0]# cat manifests/prometheus-service.yaml
apiVersion: v1
kind: Service
metadata:
labels:
app.kubernetes.io/component: prometheus
app.kubernetes.io/instance: k8s
app.kubernetes.io/name: prometheus
app.kubernetes.io/part-of: kube-prometheus
app.kubernetes.io/version: 2.32.1
name: prometheus-k8s
namespace: monitoring
spec:
type: LoadBalancer
ports:
- name: web
port: 9090
targetPort: web
- name: reloader-web
port: 8080
targetPort: reloader-web
selector:
app.kubernetes.io/component: prometheus
app.kubernetes.io/instance: k8s
app.kubernetes.io/name: prometheus
app.kubernetes.io/part-of: kube-prometheus
sessionAffinity: ClientIP
[root@master1 kube-prometheus-0.10.0]#
修改镜像地址
如果访问不了 gcr.io 执行下面的替换为 dockerhub 的
sed -i '/image:/s@k8s.gcr.io/kube-state-metrics@willdockerhub@' $(grep -l image: manifests/*.yaml)
部署 CRD
[root@master1 kube-prometheus-0.10.0]# kubectl create -f manifests/setup
customresourcedefinition.apiextensions.k8s.io/alertmanagerconfigs.monitoring.coreos.com created
customresourcedefinition.apiextensions.k8s.io/alertmanagers.monitoring.coreos.com created
customresourcedefinition.apiextensions.k8s.io/podmonitors.monitoring.coreos.com created
customresourcedefinition.apiextensions.k8s.io/probes.monitoring.coreos.com created
customresourcedefinition.apiextensions.k8s.io/prometheuses.monitoring.coreos.com created
customresourcedefinition.apiextensions.k8s.io/prometheusrules.monitoring.coreos.com created
customresourcedefinition.apiextensions.k8s.io/servicemonitors.monitoring.coreos.com created
customresourcedefinition.apiextensions.k8s.io/thanosrulers.monitoring.coreos.com created
namespace/monitoring created
[root@master1 kube-prometheus-0.10.0]#
使用 apply 时提示 annotations 过长,使用 create 代替
如果出现请求vip 失败 delete -f 之后重新create 即可
The CustomResourceDefinition "prometheuses.monitoring.coreos.com" is invalid: metadata.annotations: Too long: must have at most 262144 bytes
部署 kube-prometheus
[root@master1 kube-prometheus-0.10.0]# kubectl create -f manifests/setup
customresourcedefinition.apiextensions.k8s.io/alertmanagerconfigs.monitoring.coreos.com created
...
namespace/monitoring created
[root@master1 kube-prometheus-0.10.0]# kubectl apply -f manifests/
...
configmap/grafana-dashboards created
deployment.apps/grafana created
service/grafana created
serviceaccount/grafana created
servicemonitor.monitoring.coreos.com/grafana created
prometheusrule.monitoring.coreos.com/kube-prometheus-rules created
clusterrole.rbac.authorization.k8s.io/kube-state-metrics created
clusterrolebinding.rbac.authorization.k8s.io/kube-state-metrics created
deployment.apps/kube-state-metrics created
...
[root@master1 kube-prometheus-0.10.0]#
检查部署情况
kubectl get po -n monitoring
输出如下
NAME READY STATUS RESTARTS AGE
alertmanager-main-0 2/2 Running 0 3m
alertmanager-main-1 2/2 Running 0 3m
alertmanager-main-2 2/2 Running 0 2m59s
blackbox-exporter-6798fb5bb4-dmjxs 3/3 Running 0 3m27s
grafana-78d8cfccff-snlqn 1/1 Running 0 3m22s
kube-state-metrics-5fcb7d6fcb-d6qdg 3/3 Running 0 3m20s
node-exporter-4qq75 2/2 Running 0 3m18s
node-exporter-c7v7m 2/2 Running 0 3m18s
node-exporter-jzc4t 2/2 Running 0 3m18s
prometheus-adapter-7dc46dd46d-bwsg6 1/1 Running 0 3m10s
prometheus-adapter-7dc46dd46d-mgrvj 1/1 Running 0 3m10s
prometheus-k8s-0 2/2 Running 0 2m57s
prometheus-k8s-1 2/2 Running 0 2m57s
prometheus-operator-7ddc6877d5-znn7h 2/2 Running 0 3m8s
kubectl get svc -n monitoring
输出如下
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
alertmanager-main ClusterIP 10.107.184.145 <none> 9093/TCP,8080/TCP 3m52s
alertmanager-operated ClusterIP None <none> 9093/TCP,9094/TCP,9094/UDP 3m23s
blackbox-exporter ClusterIP 10.98.182.126 <none> 9115/TCP,19115/TCP 3m50s
grafana LoadBalancer 10.111.160.67 10.10.1.151 3000:30991/TCP 3m45s
kube-state-metrics ClusterIP None <none> 8443/TCP,9443/TCP 3m43s
node-exporter ClusterIP None <none> 9100/TCP 3m41s
prometheus-adapter ClusterIP 10.108.64.100 <none> 443/TCP 3m34s
prometheus-k8s LoadBalancer 10.102.246.195 10.10.1.151 9090:30280/TCP,8080:30097/TCP 3m38s
prometheus-operated ClusterIP None <none> 9090/TCP 3m20s
prometheus-operator ClusterIP None <none> 8443/TCP 3m32s
打开浏览器，输入：http://192.168.122.194:3000 访问 grafana
默认用户名和密码为：admin，admin
自带的 dashboard，直接可以使用。
helm
https://huangzhongde.cn/istio/Chapter4/Chapter4-1.html
期待你的
分享
点赞
在看
添加博主
微信号｜yfq940-
原文作者：步步运维步步坑；如有侵权，请联系管理员删除。
上一篇：
K8s 服务网格iStio
下一篇：
k8s部署维护问题汇总
请登录后发表评论
登录
全部评论
