【k8s篇】搭建k8环境
环境部署
hostnamectl set-hostname k8s-master
hostnamectl set-hostname k8s-node1
hostnamectl set-hostname k8s-node2
# 将 SELinux 设置为 permissive 模式（相当于将其禁用）
sudo setenforce 0
sudo sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config
#关闭swap
swapoff -a
sed -ri 's/.*swap.*/#&/' /etc/fstab
#允许 iptables 检查桥接流量
cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
br_netfilter
EOF
cat > k8s.conf <<EOF
#开启网桥模式
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
#开启转发
net.ipv4.ip_forward = 1
##关闭ipv6
net.ipv6.conf.all.disable_ipv6=1
EOF
cp k8s.conf /etc/sysctl.d/k8s.conf
sysctl -p /etc/sysctl.d/k8s.conf
# 设置系统时区为 中国/上海
timedatectl set-timezone Asia/Shanghai
# 将当前的UTC时间写入硬件时钟
timedatectl set-local-rtc 0
# 重启依赖于系统时间的服务
systemctl restart rsyslog
systemctl restart crond
# ipvs前置条件准备
ipvs 前置准备
# step1
modprobe br_netfilter
# step2
cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF
# step3
chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack_ipv4
安装kubelet、kubeadm、kubectl
cat <<EOF | sudo tee /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
exclude=kubelet kubeadm kubectl
EOF
sudo yum install -y kubelet-1.20.9 kubeadm-1.20.9 kubectl-1.20.9 --disableexcludes=kubernetes
sudo systemctl enable --now kubelet
下载各个机器需要的镜像
sudo tee ./images.sh <<-'EOF'
#!/bin/bash
images=(
kube-apiserver:v1.20.9
kube-proxy:v1.20.9
kube-controller-manager:v1.20.9
kube-scheduler:v1.20.9
coredns:1.7.0
etcd:3.4.13-0
pause:3.2
)
for imageName in ${images[@]} ; do
docker pull registry.cn-hangzhou.aliyuncs.com/lfy_k8s_images/$imageName
done
EOF
chmod +x ./images.sh && ./images.sh
k8s 部署文档
1. 使用kubeadm 部署
kubeadm init --config=kubeadm-config.yaml
2. 安装之后命令
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
export KUBECONFIG=/etc/kubernetes/admin.conf
// node 节点加入k8s集群
kubeadm join 8.136.193.241:6443 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:c5db44b70324412a828c56da22561693c857cdf773b5ebdfdda829c23f3b61e8
// token 有效期24h 需要执行以下命令
kubeadm token create --print-join-command
// 获取加入集群token
kubeadm join 8.136.193.241:6443 --token fvnsnp.pl6s8z7kmch1rd1j --discovery-token-ca-cert-hash sha256:c9a0970b584c08e4e2796f5aee2468bdc104b263d3794f2658df296dd493fe7d
3.卸载集群命令
kubeadm reset -f
iptables -F && iptables -t nat -F && iptables -t mangle -F && iptables -X
rm -rf /etc/cni/net.d
rm -rf $HOME/.kube/config
ipvsadm --clear
4.网络配置
// 每个节点添加hosts
cat >> /etc/hosts <<EOF
8.136.193.241 k8s-master
139.198.187.102 k8s-node1
139.198.154.81 k8s-node2
EOF
// 添加虚拟网卡
cat > /etc/sysconfig/network-scripts/ifcfg-eth0:1 <<EOF
BOOTPROTO=static
DEVICE=eth0:1
IPADDR=8.136.193.241
PREFIX=32
TYPE=Ethernet
USERCTL=no
ONBOOT=yes
EOF
cat > /etc/sysconfig/network-scripts/ifcfg-eth0:1 <<EOF
BOOTPROTO=static
DEVICE=eth0:1
IPADDR=139.198.187.102
PREFIX=32
TYPE=Ethernet
USERCTL=no
ONBOOT=yes
EOF
cat > /etc/sysconfig/network-scripts/ifcfg-eth0:1 <<EOF
BOOTPROTO=static
DEVICE=eth0:1
IPADDR=139.198.154.81
PREFIX=32
TYPE=Ethernet
USERCTL=no
ONBOOT=yes
EOF
// step2 如果是centos8，需要重启
systemctl restart network
nmcli c reload eth0
nmcli c up eth0
// step3 查看新建的IP是否进去
ip addr
修改kubelet启动参数（重点，所有节点都要操作）
# 此文件安装kubeadm后就存在了
vim /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf
# 注意，这步很重要，如果不做，节点仍然会使用内网IP注册进集群
# 在末尾添加参数 --node-ip=公网IP
# Note: This dropin only works with kubeadm and kubelet v1.11+
[Service]
Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf"
Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml"
# This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically
EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
# This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use
# the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file.
EnvironmentFile=-/etc/sysconfig/kubelet
ExecStart=
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS
--node-ip=139.198.187.102
--node-ip=139.198.154.81
--node-ip=8.136.193.241
5. 添加 iptables
--- master
iptables -t nat -A OUTPUT -d 10.130.41.20 -j DNAT --to-destination 139.198.187.102
iptables -t nat -A OUTPUT -d 10.130.59.55 -j DNAT --to-destination 139.198.154.81
iptables -t nat -A OUTPUT -d 8.136.193.241 -j DNAT --to-destination 172.20.126.54
--- node1
iptables -t nat -A OUTPUT -d 10.130.59.55 -j DNAT --to-destination 139.198.154.81
iptables -t nat -A OUTPUT -d 172.20.126.54 -j DNAT --to-destination 8.136.193.241
iptables -t nat -A OUTPUT -d 139.198.187.102 -j DNAT --to-destination 10.130.41.20
---node2
iptables -t nat -A OUTPUT -d 139.198.154.81 -j DNAT --to-destination 10.130.59.55
iptables -t nat -A OUTPUT -d 172.20.126.54 -j DNAT --to-destination 8.136.193.241
iptables -t nat -A OUTPUT -d 10.130.41.20 -j DNAT --to-destination 139.198.187.102
6.修改kube-apiserver参数（主节点）
# 修改两个信息，添加--bind-address和修改--advertise-address
vim /etc/kubernetes/manifests/kube-apiserver.yaml
apiVersion: v1
kind: Pod
metadata:
annotations:
kubeadm.kubernetes.io/kube-apiserver.advertise-address.endpoint: 47.74.22.13:6443
creationTimestamp: null
labels:
component: kube-apiserver
tier: control-plane
name: kube-apiserver
namespace: kube-system
spec:
containers:
- command:
- kube-apiserver
- --advertise-address=47.74.22.13 #修改为公网IP
- --bind-address=0.0.0.0 #添加此参数
- --allow-privileged=true
- --authorization-mode=Node,RBAC
- --client-ca-file=/etc/kubernetes/pki/ca.crt
- --enable-admission-plugins=NodeRestriction
- --enable-bootstrap-token-auth=true
- --etcd-cafile=/etc/kubernetes/pki/etcd/ca.crt
- --etcd-certfile=/etc/kubernetes/pki/apiserver-etcd-client.crt
- --etcd-keyfile=/etc/kubernetes/pki/apiserver-etcd-client.key
- --etcd-servers=https://127.0.0.1:2379
- --insecure-port=0
- --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt
- --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key
- --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
- --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt
- --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key
- --requestheader-allowed-names=front-proxy-client
- --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt
- --requestheader-extra-headers-prefix=X-Remote-Extra-
- --requestheader-group-headers=X-Remote-Group
- --requestheader-username-headers=X-Remote-User
- --secure-port=6443
- --service-account-key-file=/etc/kubernetes/pki/sa.pub
- --service-cluster-ip-range=10.96.0.0/12
- --tls-cert-file=/etc/kubernetes/pki/apiserver.crt
- --tls-private-key-file=/etc/kubernetes/pki/apiserver.key
image: k8s.gcr.io/kube-apiserver:v1.18.0
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 8
httpGet:
host: 175.24.19.12
path: /healthz
port: 6443
scheme: HTTPS
initialDelaySeconds: 15
timeoutSeconds: 15
name: kube-apiserver
resources:
requests:
cpu: 250m
volumeMounts:
- mountPath: /etc/ssl/certs
name: ca-certs
readOnly: true
- mountPath: /etc/pki
name: etc-pki
readOnly: true
- mountPath: /etc/kubernetes/pki
name: k8s-certs
readOnly: true
hostNetwork: true
priorityClassName: system-cluster-critical
volumes:
- hostPath:
path: /etc/ssl/certs
type: DirectoryOrCreate
name: ca-certs
- hostPath:
path: /etc/pki
type: DirectoryOrCreate
name: etc-pki
- hostPath:
path: /etc/kubernetes/pki
type: DirectoryOrCreate
name: k8s-certs
status: {}
7. 修改flannel文件并安装（主节点）
# 下载flannel的yaml配置文件
# 共修改两个地方，一个是args下，添加
args:
- --public-ip=$(PUBLIC_IP) # 添加此参数，申明公网IP
- --iface=eth0 # 添加此参数，绑定网卡
# 然后是env下
env:
- name: PUBLIC_IP #添加环境变量
valueFrom:
fieldRef:
fieldPath: status.podIP
...略
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: kube-flannel-ds-amd64
namespace: kube-system
labels:
tier: node
app: flannel
spec:
selector:
matchLabels:
app: flannel
template:
metadata:
labels:
tier: node
app: flannel
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: beta.kubernetes.io/os
operator: In
values:
- linux
- key: beta.kubernetes.io/arch
operator: In
values:
- amd64
hostNetwork: true
tolerations:
- operator: Exists
effect: NoSchedule
serviceAccountName: flannel
initContainers:
- name: install-cni
image: quay.io/coreos/flannel:v0.11.0-amd64
command:
- cp
args:
- -f
- /etc/kube-flannel/cni-conf.json
- /etc/cni/net.d/10-flannel.conflist
volumeMounts:
- name: cni
mountPath: /etc/cni/net.d
- name: flannel-cfg
mountPath: /etc/kube-flannel/
containers:
- name: kube-flannel
image: quay.io/coreos/flannel:v0.11.0-amd64
command:
- /opt/bin/flanneld
args:
- --ip-masq
- --public-ip=$(PUBLIC_IP) # 添加此参数，申明公网IP
- --iface=eth0 # 添加此参数，绑定网卡
- --kube-subnet-mgr
resources:
requests:
cpu: "100m"
memory: "50Mi"
limits:
cpu: "100m"
memory: "50Mi"
securityContext:
privileged: false
capabilities:
add: ["NET_ADMIN"]
env:
- name: PUBLIC_IP #添加环境变量
valueFrom: #
fieldRef: #
fieldPath: status.podIP #
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
... 略
8 手动开启配置，开启ipvs转发模式（主节点）
# 前面都成功了，但是有时候默认并不会启用`IPVS`模式，那就手动修改一下，只修改一处
# 修改后，如果没有及时生效，请删除kube-proxy，会自动重新创建，然后使用ipvsadm -Ln命令，查看是否生效
# ipvsadm没有安装的，使用yum install ipvsadm安装
kubectl edit configmaps -n kube-system kube-proxy
---
apiVersion: v1
data:
config.conf: |-
apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: 0.0.0.0
clientConnection:
acceptContentTypes: ""
burst: 0
contentType: ""
kubeconfig: /var/lib/kube-proxy/kubeconfig.conf
qps: 0
clusterCIDR: 10.244.0.0/16
configSyncPeriod: 0s
conntrack:
maxPerCore: null
min: null
tcpCloseWaitTimeout: null
tcpEstablishedTimeout: null
detectLocalMode: ""
enableProfiling: false
healthzBindAddress: ""
hostnameOverride: ""
iptables:
masqueradeAll: false
masqueradeBit: null
minSyncPeriod: 0s
syncPeriod: 0s
ipvs:
excludeCIDRs: null
minSyncPeriod: 0s
scheduler: ""
strictARP: false
syncPeriod: 0s
tcpFinTimeout: 0s
tcpTimeout: 0s
udpTimeout: 0s
kind: KubeProxyConfiguration
metricsBindAddress: ""
mode: "ipvs" # 如果为空，请填入`ipvs`
nodePortAddresses: null
oomScoreAdj: null
portRange: ""
showHiddenMetricsForVersion: ""
udpIdleTimeout: 0s
winkernel:
enableDSR: false
networkName: ""
...
9. 安装 kubesphere
kubectl describe node k8s-master | grep Taint
// 取消master 污点不调度
kubectl taint nodes k8s-master node-role.kubernetes.io/master:NoSchedule-
kubectl apply -f openebs-operator.yml
kubectl apply -f https://openebs.github.io/charts/openebs-operator-1.5.0.yaml
kubectl patch storageclass openebs-hostpath -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'
kubectl apply -f kubesphere-installer.yml
kubectl apply -f cluster-configuration.yml
10.helm
helm install minio
–namespace kubesphere-system --create-namespace
–set accessKey=admin,secretKey=Qazwsx123456@
–set mode=distributed
–set replicas=1
–set service.type=NodePort
–set persistence.size=10Gi
minio/minio
- 感谢你赐予我前进的力量
-
微信 
支付宝
赞赏者名单
因为你们的支持让我意识到写文章的价值🙏
本文是原创文章，采用 CC BY-NC-ND 4.0 协议，完整转载请注明来自 dreamer
.png)
评论
匿名评论
隐私政策
你无需删除空行，直接评论以获取最佳展示效果
