# 在192.168.182.110执行
hostnamectl set-hostname  local-168-182-110

# 在192.168.182.111执行
hostnamectl set-hostname  local-168-182-110

# 在192.168.182.112执行
hostnamectl set-hostname  local-168-182-110

# 在192.168.182.113执行
hostnamectl set-hostname  local-168-182-110
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.

cat >> /etc/hosts<<EOF
192.168.182.110 local-168-182-110
192.168.182.111 local-168-182-111
192.168.182.112 local-168-182-112
192.168.182.113 local-168-182-113
# VIP
cluster-endpoint 192.168.182.211
EOF
1.
2.
3.
4.
5.
6.
7.
8.

# 直接一直回车就行
ssh-keygen

ssh-copy-id -i ~/.ssh/id_rsa.pub root@local-168-182-110
ssh-copy-id -i ~/.ssh/id_rsa.pub root@local-168-182-111
ssh-copy-id -i ~/.ssh/id_rsa.pub root@local-168-182-112
ssh-copy-id -i ~/.ssh/id_rsa.pub root@local-168-182-113
1.
2.
3.
4.
5.
6.
7.

yum install chrony -y
systemctl start chronyd
systemctl enable chronyd
chronyc sources
1.
2.
3.
4.

systemctl stop firewalld
systemctl disable firewalld
1.
2.

# 临时关闭；关闭swap主要是为了性能考虑
swapoff -a
# 可以通过这个命令查看swap是否关闭了
free
# 永久关闭        
sed -ri 's/.*swap.*/#&/' /etc/fstab
1.
2.
3.
4.
5.
6.

# 临时关闭
setenforce 0
# 永久禁用
sed -i 's/^SELINUX=enforcing$/SELINUX=disabled/' /etc/selinux/config
1.
2.
3.
4.

sudo modprobe br_netfilter
lsmod | grep br_netfilter
1.
2.

cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF

sudo modprobe overlay
sudo modprobe br_netfilter

# 设置所需的 sysctl 参数，参数在重新启动后保持不变
cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
EOF

# 应用 sysctl 参数而不重新启动
sudo sysctl --system
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.

# 下载
wget https://download.docker.com/linux/static/stable/x86_64/docker-20.10.20.tgz
# 解压
tar -zxvf docker-20.10.20.tgz
1.
2.
3.
4.

cp -p docker/* /usr/bin
1.

cat> /usr/lib/systemd/system/docker.service <<EOF
[Unit]
Descriptinotallow=Docker Application Container Engine
Documentatinotallow=http://docs.docker.com
After=network.target docker.socket
[Service]
Type=notify
Envirnotallow=-/run/flannel/docker
WorkingDirectory=/usr/local/bin
ExecStart=/usr/bin/dockerd \
                -H tcp://0.0.0.0:4243 \
                -H unix:///var/run/docker.sock \
                --selinux-enabled=false \
                --log-opt max-size=1g
ExecReload=/bin/kill -s HUP $MAINPID
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
# Uncomment TasksMax if your systemd version supports it.
# Only systemd 226 and above support this version.
#TasksMax=infinity
TimeoutStartSec=0
# set delegate yes so that systemd does not reset the cgroups of docker containers
Delegate=yes
# kill only the docker process, not all processes in the cgroup
KillMode=process
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOF
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.

systemctl daemon-reload

# 设置为开机自启并现在立刻启动服务 --now：立刻启动服务
systemctl enable --now docker

# 查看容器状态
systemctl status docker
1.
2.
3.
4.
5.
6.
7.

cat >/etc/docker/daemon.json<<EOF
{
   "registry-mirrors": ["http://hub-mirror.c.163.com"],
    "exec-opts": ["native.cgroupdriver=systemd"]
}
EOF

systemctl restart docker
1.
2.
3.
4.
5.
6.
7.
8.

wget https://dl.k8s.io/v1.23.6/kubernetes-server-linux-amd64.tar.gz

tar -xf kubernetes-server-linux-amd64.tar.gz

cd kubernetes/server/bin
cp -ar kubelet kubeadm kubectl /usr/bin/
1.
2.
3.
4.
5.
6.

cat >/usr/lib/systemd/system/kubelet.service <<EOF
[Unit]
Descriptinotallow=kubelet: The Kubernetes Node Agent
Documentatinotallow=https://kubernetes.io/docs/

[Service]
ExecStart=/usr/bin/kubelet
Restart=always
StartLimitInterval=0
RestartSec=10

[Install]
WantedBy=multi-user.target
EOF

#注意ExecStart内容换行
mkdir -p /usr/lib/systemd/system/kubelet.service.d/
cat >/usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf <<EOF
[Service]
Envirnotallow="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubecnotallow=/etc/kubernetes/bootstrap-kubelet.conf --kubecnotallow=/etc/kubernetes/kubelet.conf"
Envirnotallow="KUBELET_CONFIG_ARGS=--cnotallow=/var/lib/kubelet/config.yaml"
Envirnotallow=-/var/lib/kubelet/kubeadm-flags.env
Envirnotallow=-/etc/sysconfig/kubelet
ExecStart=
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS
EOF
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.

sh image_load.sh
1.

# 设置为开机自启并现在立刻启动服务 --now：立刻启动服务
systemctl enable --now kubelet
1.
2.

wget https://www.keepalived.org/software/keepalived-2.2.7.tar.gz --no-check-certificate

tar -zxvf keepalived-2.2.7.tar.gz

# 安装openssl（编译keepalived需要）
yum -y install openssl openssl-devel

# 编译 keepalived
yum install gcc -y

cd keepalived-2.2.7 && ./configure --prefix=/usr/local/keepalived  --syscnotallow=/etc
make && make install

# 配置环境变量
cp /usr/local/keepalived/sbin/keepalived /usr/sbin/keepalived
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.

cat > /etc/keepalived/keepalived.conf<<EOF
! Configuration File for keepalived 
global_defs {
    #每个keepalived取个不同名称
    router_id 56
}
vrrp_instance VI_1 {
    # MASTER为主实例,BACKUP 为从实例
    state MASTER
    # 网卡名称
    interface ens33
    # 主备这里要配置为同样的 
    virtual_router_id 68
    # 优先级,主要高于备. 一般主配置为100  备配置为80
    priority 100
    advert_int 1
    # 主动抢占,主备都开启的话,服务器抢占过去后,要等这个服务器keepalived停掉才会漂移到另一台
    nopreempt
    authentication {
        # 主备必须配置成同样的
        auth_type PASS
         # 主备必须配置成同样的
        auth_pass 1111
    }
    virtual_ipaddress {
      # vip,主备必须配置一样
      192.168.182.211
    }
}
EOF
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.

cat > /etc/keepalived/keepalived.conf<<EOF
! Configuration File for keepalived 
global_defs {
    #每个keepalived取个不同名称
    router_id 56
}
vrrp_instance VI_1 {
    # MASTER为主实例,BACKUP 为从实例
    state BACKUP
    # 网卡名称
    interface ens33
    # 主备这里要配置为同样的 
    virtual_router_id 68
    # 优先级,主要高于备. 一般主配置为100  备配置为80
    priority 80
    advert_int 1
    # 主动抢占,主备都开启的话,服务器抢占过去后,要等这个服务器keepalived停掉才会漂移到另一台
    nopreempt
    authentication {
        # 主备必须配置成同样的
        auth_type PASS
         # 主备必须配置成同样的
        auth_pass 1111
    }
    virtual_ipaddress {
      # vip,主备必须配置一样
      192.168.182.211
    }
}
EOF
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.

cat > /usr/lib/systemd/system/keepalived.service <<EOF
[Unit]
Descriptinotallow=LVS and VRRP High Availability Monitor
After=network-online.target syslog.target
Wants=network-online.target
Documentatinotallow=man:keepalived(8)
Documentatinotallow=man:keepalived.conf(5)
Documentatinotallow=man:genhash(1)
Documentatinotallow=https://keepalived.org

[Service]
Type=forking
PIDFile=/run/keepalived.pid
KillMode=process
Envirnotallow=-/usr/local/keepalived/sysconfig/keepalived
ExecStart=/usr/local/keepalived/sbin/keepalived  $KEEPALIVED_OPTIONS
ExecReload=/bin/kill -HUP $MAINPID

[Install]
WantedBy=multi-user.target

EOF
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.

systemctl daemon-reload
systemctl enable --now keepalived
1.
2.

kubeadm config print init-defaults > kubeadm-config.yaml
1.

kubeadm init --cnotallow=kubeadm-config.yaml --experimental-upload-certs | tee kubeadm-init.log
1.

kubeadm init \
  --apiserver-advertise-address=192.168.182.211 \
  --image-repository registry.aliyuncs.com/google_containers \
  --control-plane-endpoint=cluster-endpoint \
  --kubernetes-version v1.23.6 \
  --service-cidr=10.1.0.0/16 \
  --pod-network-cidr=10.244.0.0/16 \
  --v=5
1.
2.
3.
4.
5.
6.
7.
8.

# wget https://raw.githubusercontent.com/flannel-io/flannel/v0.20.2/Documentation/kube-flannel.yml
kubeclt apply -f kube-flannel.yml
1.
2.

# 在需要添加的master节点上执行

CERT_KEY=`ssh cluster-endpoint "kubeadm init phase upload-certs --upload-certs|tail -1"`

join_str=`ssh cluster-endpoint kubeadm token create --print-join-command`

echo $join_str " --control-plane --certificate-key $CERT_KEY --v=5"

# 拿到上面打印的命令在需要添加的节点上执行

# --control-plane 标志通知 kubeadm join 创建一个新的控制平面。加入master必须加这个标记
# --certificate-key ... 将导致从集群中的 kubeadm-certs Secret 下载控制平面证书并使用给定的密钥进行解密。这里的值就是上面这个命令（kubeadm init phase upload-certs --upload-certs）打印出的key。

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

# 去掉master污点
# kubectl taint nodes `hostname` node-role.kubernetes.io/master:NoSchedule- 2>/dev/null
# kubectl taint nodes `hostname` node.kubernetes.io/not-ready:NoSchedule- 2>/dev/null
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.

CERT_KEY=`ssh cluster-endpoint "kubeadm init phase upload-certs --upload-certs|tail -1"`

join_str=`ssh cluster-endpoint kubeadm token create --print-join-command`

echo $join_str " --certificate-key $CERT_KEY --v=5"
1.
2.
3.
4.
5.

modprobe -- ip_vs
modprobe -- ip_vs_sh
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
# 验证开启了ipvs
lsmod |grep ip_vs
1.
2.
3.
4.
5.
6.

yum install ipvsadm -y
1.

wget https://kernel.org/pub/linux/utils/kernel/ipvsadm/ipvsadm-1.31.tar.gz --no-check-certificate
tar -xzvf ipvsadm-1.31.tar.gz && cd ipvsadm-1.31

# 先安装依赖包
yum install -y libnl libnl-devel gcc
# 编译安装
make && make install

# 查看版本号
ipvsadm -version
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.

kubectl edit  configmap -n kube-system  kube-proxy
1.

# 先查看
kubectl get pod -n kube-system | grep kube-proxy
# 再delete让它自拉起
kubectl get pod -n kube-system | grep kube-proxy |awk '{system("kubectl delete pod "$1" -n kube-system")}'
# 再查看
kubectl get pod -n kube-system | grep kube-proxy
1.
2.
3.
4.
5.
6.

# wget https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.2.0/deploy/static/provider/cloud/deploy.yaml -O /tmp/deploy.yaml

kubectl apply -f /tmp/deploy.yaml
1.
2.
3.

# 下载包
wget https://get.helm.sh/helm-v3.7.1-linux-amd64.tar.gz -O /tmp/helm-v3.7.1-linux-amd64.tar.gz
# 解压压缩包
tar -xf /tmp/helm-v3.7.1-linux-amd64.tar.gz -C /root/
# 制作软连接
ln -s /root/linux-amd64/helm /usr/local/bin/helm
1.
2.
3.
4.
5.
6.

# 添加helm仓库源
helm repo add nfs-subdir-external-provisioner https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner/
1.
2.

#### 安装nfs
yum -y install  nfs-utils rpcbind

# 服务端
mkdir -p /opt/nfsdata
# 授权共享目录
chmod 666 /opt/nfsdata
cat > /etc/exports<<EOF
/opt/nfsdata *(rw,no_root_squash,no_all_squash,sync)
EOF
# 配置生效
exportfs -r

systemctl enable --now rpcbind
systemctl enable --now nfs-server
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.

yum -y install rpcbind
systemctl enable --now rpcbind
1.
2.

### helm安装nfs provisioner，安装包在提供的资源里也有，可以不用再去外网下载
helm install nfs-subdir-external-provisioner ./nfs-subdir-external-provisioner-4.0.17.tgz \
  --namespace=nfs-provisioner \
  --create-namespace \
  --set image.repository=willdockerhub/nfs-subdir-external-provisioner \
  --set image.tag=v4.0.2 \
  --set replicaCount=2 \
  --set storageClass.name=nfs-client \
  --set storageClass.defaultClass=true \
  --set nfs.server=cluster-endpoint \
  --set nfs.path=/opt/nfsdata

# 查看
kubectl get pods,deploy,sc -n nfs-provisioner
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.

kubectl apply -f dashboard.yaml
# 查看
kubectl get pods,svc -n kubernetes-dashboard
1.
2.
3.

cat >ServiceAccount.yaml<<EOF
apiVersion: v1
kind: ServiceAccount
metadata:
  name: admin-user
  namespace: kubernetes-dashboard
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: admin-user
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: admin-user
  namespace: kubernetes-dashboard
EOF
kubectl apply -f ServiceAccount.yaml
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.

kubectl -n kubernetes-dashboard create token admin-user
1.

mkdir stl && cd stl
# 生成 CA 证书私钥
openssl genrsa -out ca.key 4096
# 生成 CA 证书
openssl req -x509 -new -nodes -sha512 -days 3650 \
 -subj "/C=CN/ST=Guangdong/L=Shenzhen/O=harbor/OU=harbor/CN=myharbor.com" \
 -key ca.key \
 -out ca.crt
# 创建域名证书，生成私钥
openssl genrsa -out myharbor.com.key 4096
# 生成证书签名请求 CSR
openssl req -sha512 -new \
    -subj "/C=CN/ST=Guangdong/L=Shenzhen/O=harbor/OU=harbor/CN=myharbor.com" \
    -key myharbor.com.key \
    -out myharbor.com.csr
# 生成 x509 v3 扩展
cat > v3.ext <<-EOF
authorityKeyIdentifier=keyid,issuer
basicCnotallow=CA:FALSE
keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment
extendedKeyUsage = serverAuth
subjectAltName = @alt_names

[alt_names]
DNS.1=myharbor.com
DNS.2=*.myharbor.com
DNS.3=hostname
EOF
#创建 Harbor 访问证书
openssl x509 -req -sha512 -days 3650 \
    -extfile v3.ext \
    -CA ca.crt -CAkey ca.key -CAcreateserial \
    -in myharbor.com.csr \
    -out myharbor.com.crt
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.
33.
34.

kubectl create secret tls myharbor.com --key myharbor.com.key --cert myharbor.com.crt -n harbor
kubectl get secret myharbor.com -n harbor
1.
2.

helm repo add harbor https://helm.goharbor.io
1.

# 下载安装包
helm pull harbor/harbor
# harbor-1.11.1.tgz 安装包在提供的资源包里是有的，可以不用再去外网下载的。
helm install myharbor ./harbor-1.11.1.tgz \
  --namespace=harbor --create-namespace \
  --set expose.ingress.hosts.core=myharbor.com \
  --set expose.ingress.hosts.notary=notary.myharbor.com \
  --set-string expose.ingress.annotations.'nginx\.org/client-max-body-size'="1024m" \
  --set expose.tls.secretName=myharbor.com \
  --set persistence.persistentVolumeClaim.registry.storageClass=nfs-client \
  --set persistence.persistentVolumeClaim.jobservice.storageClass=nfs-client \
  --set persistence.persistentVolumeClaim.database.storageClass=nfs-client \
  --set persistence.persistentVolumeClaim.redis.storageClass=nfs-client \
  --set persistence.persistentVolumeClaim.trivy.storageClass=nfs-client \
  --set persistence.persistentVolumeClaim.chartmuseum.storageClass=nfs-client \
  --set persistence.enabled=true \
  --set externalURL=https://myharbor.com \
  --set harborAdminPassword=Harbor12345
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.

kubectl get pods,ingress -n harbor
1.

192.168.182.110 myharbor.com
1.