Kubernetes 3 masters on ceontos7

===========사전작업===========

hostnamectl set-hostname k8s-m01

vi /etc/hosts

10.0.2.221 k8s-m01

10.0.2.222 k8s-m02

10.0.2.223 k8s-m03

10.0.2.220 k8s-vip

============커널업데이트==================

mkdir -p /etc/yum.repos.d/bak

cd /etc/yum.repos.d

mv *.repo ./bak

curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo

curl -o /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo

find . -name "*.repo" -exec sed -i 's/gpgcheck=1/gpgcheck=0/g' {} \;

yum -y update

rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org

rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm

yum --enablerepo=elrepo-kernel install -y kernel-ml

grub2-mkconfig -o /boot/grub2/grub.cfg

grub2-set-default 0

reboot

uname -a

======================소프트웨어 설치 준비=========

yum install -y yum-utils device-mapper-persistent-data lvm2

yum-config-manager -y --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

cat <<EOF > /etc/yum.repos.d/kubernetes.repo

[kubernetes]

name=Kubernetes

baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/

enabled=1

gpgcheck=1

repo_gpgcheck=1

gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg

EOF

cd /etc/yum.repos.d/

find . -name "*.repo" -exec sed -i 's/gpgcheck=1/gpgcheck=0/g' {} \;

yum install -y htop tree wget jq git net-tools ntpdate nc

timedatectl set-timezone Asia/Seoul && date && echo 'Asia/Seoul' > /etc/timezone

sed -i 's/#Storage=auto/Storage=auto/g' /etc/systemd/journald.conf && mkdir -p /var/log/journal && systemd-tmpfiles --create --prefix /var/log/journal

systemctl restart systemd-journald.service

ls -al /var/log/journal

echo 'export HISTTIMEFORMAT="%Y-%m-%d %T "' >> ~/.bashrc && source ~/.bashrc

=================도커 설치

yum search docker-ce --showduplicates

yum search docker-compose --showduplicates

yum install docker-ce-20.10.3-3.el7.x86_64 docker-compose-1.18.0-4.el7.noarch

systemctl enable docker && systemctl start docker && systemctl status docker

systemctl restart docker

docker info

===============쿠버네티스 설치

yum search kubeadm kubelet --showduplicates

yum install -y kubeadm-1.20.2-0.x86_64 kubelet-1.20.2-0.x86_64 kubectl-1.20.2-0.x86_64

systemctl enable kubelet && systemctl start kubelet && systemctl status kubelet

==============방화벽 해제

==마스터노드 만

firewall-cmd --zone=public --add-port=6443/tcp --permanent

firewall-cmd --zone=public --add-port=2379-2380/tcp --permanent

firewall-cmd --zone=public --add-port=10250/tcp --permanent

firewall-cmd --zone=public --add-port=10251/tcp --permanent

firewall-cmd --zone=public --add-port=10252/tcp --permanent

firewall-cmd --zone=public --add-port=30000-32767/tcp --permanent

firewall-cmd --add-masquerade --permanent

firewall-cmd --reload

firewall-cmd --list-all --zone=public

==워커노드 만

firewall-cmd --zone=public --add-port=10250/tcp --permanent

firewall-cmd --zone=public --add-port=30000-32767/tcp --permanent

firewall-cmd --add-masquerade --permanent

firewall-cmd --reload

firewall-cmd --list-all --zone=public

iptables -D INPUT -j REJECT --reject-with icmp-host-prohibited

echo '5,15,25,35,45,55 * * * * /usr/sbin/iptables -D INPUT -j REJECT --reject-with icmp-host-prohibited' >> /var/spool/cron/root && crontab -l

========리눅스 시스템 설정

sed -i 's/SELINUX=enforcing/SELINUX=permissive/g' /etc/selinux/config

setenforce 0

getenforce

cat <<EOF > /etc/sysctl.d/k8s.conf

net.bridge.bridge-nf-call-ip6tables = 1

net.bridge.bridge-nf-call-iptables = 1

net.ipv4.ip_forward = 1

EOF

cat /proc/swaps

swapoff -a

cat /proc/swaps

sed -i '/swap/d' /etc/fstab

cat /etc/fstab

=====마스터 노드끼리 신뢰 설정

yum install -y sshpass

=====================================이미지 스냅샵 마스터 02,03 생성

====모든노드에서 서로

ssh k8s-m01

ssh k8s-m02

ssh k8s-m03

===========on k8s-m01

export SSHHOST=k8s-m02

ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa

cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys

sshpass -p 'shinhan!!2' scp ~/.ssh/authorized_keys root@${SSHHOST}:~/.ssh/

sshpass -p 'shinhan!!2' ssh ${SSHHOST}

===========on k8s-m02

export SSHHOST=k8s-m03

ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa

cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys

sshpass -p 'shinhan!!2' scp ~/.ssh/authorized_keys root@${SSHHOST}:~/.ssh/

sshpass -p 'shinhan!!2' ssh ${SSHHOST}

===========on k8s-m03

export SSHHOST=k8s-m01

ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa

cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys

sshpass -p 'shinhan!!2' scp ~/.ssh/authorized_keys root@${SSHHOST}:~/.ssh/

sshpass -p 'shinhan!!2' ssh ${SSHHOST}

===========on k8s-m01

scp ~/.ssh/authorized_keys k8s-m01:/root/.ssh/

scp ~/.ssh/authorized_keys k8s-m02:/root/.ssh/

scp ~/.ssh/authorized_keys k8s-m03:/root/.ssh/

=============모든 마스터 노드

ssh k8s-m01 "hostname && pwd"

ssh k8s-m02 "hostname && pwd"

ssh k8s-m03 "hostname && pwd"

===============마스터

kubeadm config images list --kubernetes-version=v1.20.2

docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-apiserver:v1.20.2

docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-controller-manager:v1.20.2

docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-scheduler:v1.20.2

docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-proxy:v1.20.2

docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.2

docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/etcd:3.4.13-0

docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/coredns:1.7.0

docker pull quay.io/tigera/operator:v1.13.5

docker pull calico/cni:v3.17.2

docker pull calico/kube-controllers:v3.17.2

docker pull calico/node:v3.17.2

docker pull calico/pod2daemon-flexvol:v3.17.2

docker pull calico/typha:v3.17.2

docker pull osixia/keepalived:2.0.20

docker pull nginx:1.19.7-alpine

docker pull k8s.gcr.io/metrics-server/metrics-server:v0.4.2

docker pull kubernetesui/dashboard:v2.2.0

docker pull kubernetesui/metrics-scraper:v1.0.6

=========마스터 01

git clone https://github.com/cookeem/kubeadm-ha.git

cd kubeadm-ha

cd binary

tar zxvf helm-v2.17.0-linux-amd64.tar.gz

mv linux-amd64/helm /usr/bin/

rm -rf linux-amd64

helm --help

cd kubeadm-ha

vi k8s-install-info.yaml

mkdir -p output

helm template k8s-install --output-dir output -f k8s-install-info.yaml

cd output/k8s-install/templates/

sed -i '1,2d' create-config.sh

sh create-config.sh

=========모든 마스터 노드

docker ps

=========마스터 01에서만

cd /root/kubeadmin/output/k8s-install/templates/

kubeadm init --config=kubeadm-config.yaml --upload-certs

=============토큰 저장해 놓기

Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

mkdir -p $HOME/.kube

sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config

sudo chown $(id -u):$(id -g) $HOME/.kube/config

Alternatively, if you are the root user, you can run:

export KUBECONFIG=/etc/kubernetes/admin.conf

You should now deploy a pod network to the cluster.

Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:

https://kubernetes.io/docs/concepts/cluster-administration/addons/

You can now join any number of the control-plane node running the following command on each as root:

kubeadm join 10.0.2.220:16443 --token m909kk.56783larul8usrzd \

--discovery-token-ca-cert-hash sha256:b59fd52d91ddc4e969ee14490e0a5fc06f9291f8383acbcd58f638ef56e92d85 \

--control-plane --certificate-key d03dae026dd9b95f658c7eddfb85ce75efb77393719d11d2ab4629355bd0e595

Please note that the certificate-key gives access to cluster sensitive data, keep it secret!

As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use

"kubeadm init phase upload-certs --upload-certs" to reload certs afterward.

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 10.0.2.220:16443 --token m909kk.56783larul8usrzd \

--discovery-token-ca-cert-hash sha256:b59fd52d91ddc4e969ee14490e0a5fc06f9291f8383acbcd58f638ef56e92d85

=======모든 마스터 노드

cat <<EOF >> ~/.bashrc

export KUBECONFIG=/etc/kubernetes/admin.conf

EOF

source ~/.bashrc

============마스터 01

kubectl get pods -A

kubectl apply -f calico-v3.17.2/tigera-operator.yaml

sleep 1

kubectl apply -f calico-v3.17.2/custom-resources.yaml

kubectl get pods -A

============마스터 02 조인

kubeadm join 10.0.2.220:16443 --token m909kk.56783larul8usrzd \

--discovery-token-ca-cert-hash sha256:b59fd52d91ddc4e969ee14490e0a5fc06f9291f8383acbcd58f638ef56e92d85 \

--control-plane --certificate-key d03dae026dd9b95f658c7eddfb85ce75efb77393719d11d2ab4629355bd0e595

============마스터 03 조인

kubeadm join 10.0.2.220:16443 --token m909kk.56783larul8usrzd \

--discovery-token-ca-cert-hash sha256:b59fd52d91ddc4e969ee14490e0a5fc06f9291f8383acbcd58f638ef56e92d85 \

--control-plane --certificate-key d03dae026dd9b95f658c7eddfb85ce75efb77393719d11d2ab4629355bd0e595

============마스터 01 조회

kubectl get pods -A

============모든 노드

kubectl get nodes