搭建CDH离线仓库

服务器硬件配置及环境

项目 说明
CPU 1核
内存 1GB
硬盘 40GB
系统 CentOS 7.5
时间同步服务 chrony
WEB服务 nginx

搭建过程

  1. 使用root登录repo-1服务器
  2. 整理以下脚本中的变量内容(包括环境信息和依赖包下载脚本部分), 并保存为repo.sh文件在repo服务器上执行.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
#!/bin/bash

# 初始化环境信息
repo_1_hostname='repo-1.cdh.lab'
repo_1_ip='192.168.60.110'
cm_1_ip='192.168.60.111'
master_1_ip='192.168.60.112'
work_1_ip='192.168.60.113'
work_2_ip='192.168.60.114'
allow_chrony_nets='192.168.60.0/24'

hostnamectl set-hostname ${repo_1_hostname}

systemctl stop firewalld
systemctl disable firewalld
setenforce 0
sed -i "s/SELINUX=.*/SELINUX=disabled/g" /etc/selinux/config

cat >>/etc/sysconfig/network <<EOF
HOSTNAME=$(hostname)
EOF

# 因避免搭建dns服务器, 所以使用hosts添加本集群里的服务器对应的域名
cat >>/etc/hosts <<EOF
${repo_1_ip} repo-1.cdh.lab repo-1
${cm_1_ip} cm-1.cdh.lab cm-1
${master_1_ip} master-1.cdh.lab master-1
${work_1_ip} work-1.cdh.lab work-1
${work_2_ip} work-2.cdh.lab work-2
EOF

yum install yum-utils createrepo chrony -y

# 开启chrony服务用于同步集群内时间
# 如果此服务器无法连接外网, 请在此处手动设置好服务器的时间
\cp -f /etc/chrony.conf /etc/chrony.conf.bk
cat > /etc/chrony.conf << EOF
driftfile /var/lib/chrony/drift
makestep 1.0 3
logdir /var/log/chrony
rtcsync
server 0.cn.pool.ntp.org
server 1.cn.pool.ntp.org
server 2.cn.pool.ntp.org
server 3.cn.pool.ntp.org

allow $allow_chrony_nets
EOF

zdump /usr/share/zoneinfo/Asia/Shanghai
\mv -f /etc/localtime /etc/localtime.old
ln -s /usr/share/zoneinfo/Asia/Shanghai /etc/localtime

systemctl enable chronyd
systemctl restart chronyd
chronyc sources -v
hwclock --systohc

# 添加mariadb源, 按照cdh-5.13的官方要求, 仅测试过是mariadb-5.5或者mariadb-10.0
cat > /etc/yum.repos.d/mariadb-10.0.repo << EOF
# MariaDB 10.0 CentOS repository list - created 2018-07-22 07:46 UTC
# http://downloads.mariadb.org/mariadb/repositories/
[mariadb-10.0]
name = MariaDB
baseurl = http://mirrors.ustc.edu.cn/mariadb/yum/10.0/centos7-amd64
gpgkey=https://mirrors.ustc.edu.cn/mariadb/yum/RPM-GPG-KEY-MariaDB
gpgcheck=1
EOF

yum makecache

# 初始化repo目录结构
mkdir -p /data/repo/parcel/cdh
mkdir -p /data/repo/parcel/kafka
mkdir -p /data/repo/yum/el7
mkdir -p /data/repo/pkg
mkdir -p /data/repo/iso
useradd repo
chown repo:repo -R /data/repo
su - repo

# /data/repo
# 静态资源目录结构
# 由于很多包比较大
# 可以选择在工作机上先下载好各个包并按照此目录结构将包上传到此服务器, 并注此脚本的释下载部分
# .
# ├── iso
# │   └── CentOS-7-x86_64-DVD-1804.iso
# ├── parcel
# │   ├── cdh
# │   │   ├── CDH-5.13.3-1.cdh5.13.3.p0.2-el7.parcel
# │   │   └── manifest.json
# │   └── kafka
# │   ├── KAFKA-3.1.0-1.3.1.0.p0.35-el7.parcel
# │   └── manifest.json
# ├── pkg
# │   ├── jdk-8u162-linux-x64.tar.gz
# │   └── mysql-connector-java-5.1.46.tar.gz
# └── yum
# ├── cdh5.13.3-centos7.tar.gz
# └── cm5.13.3-centos7.tar.gz

# 下载cdh的parcel包
cd /data/repo/parcel/cdh || exit 1
curl https://archive.cloudera.com/cdh5/parcels/5.13/CDH-5.13.3-1.cdh5.13.3.p0.2-el7.parcel -O
curl https://archive.cloudera.com/cdh5/parcels/5.13/manifest.json -O

# 下载kafka的parcel包
cd /data/repo/parcel/kafka || exit 1
curl https://archive.cloudera.com/kafka/parcels/latest/KAFKA-3.1.0-1.3.1.0.p0.35-el7.parcel -O
curl https://archive.cloudera.com/kafka/parcels/latest/manifest.json -O

# 下载需要用到的其它包
cd /data/repo/pkg || exit 1
curl http://ftp.osuosl.org/pub/funtoo/distfiles/oracle-java/jdk-8u162-linux-x64.tar.gz -O
curl http://ftp.ntu.edu.tw/MySQL/Downloads/Connector-J/mysql-connector-java-5.1.46.tar.gz -O

# 下载cdh5和cm5的repo-as-tarball包, 此包解压出来就是yum的各个依赖包
cd /data/repo/yum || exit 1
curl https://archive.cloudera.com/cdh5/repo-as-tarball/5.13.3/cdh5.13.3-centos7.tar.gz -O
curl http://archive.cloudera.com/cm5/repo-as-tarball/5.13.3/cm5.13.3-centos7.tar.gz -O
tar zxf cdh5.13.3-centos7.tar.gz
tar zxf cm5.13.3-centos7.tar.gz

# 同步mariadb-10.0源
cd /data/repo/yum || exit 1
reposync -r mariadb-10.0
cd /data/repo/yum/mariadb-10.0 || exit 1
createrepo .
curl https://mirrors.ustc.edu.cn/mariadb/yum/RPM-GPG-KEY-MariaDB -O

# 使用centos的dvd镜像为基础创建centos的base本地源
cd /data/repo/iso || exit 1
curl http://mirrors.ustc.edu.cn/centos/7/isos/x86_64/CentOS-7-x86_64-DVD-1804.iso -O

exit # 切换到root用户下
chown repo:repo -R /data/repo
cat >> /etc/fstab << EOF
/data/repo/iso/CentOS-7-x86_64-DVD-1804.iso /data/repo/yum/el7 iso9660 loop 0 0
EOF
mount -a

curl http://nginx.org/packages/centos/7/x86_64/RPMS/nginx-1.14.0-1.el7_4.ngx.x86_64.rpm -O
yum install ./nginx-1.14.0-1.el7_4.ngx.x86_64.rpm -y
cat > /etc/nginx/conf.d/repo.conf << EOF
server {
listen 8889;
server_name 0.0.0.0;
location / {
root /data/repo;
autoindex on;
}
}
EOF

sed -i 's/user nginx;/user repo;/g' /etc/nginx/nginx.conf

systemctl enable nginx
systemctl start nginx
显示 Gitment 评论