Merge pull request #38 from bai-charisu/master

fix ansible and deploy.md
opencurve · Jul 27, 2020 · af30ce9 · af30ce9
2 parents c987bfb + 520d4a7
commit af30ce9
Show file tree

Hide file tree

Showing 23 changed files with 265 additions and 118 deletions.
diff --git a/curve-ansible/README b/curve-ansible/README
@@ -73,6 +73,25 @@ cluster_map:
 如果是多机部署，则需要把internalport和externalport都设置成0。需要注意这里zone的数量要大于等于logicalpools里面指定的zonenum。
 logicalpools是逻辑池的列表，除了名字，需要指定它所属的物理池、类型、副本数量、copyset数量zone数量以及scatterwidth。目前的类型
 只支持0，page file类型。
+    如果需要格式化磁盘，则需要额外在group_vars/chunkservers.yml中指定磁盘的列表。比如：
+  disk_list:
+    - sda
+    - sdb
+    - sdc
+group_vars/chunkservers.yml中指定的变量是所有chunkserver共用的，如果某个chunkserver上的磁盘列表跟其他的不一致，则需要在host_vars
+中额外给出。比如server.ini中chunkserver是下面三台机器：
+[chunkservers]
+server1 ansible_ssh_host=10.192.100.1
+server2 ansible_ssh_host=10.192.100.2
+server3 ansible_ssh_host=10.192.100.3
+假设server1和server2对应的都是sda,sdb,sdc, server3使用的是sda,sdb,sdd,sde这四块盘。那么server1和server2的disk_list不需要在host_vars
+额外指出，而server3的disk_list需要额外指出。具体做法是在host_vars下面新建一个server3.yml。
+里面的内容为：
+  disk_list:
+    - sda
+    - sdb
+    - sdd
+    - sde
 
 1.1 部署etcd
 部署命令：
@@ -204,15 +223,19 @@ curve-sdk主要给cinder，nova等服务使用，只需要更新包和配置即
 │   └── wait_copysets_status_healthy.yml                    # 在一段时间内循环检查copyset健康状态
 │   ├── wait_until_server_down.yml                          # 等待直到server停掉
 │   └── wait_until_server_up.yml                            # 等待直到server起来
-├── roles                                                   # roles也是用来存放可以复用的代码，一个role内的task存在关联
 ├── group_vars                                              # 组变量
 │   ├── all.yml                                             # 属于所有host的变量
+│   ├── chunkservers.yml                                    # 属于chunkserver的变量
 │   └── mds.yml                                             # 属于mds的变量
+├── host_vars                                               # 主机变量, 其中的文件名要合inventory中定义的主机名一致
+│   └── localhost.yml                                       # 属于localhost的变量，host_vars优先级高于group_vars
+├── roles                                                   # roles也是用来存放可以复用的代码，一个role内的task存在关联
 │   ├── format_chunkserver                                  # 用来格式化chunkserver的role
 │   │   ├── defaults
 │   │   │   └── main.yml                                    # 存放带默认值的变量
 │   │   └── tasks
 │   │       ├── include
+│   │       │   ├── prepare_chunkserver_with_disk_format.yml    # 使用格式化磁盘的方式准备data目录
 │   │       │   └── prepare_chunkserver_without_disk_format.yml # 使用非格式化磁盘的方式准备data目录
 │   │       └── main.yml
 │   ├── restart_service                                     # 用来重启服务的role
@@ -241,19 +264,25 @@ curve-sdk主要给cinder，nova等服务使用，只需要更新包和配置即
 │   │   │   │   ├── install_curve-mds.yml                   # 安装mds
 │   │   │   │   ├── install_curve-nbd.yml                   # 安装nbd
 │   │   │   │   ├── install_curve-sdk.yml                   # 安装curve-sdk
+│   │   │   │   ├── install_curve-snapshotcloneserver-nginx.yml  # 安装快照克隆使用的Nginx
+│   │   │   │   ├── install_curve-snapshotcloneserver.yml   # 部署快照克隆
 │   │   │   │   ├── install_curve-tools.yml                 # 安装部署工具
+│   │   │   │   ├── install_daemon.yml                      # 安装daemon
 │   │   │   │   ├── install_deb_package.yml                 # 安装debian包
 │   │   │   │   ├── install_etcd.yml                        # 安装etcd
+│   │   │   │   ├── install_jemalloc.yml                    # 安装jemalloc
 │   │   │   │   ├── install_libuuid.yml                     # 安装libuuid
-│   │   │   │   ├── install_lib_with_source_code.yml        # 从源码安装库
+│   │   │   │   └── install_with_yum_apt.yml                # 用apt或yum的方式安装
+│   │   │   │   ├── install_with_source_code.yml            # 从源码安装
 │   │   │   │   └── install_nebd.yml                        # 安装nebd
 │   │   │   └── main.yml
 │   │   ├── templates                                       # 存放模板
 │   │   │   ├── chunkserver_ctl.sh.j2                       # chunkserver启动脚本的模板
 │   │   │   ├── chunkserver_deploy.sh.j2                    # chunkserver格式化脚本的模板
 │   │   │   ├── etcd-daemon.sh.j2                           # etcd启动脚本的模板
 │   │   │   ├── mds-daemon.sh.j2                            # mds启动脚本的模板
-│   │   │   └── nebd-daemon.j2                              # nebd-server启动脚本的模板
+│   │   │   ├── nebd-daemon.j2                              # nebd-server启动脚本的模板
+│   │   │   └── snapshot-daemon.sh.j2                       # 快照克隆启动脚本模板
 │   │   └── vars
 │   │       └── main.yml
 │   ├── set_leader_and_follower_list                        # 设置leader和follower列表
@@ -292,7 +321,9 @@ curve-sdk主要给cinder，nova等服务使用，只需要更新包和配置即
 │       │   │   ├── start_chunkserver.yml                  # 启动chunkserver
 │       │   │   ├── start_etcd.yml                         # 启动etcd
 │       │   │   ├── start_mds.yml                          # 启动mds
-│       │   │   └── start_nebd.yml                         # 启动nebd
+│       │   │   ├── start_nebd.yml                         # 启动nebd
+│       │   │   ├── start_snapshotcloneserver_nginx.yml    # 启动Nginx
+│       │   │   └── start_snapshotcloneserver.yml          # 启动快照克隆
 │       │   └── main.yml
 │       └── vars
 │           └── main.yml
@@ -312,6 +343,8 @@ curve-sdk主要给cinder，nova等服务使用，只需要更新包和配置即
 ├── deploy_mds.yml                                          # 部署mds
 ├── deploy_nbd.yml                                          # 部署nbd
 ├── deploy_nebd.yml                                         # 部署nebd
+├── deploy_snapshotcloneserver_nginx.yml                    # 部署快照克隆所用Nginx
+├── deploy_snapshotcloneserver.yml                          # 部署快照克隆
 ├── README                                                  # 本帮助文档
 ├── client.ini                                              # client的inventory文件
 └── server.ini                                              # server的列表，包括mds，快照克隆，etcd，chunkserver
diff --git a/curve-ansible/client.ini b/curve-ansible/client.ini
@@ -32,3 +32,4 @@ curve_include_dir=/usr/include
 curvefs_dir=/usr/curvefs
 ansible_ssh_port=22
 lib_install_prefix=/usr/local
+bin_install_prefix=/usr
diff --git a/curve-ansible/deploy_curve.yml b/curve-ansible/deploy_curve.yml
@@ -16,6 +16,15 @@
 #
 # The deploy playbook of Curve
 
+# 检查ansible版本
+- name: Stop if ansible version not match, make sure that the Ansible version is Ansible 2.5.9
+  hosts: localhost
+  gather_facts: no
+  tasks:
+    - assert:
+        that:
+          - ansible_version.full|version_compare('2.5.9', '==')
+
 # 部署curve集群
 
 - name: deploy etcd

diff --git a/...ks/include/determine_if_lib_installed.yml → curve-ansible/group_vars/chunkservers.yml b/...ks/include/determine_if_lib_installed.yml → curve-ansible/group_vars/chunkservers.yml
@@ -15,12 +15,7 @@
 #  limitations under the License.
 #
 
-- name: determine if lib installed
-  ignore_errors: true
-  shell: /sbin/ldconfig -p | grep {{ lib_name }}
-  register: ldconfig_res
-
-- name: set lib_installed
-  set_fact:
-    lib_installed: true
-  when: ldconfig_res.stdout != ""
+disk_list:
+  - sda
+  - sdb
+  - sdc
diff --git a/curve-ansible/group_vars/mds.yml b/curve-ansible/group_vars/mds.yml
@@ -1,4 +1,19 @@
 ---
+#
+#  Copyright (c) 2020 NetEase Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
 
 # 集群拓扑信息
 cluster_map:

diff --git a/curve-ansible/host_vars/localhost.yml b/curve-ansible/host_vars/localhost.yml
@@ -0,0 +1,22 @@
+---
+#
+#  Copyright (c) 2020 NetEase Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+disk_list:
+  - sda
+  - sdb
+  - sdc
+  - sdd
diff --git a/...e-ansible/roles/format_chunkserver/tasks/include/prepare_chunkserver_with_disk_format.yml b/...e-ansible/roles/format_chunkserver/tasks/include/prepare_chunkserver_with_disk_format.yml
@@ -0,0 +1,21 @@
+---
+#
+#  Copyright (c) 2020 NetEase Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+- name: deploy chunkserver with format disk
+  shell: sudo ./chunkserver_ctl.sh deploy all --noConfirm
+  async: 0
+  poll: 3
diff --git a/curve-ansible/roles/format_chunkserver/tasks/main.yml b/curve-ansible/roles/format_chunkserver/tasks/main.yml
@@ -16,9 +16,7 @@
 #
 
 - name: deploy chunkserver with format disk
-  shell: sudo ./chunkserver_ctl.sh deploy all --noConfirm
-  async: 0
-  poll: 3
+  include_tasks: "include/prepare_chunkserver_with_disk_format.yml"
   when: chunkserver_format_disk
 
 - name: deploy chunkserver without format disk

diff --git a/curve-ansible/roles/install_package/defaults/main.yml b/curve-ansible/roles/install_package/defaults/main.yml
@@ -44,8 +44,6 @@ etcd_download_dir: /tmp
 install_etcd_dir: /usr/bin
 
 download_dir: /tmp
-libunwind_download_url: http://download.savannah.gnu.org/releases/libunwind/libunwind-1.1.tar.gz
 jemalloc_download_url: https://github.com/jemalloc/jemalloc/releases/download/3.6.0/jemalloc-3.6.0.tar.bz2
 lib_install_prefix: /usr/local
-libnl_download_url: http://www.infradead.org/~tgr/libnl/files/libnl-3.2.25.tar.gz
 daemon_download_url: http://libslack.org/daemon/download/daemon-0.6.4.tar.gz
diff --git a/curve-ansible/roles/install_package/tasks/include/install_curve-chunkserver.yml b/curve-ansible/roles/install_package/tasks/include/install_curve-chunkserver.yml
@@ -17,15 +17,28 @@
 
 - name: install libunwind
   vars:
-    lib_name: libunwind
-    download_url: "{{ libunwind_download_url }}"
-  include_tasks: install_lib_with_source_code.yml
+    apt_lib_name: libunwind8 libunwind8-dev
+    yum_lib_name: libunwind libunwind-devel
+  include_tasks: install_with_yum_apt.yml
+
+- name: install bzip2
+  vars:
+    apt_lib_name: bzip2
+    yum_lib_name: bzip2
+  include_tasks: install_with_yum_apt.yml
 
 - name: install jemalloc
   vars:
     lib_name: jemalloc
     download_url: "{{ jemalloc_download_url }}"
-  include_tasks: install_lib_with_source_code.yml
+  include_tasks: install_jemalloc.yml
+
+- name: install psmisc
+  vars:
+    apt_lib_name: psmisc
+    yum_lib_name: psmisc
+  include_tasks: install_with_yum_apt.yml
+  when: chunkserver_format_disk
 
 - name: install chunkserver bin
   vars:

diff --git a/curve-ansible/roles/install_package/tasks/include/install_curve-mds.yml b/curve-ansible/roles/install_package/tasks/include/install_curve-mds.yml
@@ -17,12 +17,15 @@
 
 - name: install libunwind
   vars:
-    lib_name: libunwind
-    download_url: "{{ libunwind_download_url }}"
-  include_tasks: install_lib_with_source_code.yml
+    apt_lib_name: libunwind8 libunwind8-dev
+    yum_lib_name: libunwind libunwind-devel
+  include_tasks: install_with_yum_apt.yml
 
 - name: install libuuid
-  include_tasks: install_libuuid.yml
+  vars:
+    apt_lib_name: libunwind8 uuid-dev
+    yum_lib_name: libunwind libuuid-devel
+  include_tasks: install_with_yum_apt.yml
 
 - name: install daemon
   include_tasks: install_daemon.yml

diff --git a/curve-ansible/roles/install_package/tasks/include/install_curve-nbd.yml b/curve-ansible/roles/install_package/tasks/include/install_curve-nbd.yml
@@ -17,15 +17,15 @@
 
 - name: install libunwind
   vars:
-    lib_name: libunwind
-    download_url: "{{ libunwind_download_url }}"
-  include_tasks: install_lib_with_source_code.yml
+    apt_lib_name: libunwind8 libunwind8-dev
+    yum_lib_name: libunwind libunwind-devel
+  include_tasks: install_with_yum_apt.yml
 
 - name: install libnl
   vars:
-    lib_name: libnl-3
-    download_url: "{{ libnl_download_url}}"
-  include_tasks: install_lib_with_source_code.yml
+    apt_lib_name: libnl-3-dev libnl-genl-3-dev
+    yum_lib_name: libnl3 libnl3-devel
+  include_tasks: install_with_yum_apt.yml
 
 - name: install nbd bin
   vars:

diff --git a/curve-ansible/roles/install_package/tasks/include/install_curve-sdk.yml b/curve-ansible/roles/install_package/tasks/include/install_curve-sdk.yml
@@ -17,9 +17,9 @@
 
 - name: install libunwind
   vars:
-    lib_name: libunwind
-    download_url: "{{ libunwind_download_url }}"
-  include_tasks: install_lib_with_source_code.yml
+    apt_lib_name: libunwind8 libunwind8-dev
+    yum_lib_name: libunwind libunwind-devel
+  include_tasks: install_with_yum_apt.yml
 
 - name: install curve-sdk bin
   vars:

diff --git a/curve-ansible/roles/install_package/tasks/include/install_curve-snapshotcloneserver.yml b/curve-ansible/roles/install_package/tasks/include/install_curve-snapshotcloneserver.yml
@@ -17,12 +17,15 @@
 
 - name: install libunwind
   vars:
-    lib_name: libunwind
-    download_url: "{{ libunwind_download_url }}"
-  include_tasks: install_lib_with_source_code.yml
+    apt_lib_name: libunwind8 libunwind8-dev
+    yum_lib_name: libunwind libunwind-devel
+  include_tasks: install_with_yum_apt.yml
 
 - name: install libuuid
-  include_tasks: install_libuuid.yml
+  vars:
+    apt_lib_name: libunwind8 uuid-dev
+    yum_lib_name: libunwind libuuid-devel
+  include_tasks: install_with_yum_apt.yml
 
 - name: install daemon
   include_tasks: install_daemon.yml

diff --git a/curve-ansible/roles/install_package/tasks/include/install_daemon.yml b/curve-ansible/roles/install_package/tasks/include/install_daemon.yml
@@ -28,5 +28,6 @@
 - name: install lib if not installed
   vars:
     download_url: "{{ daemon_download_url }}"
+    prefix: "{{ bin_install_prefix }}"
   include_tasks: install_with_source_code.yml
   when: not daemon_installed
diff --git a/.../include/install_lib_with_source_code.yml → ...ackage/tasks/include/install_jemalloc.yml b/.../include/install_lib_with_source_code.yml → ...ackage/tasks/include/install_jemalloc.yml
@@ -15,21 +15,24 @@
 #  limitations under the License.
 #
 
-- name: determine if lib installed
+- name: determine if jemalloc installed
   ignore_errors: true
-  shell: /sbin/ldconfig -p | grep {{ lib_name }}
-  register: ldconfig_res
+  shell: ls {{ lib_install_prefix }}/lib | grep jemalloc
+  register: ls_res
 
 - name: set lib_installed true
   set_fact:
     lib_installed: true
-  when: ldconfig_res.stdout != ""
+  when: ls_res.stdout != ""
 
 - name: set lib_installed false
   set_fact:
     lib_installed: false
-  when: ldconfig_res.stdout == ""
+  when: ls_res.stdout == ""
 
 - name: install lib if not installed
+  vars:
+    prefix: "{{ lib_install_prefix }}"
+    lib_name: jemalloc
   include_tasks: install_with_source_code.yml
   when: not lib_installed
diff --git a/curve-ansible/roles/install_package/tasks/include/install_nebd.yml b/curve-ansible/roles/install_package/tasks/include/install_nebd.yml
@@ -17,15 +17,21 @@
 
 - name: install libunwind
   vars:
-    lib_name: libunwind
-    download_url: "{{ libunwind_download_url }}"
-  include_tasks: install_lib_with_source_code.yml
+    apt_lib_name: libunwind8 libunwind8-dev
+    yum_lib_name: libunwind libunwind-devel
+  include_tasks: install_with_yum_apt.yml
+
+- name: install bzip2
+  vars:
+    apt_lib_name: bzip2
+    yum_lib_name: bzip2
+  include_tasks: install_with_yum_apt.yml
 
 - name: install jemalloc
   vars:
     lib_name: jemalloc
     download_url: "{{ jemalloc_download_url }}"
-  include_tasks: install_lib_with_source_code.yml
+  include_tasks: install_jemalloc.yml
 
 - name: install daemon
   include_tasks: install_daemon.yml