一、准备工作
1. 安装操作系统Centos6.9 ，略。
2. 配置ip 和主机名
ip 配置如下：
节点1 ：
[root@localhost ~]# vi /etc/hosts
#public ip (bond0)
192.168.1.1 TRAC1
192.168.1.2 TRAC2
#private ip (em3)
1 TRAC1-priv
10.1.1.2 TRAC2-priv
#public vip
192.168.1.3 TRAC1-vip
192.168.1.4 TRAC2-vip
#scan ip
192.168.1.5 TRAC
节点2 ：
[root@localhost ~]# vi /etc/hosts
#public ip (bond0)
192.168.1.1 TRAC1
192.168.1.2 TRAC2
#private ip (em3)
10.1.1.1 TRAC1-priv
10.1.1.2 TRAC2-priv
#public vip
192.168.1.3 TRAC1-vip
192.168.1.4 TRAC2-vip
#scan ip
192.168.1.5 TRAC
3. 配置主机名：
节点1 ：
[root@localhost ~]# hostname TRAC1
[root@localhost ~]# vi /etc/sysconfig/network
NETWORKING=yes
HOSTNAME=TRAC1
节点2 ：
[root@localhost ~]# hostname TRAC2
[root@localhost ~]# vi /etc/sysconfig/network
NETWORKING=yes
HOSTNAME=TRAC2
检查HOSTS 配置：以上的hostname 和IP 都ping 一遍（此时vip 的ip 和host 应该是不通的）
4. 配置IP
节点1 ：
[root@localhost network-scripts]# vi ifcfg-em3
DEVICE=em3
HWADDR=18:66:DA:EF:22:E2
TYPE=Ethernet
UUID=c3096b1c-94dd-4eb2-88a0-e760eb95f23a
>
NM_CONTROLLED=yes
BOOTPROTO=none
IPADDR=10.1.1.1
NETMASK=255.255.255.0
IPV6INIT=no
USERCTL=no
[root@localhost network-scripts]# service network restart
节点2
[root@localhost network-scripts]# vi ifcfg-em3
DEVICE=em3
HWADDR=18:66:DA:EF:30:2A
TYPE=Ethernet
UUID=74c89752-ae33-445b-88de-7125531594f8
>
NM_CONTROLLED=yes
BOOTPROTO=none
IPADDR=10.1.1.2
NETMASK=255.255.255.0
IPV6INIT=no
USERCTL=no
[root@localhost network-scripts]# service network restart
5. 两节点配置ntp 服务
crontab -l
*/5 * * * * /usr/sbin/ntpdate “ 服务器IP ”>> /root/sync_date.log 2>&1
mv /etc/ntp.conf /etc/ntp.conf.old
6. 配置DNS nameserver
cp /etc/resolv.conf /etc/resolv.conf.bak
vi /etc/resolv.conf
nameserver 192.168.1.245
7. 修改内核参数
分别在两节点上执行：
cp /etc/sysctl.conf /etc/sysctl.conf.bak
vi /etc/sysctl.conf 末尾
# 注释掉kernel.shmmax 和kernel.shmall 两行
kernel.shmall = 11111 ( 单位：字节 物理内存/4096)
kernel.shmmax = 11111 （单位：字节 物理内存-1 ）
添加：
fs.aio-max-nr = 1048576
fs.file-max = 6815744
kernel.shmmax = 135221043199
kernel.shmmni = 4096
kernel.shmall = 33012950
kernel.sem = 250 32000 100 128
net.ipv4.ip_local_port_range = 9000 65500
net.core.rmem_default = 262144
net.core.rmem_max = 4194304
net.core.wmem_default = 262144
net.core.wmem_max = 1048576
以上参数为使用oracle yum 自动配置的参数
重新加载生效：
[root@TRAC1 ~]# sysctl -p
8. 安装必要的包
两节点分别执行如下三步：
① 配置yum
[root@TRAC1 grid]# cd /etc/yum.repos.d/
[root@TRAC1 yum.repos.d]# cp CentOS-Base.repo CentOS-Base.repo.bak
[root@TRAC1 yum.repos.d]# vi /etc/yum.repos.d/CentOS-Base.repo
[base]
name=CentOS-$releasever - Base
baseurl=http://192.168.1.10/centos6.9
enabled=1
gpgcheck=0
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6
[root@TRAC1 yum.repos.d]# yum clean all
Loaded plugins: fastestmirror, refresh-packagekit, security
Cleaning repos: base
Cleaning up Everything
Cleaning up list of fastest mirrors
[root@TRAC1 yum.repos.d]# yum makecache
Loaded plugins: fastestmirror, refresh-packagekit, security
Determining fastest mirrors
Metadata Cache Created
…….
[root@TRAC1 yum.repos.d]#
② 检查哪些包没安装
for i in binutils compat-gcc-34 compat-libstdc++-296 control-center \
gcc gcc-c++ glibc glibc-common glibc-devel libaio libgcc elfutils-libelf-devel \
libstdc++ libstdc++-devel libXp make openmotif22 setarch \
compat-libstdc++-33 libaio-devel sysstat unixODBC unixODBC-devel \
compat-libcap1 ksh tigervnc-server
do
rpm -q $i &>/dev/null || F="$F $i"
done ;echo $F;unset F
③ 使用yum 安装这些包
[root@TRAC1 yum.repos.d]# yum install -y compat-gcc-34 compat-libstdc++-296 gcc gcc-c++ elfutils-libelf-devel libstdc++-devel libXp openmotif22 setarch compat-libstdc++-33 libaio-devel unixODBC unixODBC-devel compat-libcap1 ksh tigervnc-server
9. 修改两节点的oracle 用户限制
cp /etc/security/limits.conf /etc/security/limits.conf.bak
vi /etc/security/limits.conf 末尾添加：
oracle soft nofile 2047
oracle hard nofile 65536
oracle soft nproc 2047
oracle hard nproc 16384
oracle soft stack 10240
oracle hard stack 32768
grid soft nofile 2047
grid hard nofile 65536
grid soft nproc 2047
grid hard nproc 16384
grid soft stack 10240
grid hard stack 32768
10. 修改两节点的/etc/pam.d/login
cp /etc/pam.d/login /etc/pam.d/login.bak
vi /etc/pam.d/login 末尾添加：
session required /lib64/security/pam_limits.so
11. 关闭两节点的防火墙和selinux
[root@ ~]#
service iptables stop
chkconfig iptables off
setenforce 0
getenforce
cp /etc/selinux/config /etc/selinux/config.bak
vi /etc/selinux/config 确保以下内容
SELINUX=disabled
[root@ ~]# cat /etc/selinux/config
# This file controls the state of SELinux on the system.
# SELINUX= can take one of these three values:
# enforcing - SELinux security policy is enforced.
# permissive - SELinux prints warnings instead of enforcing.
# disabled - SELinux is fully disabled.
SELINUX=disabled
# SELINUXTYPE= type of policy in use. Possible values are:
# targeted - Only targeted network daemons are protected.
# strict - Full SELinux protection.
SELINUXTYPE=targeted
12. 更改 安装所有者的 ulimit 设置
两节点都要执行：
vi /etc/profile 末尾添加：
if [ /$USER = "oracle" ] || [ $USER = "grid" ]; then
if [ /$SHELL = "/bin/ksh" ]; then
ulimit -p 16384
ulimit -n 65536
else
ulimit -u 16384 -n 65536
fi
umask 022
fi
13. 两节点建立必要的组和用户
[root@ ~]#
groupadd -g 501 oinstall
groupadd -g 502 dba
groupadd -g 503 oper
groupadd -g 504 asmadmin
groupadd -g 505 asmdba
groupadd -g 506 asmoper
useradd -u 501 -g oinstall -G dba,oper,asmdba oracle
useradd -u 502 -g oinstall -G asmadmin,asmoper,asmdba grid
echo "123456"|passwd --stdin oracle
echo "123456"|passwd --stdin grid
用户和组的说明：
14. 两节点建立安装目录
[root@ ~]#
chown -R oracle:oinstall /home/oracle
chown -R grid:oinstall /home/grid
---Grid/oracle Infrastructure BASE:
mkdir -p /home/app/{oracle,grid}
---Grid Infrastructure HOME:
mkdir -p /home/app/grid/{base,crs}
chown -R grid:oinstall /home/app/grid
chown -R oracle:oinstall /home/app/oracle
chmod 755 /home/oracle
chmod 755 /home/grid
chmod -R 775 /home/app
15. 两节点设置oracle 和grid 用户的环境变量
su - oracle
vi .bash_profile 末尾添加：
export ORACLE_BASE=/home/app/oracle
export ORACLE_HOME=/home/app/oracle/product/11.2.0
export ORACLE_SID=TRAC1
export PATH=$PATH:$ORACLE_HOME/bin
export LD_LIBRARY_PATH=$ORACLE_HOME/lib:/lib:/usr/lib
export NLS_LANG=AMERICAN_AMERICA.UTF8
su - grid
vi .bash_profile 末尾添加：
export ORACLE_BASE=/home/app/grid/base
export ORACLE_HOME=/home/app/grid/crs
export ORACLE_SID=+ASM1
export PATH=$PATH:$ORACLE_HOME/bin
export LD_LIBRARY_PATH=$ORACLE_HOME/lib:/lib:/usr/lib
export NLS_LANG=AMERICAN_AMERICA.UTF8
16. 两节点安装asm 包
①查看本机内核版本：
[root@rac1 u01]# uname -rm
2.6.18-274.el5 x86_64
②到以下网址下载对应操作系统版本的三个asm 包：
http://www.oracle.com/technetwork/topics/linux/asmlib/index-101839.html
④ 把下载下来的asm 包上传到两个节点，并安装，注意按照以下安装顺序
cd /home/software/rpm
[root@TRAC1 rpm]# rpm -ivh oracleasm-support-2.1.8-1.el6.x86_64.rpm
warning: oracleasm-support-2.1.8-1.el6.x86_64.rpm: Header V3 RSA/SHA256 Signature, key ID ec551f03: NOKEY
Preparing... ########################################### [100%]
1:oracleasm-support ########################################### [100%]
[root@TRAC1 rpm]# rpm -ivh kmod-oracleasm-2.0.8-15.el6_9.x86_64.rpm
warning: kmod-oracleasm-2.0.8-15.el6_9.x86_64.rpm: Header V3 RSA/SHA1 Signature, key ID c105b9de: NOKEY
Preparing... ########################################### [100%]
1:kmod-oracleasm ########################################### [100%]
[root@TRAC1 rpm]# rpm -ivh oracleasmlib-2.0.4-1.el6.x86_64.rpm
warning: oracleasmlib-2.0.4-1.el6.x86_64.rpm: Header V3 RSA/SHA256 Signature, key ID ec551f03: NOKEY
Preparing... ########################################### [100%]
1:oracleasmlib ########################################### [100%]
[root@TRAC1 rpm]#
16. 两节点分别配置asm ：
[root@ ~]# /etc/init.d/oracleasm configure
Configuring the Oracle ASM library driver.
This will configure the on-boot properties of the Oracle ASM library
driver. The following questions will determine whether the driver is
loaded on boot and what permissions it will have. The current values
will be shown in brackets ('[]'). Hitting <ENTER> without typing an
answer will keep that current value. Ctrl-C will abort.
Default user to own the driver interface []: grid
Default group to own the driver interface []: asmadmin
Start Oracle ASM library driver on boot (y/n) [n]: y
Scan for Oracle ASM disks on boot (y/n) [y]: y
Writing Oracle ASM library driver configuration: done
Initializing the Oracle ASMLib driver: [ OK ]
Scanning the system for Oracle ASMLib disks: [ OK ]
17. 共享磁盘( 重新) 分区
18. 创建asm 磁盘：
以下在任意一个节点执行就行了：
19. 扫描磁盘asm 磁盘，检查有哪些asm 磁盘：
每个节点都要执行：
20. ssh 双机互信
oracle/grid 用户均需要做
节点1
ssh-keygen -t rsa
ssh-keygen -t dsa
ssh-copy-id -i .ssh/id_dsa.pub TRAC2
ssh-copy-id -i .ssh/id_rsa.pub TRAC2
节点2
ssh-keygen -t rsa
ssh-keygen -t dsa
ssh-copy-id -i .ssh/id_dsa.pub TRAC1
ssh-copy-id -i .ssh/id_rsa.pub TRAC1
cat .ssh/*.pub >> .ssh/authorized_keys
节点1
cat .ssh/*.pub >> .ssh/authorized_keys
ssh TRAC1 date && ssh TRAC1-priv date && ssh TRAC2 date && ssh TRAC2-priv date
21. 再NFS 上建立disk （临时测试使用NFS ）
mount -t nfs -o rw,bg,hard,nointr,rsize=32768,wsize=32768,tcp,noac,vers=3,timeo=600,actimeo=0 188.188.42.71:/data/oracle_test /data
mount -t nfs 166.188.30.19:/data/nfs /nfs
dd if=/dev/zero of=/data/oradata/votdisk1 bs=8192 count=100000
cd /data/oradata/
chmod 660 *
chown grid:asmadmin *
22. 存储配置
1. 多路径安装
yum install -y device-mapper-multi*
chkconfig --level 2345 multipathd on
service multipathd start
multipath -ll
cp /usr/share/doc/device-mapper-multipath-0.4.9/multipath.conf /etc/multipath.conf
service multipathd restart
2.multipath.conf 配置
节点 1 ， 2 需要同步一致
vi /etc/multipath.conf
# This is a basic configuration file with some examples, for device mapper
# multipath.
# For a complete list of the default configuration values, see
# /usr/share/doc/device-mapper-multipath-0.4.9/multipath.conf.defaults
# For a list of configuration options with descriptions, see
# /usr/share/doc/device-mapper-multipath-0.4.9/multipath.conf.annotated
#
# REMEMBER: After updating multipath.conf, you must run
#
# service multipathd reload
#
# for the changes to take effect in multipathd
## By default, devices with vendor = "IBM" and product = "S/390.*" are
## blacklisted. To enable mulitpathing on these devies, uncomment the
## following lines.
#blacklist_exceptions {
# device {
# vendor "IBM"
# product "S/390.*"
# }
#}
## Use user friendly names, instead of using WWIDs as names.
#defaults {
# user_friendly_names yes
#}
##
## Here is an example of how to configure some standard options.
##
#
defaults {
udev_dir /dev
polling_interval 5
path_selector "round-robin 0"
path_grouping_policy multibus
getuid_callout "/lib/udev/scsi_id --whitelisted --device=/dev/%n"
prio alua
path_checker readsector0
rr_min_io 100
max_fds 8192
rr_weight priorities
failback immediate
no_path_retry fail
user_friendly_names yes
}
##
## The wwid line in the following blacklist section is shown as an example
## of how to blacklist devices by wwid. The 2 devnode lines are the
## compiled in default blacklist. If you want to blacklist entire types
## of devices, such as all scsi devices, you should use a devnode line.
## However, if you want to blacklist specific devices, you should use
## a wwid line. Since there is no guarantee that a specific device will
## not change names on reboot (from /dev/sda to /dev/sdb for example)
## devnode lines are not recommended for blacklisting specific devices.
##
#blacklist {
# wwid 26353900f02796769
# devnode "^(ram|raw|loop|fd|md|dm-|sr|scd|st)[0-9]*"
# devnode "^hd[a-z]"
#}
#multipaths {
# multipath {
# wwid 3600508b4000156d700012000000b0000
# alias yellow
# path_grouping_policy multibus
# path_checker readsector0
# path_selector "round-robin 0"
# failback manual
# rr_weight priorities
# no_path_retry 5
# }
# multipath {
# wwid 1DEC_____321816758474
# alias red
# }
#}
#devices {
# device {
# vendor "COMPAQ "
# product "HSV110 (C)COMPAQ"
# path_grouping_policy multibus
# getuid_callout "/lib/udev/scsi_id --whitelisted --device=/dev/%n"
# path_checker readsector0
# path_selector "round-robin 0"
# hardware_handler "0"
# failback 15
# rr_weight priorities
# no_path_retry queue
# }
# device {
# vendor "COMPAQ "
# product "MSA1000 "
# path_grouping_policy multibus
# }
#}
multipaths {
multipath {
wwid 36000d310013faa000000000000000070
alias racsas1
}
multipath {
wwid 36000d310013faa00000000000000007a
alias racsas2
}
multipath {
wwid 36000d310013faa00000000000000007b
alias racsas3
}
multipath {
wwid 36000d310013faa00000000000000007c
alias racsas4
}
multipath {
wwid 36000d310013faa00000000000000007d
alias racsas5
}
multipath {
wwid 36000d310013faa00000000000000008a
alias racssd1
}
multipath {
wwid 36000d310013faa000000000000000086
alias racssd2
}
multipath {
wwid 36000d310013faa000000000000000087
alias racssd3
}
multipath {
wwid 36000d310013faa000000000000000088
alias racssd4
}
multipath {
wwid 36000d310013faa000000000000000089
alias racssd5
}
multipath {
wwid 36000d310038170000000000000000007
alias backup1
}
multipath {
wwid 36000d31003817200000000000000000a
alias dgdata1
}
}
3. 多路径磁盘分区
仅节点2
[root@TRAC2 ~]# parted /dev/mapper/racsas1
(parted) mkpart primary 0% 100%
(parted) select /dev/mapper/racsas2 mklabel gpt mkpart primary 0% 100%
(parted) select /dev/mapper/racsas3 mklabel gpt mkpart primary 0% 100%
(parted) select /dev/mapper/racsas4 mklabel gpt mkpart primary 0% 100%
(parted) select /dev/mapper/racssd1
(parted) mklabel gpt
(parted) mkpart primary 0 10G
(parted) mkpart primary
File system type? [ext2]? ext3
Start?
Start? 10G
End? 20G
(parted) mkpart primary
File system type? [ext2]? ext3
Start? 20G
End? 30G
(parted) mkpart primary
File system type? [ext2]? ext3
Start? 30G
End? 100%
(parted) select /dev/mapper/racssd2
Using /dev/mapper/racssd2
(parted) mklabel gpt
(parted) mkpart primary 0% 100%
(parted) select /dev/mapper/racssd3 mklabel gpt mkpart primary 0% 100%
Using /dev/mapper/racssd3
(parted) select /dev/mapper/racssd4 mklabel gpt mkpart primary 0% 100%
Using /dev/mapper/racssd4
节点1
[root@TRAC1 ~]# service multipathd restart
ok
Stopping multipathd daemon: [ OK ]
Starting multipathd daemon: [ OK ]
4.udev 配置
节点1 ，2
[root@TRAC2 ~]# ll /dev/mapper/
lrwxrwxrwx 1 root root 8 Jun 13 14:21 racsas1 -> ../dm-13
lrwxrwxrwx 1 root root 8 Jun 13 14:21 racsas1p1 -> ../dm-16
lrwxrwxrwx 1 root root 7 Jun 13 14:21 racsas2 -> ../dm-5
lrwxrwxrwx 1 root root 8 Jun 13 14:21 racsas2p1 -> ../dm-17
lrwxrwxrwx 1 root root 7 Jun 13 14:21 racsas3 -> ../dm-4
lrwxrwxrwx 1 root root 8 Jun 13 14:21 racsas3p1 -> ../dm-18
lrwxrwxrwx 1 root root 7 Jun 13 14:21 racsas4 -> ../dm-3
lrwxrwxrwx 1 root root 8 Jun 13 14:21 racsas4p1 -> ../dm-19
lrwxrwxrwx 1 root root 8 Jun 13 14:21 racssd1 -> ../dm-14
lrwxrwxrwx 1 root root 8 Jun 13 14:21 racssd1p1 -> ../dm-20
lrwxrwxrwx 1 root root 8 Jun 13 14:21 racssd1p2 -> ../dm-21
lrwxrwxrwx 1 root root 8 Jun 13 14:21 racssd1p3 -> ../dm-22
lrwxrwxrwx 1 root root 8 Jun 13 14:21 racssd1p4 -> ../dm-23
lrwxrwxrwx 1 root root 7 Jun 13 14:21 racssd2 -> ../dm-7
lrwxrwxrwx 1 root root 8 Jun 13 14:21 racssd2p1 -> ../dm-24
lrwxrwxrwx 1 root root 7 Jun 13 14:21 racssd3 -> ../dm-2
lrwxrwxrwx 1 root root 8 Jun 13 14:21 racssd3p1 -> ../dm-25
lrwxrwxrwx 1 root root 7 Jun 13 14:21 racssd4 -> ../dm-6
lrwxrwxrwx 1 root root 8 Jun 13 14:21 racssd4p1 -> ../dm-26
[root@TRAC2 ~]# for i in `seq 16 26`;do printf "%s %s %s\n" "$i" "$(ll /dev/VolGroup/|grep dm-$i|awk '{print $9}')" "$(udevadm info --query=all --name=/dev/dm-$i | grep -i dm_uuid)";done
16 E: DM_UUID=part1-mpath-36000d310013fb0000000000000000015 - ARCH
17 E: DM_UUID=part1-mpath-36000d310013fb0000000000000000010 -SASDATA1
18 E: DM_UUID=part1-mpath-36000d310013fb0000000000000000012 -SASDATA2
19 E: DM_UUID=part1-mpath-36000d310013fb0000000000000000013 -SASDATA3
20 E: DM_UUID=part1-mpath-36000d310013fb0000000000000000016 -VOT1
21 E: DM_UUID=part2-mpath-36000d310013fb0000000000000000016 -VOT2
22 E: DM_UUID=part3-mpath-36000d310013fb0000000000000000016 -VOT3
23 E: DM_UUID=part4-mpath-36000d310013fb0000000000000000016 -FRA
24 E: DM_UUID=part1-mpath-36000d310013fb000000000000000000c -SSDDATA1
25 E: DM_UUID=part1-mpath-36000d310013fb000000000000000000d -SSDDATA2
26 E: DM_UUID=part1-mpath-36000d310013fb0000000000000000014 -SSDDATA3
[root@TRAC2 ~]# cd /etc/udev/rules.d/
vi /etc/udev/rules.d/99-oracle-asmdevices.rules
KERNEL=="dm-*",ENV{DM_UUID}=="part1-mpath-36000d310013fb0000000000000000015",NAME="asmdisk_arch",OWNER="grid",GROUP="asmadmin",MODE="0660"
KERNEL=="dm-*",ENV{DM_UUID}=="part1-mpath-36000d310013fb0000000000000000010",NAME="asmdisk_sasdata1",OWNER="grid",GROUP="asmadmin",MODE="0660"
KERNEL=="dm-*",ENV{DM_UUID}=="part1-mpath-36000d310013fb0000000000000000012",NAME="asmdisk_sasdata2",OWNER="grid",GROUP="asmadmin",MODE="0660"
KERNEL=="dm-*",ENV{DM_UUID}=="part1-mpath-36000d310013fb0000000000000000013",NAME="asmdisk_sasdata3",OWNER="grid",GROUP="asmadmin",MODE="0660"
KERNEL=="dm-*",ENV{DM_UUID}=="part1-mpath-36000d310013fb0000000000000000016",NAME="asmdisk_vot1",OWNER="grid",GROUP="asmadmin",MODE="0660"
KERNEL=="dm-*",ENV{DM_UUID}=="part2-mpath-36000d310013fb0000000000000000016",NAME="asmdisk_vot2",OWNER="grid",GROUP="asmadmin",MODE="0660"
KERNEL=="dm-*",ENV{DM_UUID}=="part3-mpath-36000d310013fb0000000000000000016",NAME="asmdisk_vot3",OWNER="grid",GROUP="asmadmin",MODE="0660"
KERNEL=="dm-*",ENV{DM_UUID}=="part4-mpath-36000d310013fb0000000000000000016",NAME="asmdisk_fra",OWNER="grid",GROUP="asmadmin",MODE="0660"
KERNEL=="dm-*",ENV{DM_UUID}=="part1-mpath-36000d310013fb000000000000000000c",NAME="asmdisk_ssddata1",OWNER="grid",GROUP="asmadmin",MODE="0660"
KERNEL=="dm-*",ENV{DM_UUID}=="part1-mpath-36000d310013fb000000000000000000d",NAME="asmdisk_ssddata2",OWNER="grid",GROUP="asmadmin",MODE="0660"
KERNEL=="dm-*",ENV{DM_UUID}=="part1-mpath-36000d310013fb0000000000000000014",NAME="asmdisk_ssddata3",OWNER="grid",GROUP="asmadmin",MODE="0660"
[root@TRAC2 rules.d]# udevadm control --reload-rules
[root@TRAC2 rules.d]# start_udev
[root@TRAC2 rules.d]# scp /etc/udev/rules.d/99-oracle-asmdevices.rules root@192.168.1.6:/etc/udev/rules.d/99-oracle-asmdevices.rules
节点1
[root@TRAC1 ~]# udevadm control --reload-rules
[root@TRAC1 ~]# start_udev
Starting udev: [ OK ]
[root@TRAC1 ~]#
[root@TRAC1 ~]# ll /dev/mapper/
total 0
lrwxrwxrwx 1 root root 7 Jun 13 14:35 racsas1 -> ../dm-4
lrwxrwxrwx 1 root root 15 Jun 13 14:35 racsas1p1 -> ../asmdisk_arch
lrwxrwxrwx 1 root root 7 Jun 13 14:35 racsas2 -> ../dm-5
lrwxrwxrwx 1 root root 19 Jun 13 14:35 racsas2p1 -> ../asmdisk_sasdata1
lrwxrwxrwx 1 root root 7 Jun 13 14:35 racsas3 -> ../dm-8
lrwxrwxrwx 1 root root 19 Jun 13 14:35 racsas3p1 -> ../asmdisk_sasdata2
lrwxrwxrwx 1 root root 7 Jun 13 14:35 racsas4 -> ../dm-3
lrwxrwxrwx 1 root root 19 Jun 13 14:35 racsas4p1 -> ../asmdisk_sasdata3
lrwxrwxrwx 1 root root 8 Jun 13 14:35 racssd1 -> ../dm-11
lrwxrwxrwx 1 root root 15 Jun 13 14:35 racssd1p1 -> ../asmdisk_vot1
lrwxrwxrwx 1 root root 15 Jun 13 14:35 racssd1p2 -> ../asmdisk_vot2
lrwxrwxrwx 1 root root 15 Jun 13 14:35 racssd1p3 -> ../asmdisk_vot3
lrwxrwxrwx 1 root root 14 Jun 13 14:35 racssd1p4 -> ../asmdisk_fra
lrwxrwxrwx 1 root root 7 Jun 13 14:35 racssd2 -> ../dm-6
lrwxrwxrwx 1 root root 19 Jun 13 14:35 racssd2p1 -> ../asmdisk_ssddata1
lrwxrwxrwx 1 root root 7 Jun 13 14:35 racssd3 -> ../dm-2
lrwxrwxrwx 1 root root 19 Jun 13 14:35 racssd3p1 -> ../asmdisk_ssddata2
lrwxrwxrwx 1 root root 8 Jun 13 14:35 racssd4 -> ../dm-10
lrwxrwxrwx 1 root root 19 Jun 13 14:35 racssd4p1 -> ../asmdisk_ssddata3
5. 添加新磁盘步骤
1.Dell storage management 增加新卷并映射给主机
2. 主机通过下面命令扫描磁盘，节点1 ，2 均需
for i in `ls /sys/class/scsi_host/`; do echo "- - -" >> /sys/class/scsi_host/$i/scan ; done
3. 通过下面命令查看磁盘的wwid
scsi_id --whitelisted --replace-whitespace /dev/sd*
4. 修改/etc/multipath.conf 添加新磁盘，节点1 ，2 需要同步一致
5. 重新加载多路径聚合
service multipathd reload
multipath -ll 查看多路径聚合是否成功
6. 修改/etc/udev/rules.d/99-oracle-asmdevices.rules ，节点1 ，2 需要同步一致
7. 更新udev 磁盘映射
udevadm control --reload-rules
start_udev
ll /dev/mapper/
8. 增加磁盘到asm diskgroup
select name,total_mb,free_mb from v$asm_diskgroup; 查看目前diskgroup 大小
alter diskgroup SASDATA add disk ‘/dev/asm_sas6’ rebalance power 10; 增加磁盘到diskgroup
select * from v$asm_operation; 等待operation 完成，再查看diskgroup 大小
注意：每次只能一个operation 存在
[root@TRAC1 tmp]# su - grid
[grid@TRAC1 ~]$ sqlplus / as sysasm
SQL*Plus: Release 11.2.0.4.0 Production on Tue Jun 20 10:26:37 2017
Copyright (c) 1982, 2013, Oracle. All rights reserved.
Connected to:
Oracle Database 11g Enterprise Edition Release 11.2.0.4.0 - 64bit Production
With the Real Application Clusters and Automatic Storage Management options
SQL> col name format A20
SQL> select name,total_mb,free_mb from v$asm_diskgroup;
NAME TOTAL_MB FREE_MB
-------------------- ---------- ----------
SASDATA 8388608 6758368
SSDDATA 8388608 7194758
VOT_MIRROR 42915 42087
VOT 42915 41989
SQL> alter diskgroup SASDATA add disk '/dev/asm_sas6' rebalance power 10;
Diskgroup altered.
SQL> col ERROR_CODE format a10
SQL> select * from v$asm_operation;
GROUP_NUMBER OPERATION STATE POWER ACTUAL SOFAR EST_WORK EST_RATE EST_MINUTES ERROR_CODE
------------ --------------- ------------ ---------- ---------- ---------- ---------- ---------- ----------- ----------
1 REBAL RUN 10 10 13997 326080 13339 23
6. 扩展磁盘步骤
1. Dell storage management 扩展现有卷
2. 查看现有多路径关系
[root@TRAC1 ~]# multipath -l racsas6
racsas6 (36000d310013faa0000000000000000a5) dm-20 COMPELNT,Compellent Vol
size=2.0T features='1 queue_if_no_path' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=0 status=active
|- 1:0:5:11 sdaq 66:160 active undef unknown
|- 4:0:0:11 sdaw 67:0 active undef unknown
|- 1:0:14:11 sdat 66:208 active undef unknown
`- 4:0:14:11 sday 67:32 active undef unknown
[root@TRAC1 ~]# multipath -l racssd6
racssd6 (36000d310013faa0000000000000000a8) dm-21 COMPELNT,Compellent Vol
size=2.0T features='1 queue_if_no_path' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=0 status=active
|- 1:0:3:12 sdac 65:192 active undef unknown
|- 4:0:2:12 sdax 67:16 active undef unknown
|- 1:0:15:12 sdau 66:224 active undef unknown
`- 4:0:15:12 sdba 67:64 active undef unknown
[root@TRAC1 ~]# multipath -l backup1
backup1 (36000d310038170000000000000000007) dm-1 COMPELNT,Compellent Vol
size=4.0T features='1 queue_if_no_path' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=0 status=active
|- 1:0:11:1 sde 8:64 active undef unknown
`- 4:0:8:1 sdc 8:32 active undef unknown
3. 重新扫描磁盘
multipath -l racsas6|grep sd|awk '{print "echo 1 > /sys/block/"$3"/device/rescan"}'
multipath -l racssd6|grep sd|awk '{print "echo 1 > /sys/block/"$3"/device/rescan"}'
multipath -l backup1|grep sd|awk '{print "echo 1 > /sys/block/"$3"/device/rescan"}'
执行上面打印的命令
4.service multipathd reload 重载多路径
5.multipath -l backup1 查看大小
backup1 (36000d310038170000000000000000007) dm-1 COMPELNT,Compellent Vol
size=10T features='1 queue_if_no_path' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=0 status=active
|- 4:0:11:1 sdd 8:48 active undef unknown
`- 1:0:11:1 sde 8:64 active undef unknown
6. 非ASM 磁盘扩展
[root@TRAC1 ~]# parted /dev/mapper/backup1
GNU Parted 2.1
Using /dev/mapper/backup1
Welcome to GNU Parted! Type 'help' to view a list of commands.
(parted) print
Model: Linux device-mapper (multipath) (dm)
Disk /dev/mapper/backup1: 11.0TB
Sector size (logical/physical): 512B/4096B
Partition Table: gpt
Number Start End Size File system Name Flags
1 2097kB 4398GB 4398GB ext4 primary
(parted) unit s
(parted) print
Model: Linux device-mapper (multipath) (dm)
Disk /dev/mapper/backup1: 21474836480s
Sector size (logical/physical): 512B/4096B
Partition Table: gpt
Number Start End Size File system Name Flags
1 4096s 8589930495s 8589926400s ext4 primary
(parted) rm 1
(parted) mkpart primary 4096s 100%
(parted) print
Model: Linux device-mapper (multipath) (dm)
Disk /dev/mapper/backup1: 21474836480s
Sector size (logical/physical): 512B/4096B
Partition Table: gpt
Number Start End Size File system Name Flags
1 4096s 21474832383s 21474828288s ext4 primary
[root@TRAC1 ~]# e2fsck -f /dev/mapper/backup1p1
e2fsck 1.41.12 (17-May-2010)
Pass 1: Checking inodes, blocks, and sizes
Pass 2: Checking directory structure
Pass 3: Checking directory connectivity
Pass 4: Checking reference counts
Pass 5: Checking group summary information
/dev/mapper/backup1p1: 27/268435456 files (22.2% non-contiguous), 509820413/1073740800 blocks
ASM 磁盘扩展
[root@TRAC1 ~]# resize2fs /dev/mapper/backup1p1
resize2fs 1.41.12 (17-May-2010)
Resizing the filesystem on /dev/mapper/backup1p1 to 2684353536 (4k) blocks.
The filesystem on /dev/mapper/backup1p1 is now 2684353536 blocks long.
7.ASM 磁盘扩展
alter diskgroup SASDATA resize disk SASDATA_0004;
注意：ASM 单盘不支持超过2T
7. 删除磁盘步骤-ASM
1. 查看现有磁盘
SQL> col group_name format a20;
SQL> col disk_name format a20;
SQL> col path format a20;
SQL> set linesize 200;
SQL> SELECT b.NAME as group_name, a.name as disk_name, a.PATH,a.total_mb,a.free_mb
2 FROM V$asm_disk a, v$asm_diskgroup b
3 where a.group_number = b.group_number;
GROUP_NAME DISK_NAME PATH TOTAL_MB FREE_MB
-------------------- -------------------- -------------------- ---------- ----------
VOT_MIRROR VOT_MIRROR_0002 /dev/asm_sas1p3 14306 14029
VOT_MIRROR VOT_MIRROR_0001 /dev/asm_sas1p2 14304 14028
VOT_MIRROR VOT_MIRROR_0000 /dev/asm_sas1p1 14305 14030
SASDATA SASDATA_0003 /dev/asm_sas5 2097152 1766484
SASDATA SASDATA_0001 /dev/asm_sas3 2097152 1766485
SASDATA SASDATA_0000 /dev/asm_sas2 2097152 1766484
SASDATA SASDATA_0002 /dev/asm_sas4 2097152 1766485
VOT VOT_0000 /dev/asm_ssd1p3 14306 13997
VOT VOT_0002 /dev/asm_ssd1p1 14305 13996
VOT VOT_0001 /dev/asm_ssd1p2 14304 13996
SSDDATA SSDDATA_0003 /dev/asm_ssd5 2097152 1858377
GROUP_NAME DISK_NAME PATH TOTAL_MB FREE_MB
-------------------- -------------------- -------------------- ---------- ----------
SSDDATA SSDDATA_0001 /dev/asm_ssd3 2097152 1858378
SSDDATA SSDDATA_0002 /dev/asm_ssd4 2097152 1858377
SSDDATA SSDDATA_0000 /dev/asm_ssd2 2097152 1858377
SSDDATA SSDDATA_0004 /dev/asm_ssd6 2097152 1858381
SASDATA SASDATA_0004 /dev/asm_sas6 2097152 1766490
16 rows selected.
2. 确定要删除的磁盘
SQL> alter diskgroup SASDATA drop disk SASDATA_0004 rebalance power 10;
SQL> alter diskgroup SSDDATA drop disk SSDDATA_0004 rebalance power 10;
select * from v$asm_operation; 注意：每次只能一个operation 存在
3.
3. 修改/etc/multipath.conf 和/etc/udev/rules.d/99-oracle-asmdevices.rules, 去掉对应的磁盘，节点1 和2 需要同步
4.dell storage management 里面删除对应的卷和映射
来自 “ ITPUB博客 ” ，链接：https://blog.itpub.net/69976387/viewspace-2706571/ ，如需转载，请注明出处，否则将追究法律责任。
最新文章
- ORA-39082: Object type ALTER_FUNCTION:"XXXX"."F_cccc" created with compilation
- ORA-00445: background process "m000" did not start after 120 seconds host:xxxx
- MYSQL : ERROR 1045 (28000): Access denied for user 'root'@'localhost'
- ORA-00235: control file read without a lock inconsistent due to concurrent updat
- ORA-31693 ORA-00001
- 由于IMPDP...APPEND引起的 enq: TM – contention
- ORACLE-RMAN自动备份和恢复
- Oracle 11g RAC + DG安装详解--05
- Oracle 11g RAC + DG安装详解--04
- Oracle 11g RAC + DG安装详解--03
