20160322 RAC 安装教训


RHEL6u5 11g rac  hds 存储lun被oracle识别问题处理
[root@jkddgldb1 ~]# cat /etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6


# Public Network - (bond0)
10.64.51.6       jkddgldb1
10.64.51.7       jkddgldb2
# Public Virtual IP (VIP) addresses - (bond0)
10.64.51.8      jkddgldb1-vip
10.64.51.9      jkddgldb2-vip
# Private Interconnect - (em3)
192.168.1.6  jkddgldb1-priv
192.168.1.7  jkddgldb2-priv
# Scan IP
10.64.51.10  jkddgldb-scan
[root@jkddgldb1 ~]# ifconfig  -a | more
bond0     Link encap:Ethernet  HWaddr 14:18:77:3C:E9:CC  
          inet addr:10.64.51.6  Bcast:10.64.51.15  Mask:255.255.255.240
          inet6 addr: fe80::1618:77ff:fe3c:e9cc/64 Scope:Link
          UP BROADCAST RUNNING MASTER MULTICAST  MTU:1500  Metric:1
          RX packets:1846 errors:0 dropped:0 overruns:0 frame:0
          TX packets:637 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0 
          RX bytes:251107 (245.2 KiB)  TX bytes:89349 (87.2 KiB)


bond0:1   Link encap:Ethernet  HWaddr 14:18:77:3C:E9:CC  
          inet addr:10.64.51.8  Bcast:10.64.51.15  Mask:255.255.255.240
          UP BROADCAST RUNNING MASTER MULTICAST  MTU:1500  Metric:1


em1       Link encap:Ethernet  HWaddr 14:18:77:3C:E9:CC  
          UP BROADCAST RUNNING SLAVE MULTICAST  MTU:1500  Metric:1
          RX packets:1219 errors:0 dropped:0 overruns:0 frame:0
          TX packets:637 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000 
          RX bytes:150705 (147.1 KiB)  TX bytes:89349 (87.2 KiB)
          Interrupt:35 


em2       Link encap:Ethernet  HWaddr 14:18:77:3C:E9:CC  
          UP BROADCAST RUNNING SLAVE MULTICAST  MTU:1500  Metric:1
          RX packets:627 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000 
          RX bytes:100402 (98.0 KiB)  TX bytes:0 (0.0 b)
          Interrupt:38 


em3       Link encap:Ethernet  HWaddr 14:18:77:3C:E9:CE  
          inet addr:192.168.1.6  Bcast:192.168.1.255  Mask:255.255.255.0
          inet6 addr: fe80::1618:77ff:fe3c:e9ce/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:69398 errors:0 dropped:0 overruns:0 frame:0
          TX packets:37522 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000 
          RX bytes:66852884 (63.7 MiB)  TX bytes:23598165 (22.5 MiB)
          Interrupt:34 




注意开启LINUX 异步IO
cat cat /proc/slabinfo | grep kio
kioctx               106    150    384   10    1 : tunables   54   27    8 : slabdata     15     15      0
kiocb                 10     45    256   15    1 : tunables  120   60    8 : slabdata      2      3      0


hosts 表 和主机名要小写 主机名不能下划线  
11G RAC 的信任关系也不需要手工配置了 图形界面可以自动配置,但是做
gird用户安装之前的检查
./runcluvfy.sh stage -pre crsinst -n jkddgldb1,njkddgldb2 -fixup -verbose >check.log 
检查还是需要对grid用户建立信任关系


出现以下警告可以忽略
1,交换空间 检查失败
2,NTP 配置文件
3,/etc/resolv.conf


节点1/节点2
su - grid && mkdir .ssh && cd .ssh
ssh-keygen -t rsa
ssh-keygen -t dsa 
节点1
scp id_rsa.pub jkddgldb2:~/.ssh/jkddgldb1.rsa
scp id_dsa.pub jkddgldb2:~/.ssh/jkddgldb1.dsa
scp jkddgldb2:~/.ssh/id_rsa.pub jkddgldb2.rsa
scp jkddgldb2:~/.ssh/id_dsa.pub jkddgldb2.dsa
cat jkddgldb2.rsa >> authorized_keys
cat jkddgldb2.dsa >> authorized_keys
cat id_rsa.pub >> authorized_keys
cat id_dsa.pub >> authorized_keys
节点2
cat jkddgldb1.rsa >> authorized_keys
cat jkddgldb1.dsa >> authorized_keys 
cat id_rsa.pub >> authorized_keys 
cat id_dsa.pub >> authorized_keys


检验grid用户信任关系是否配置成功
ping jkddgldb1 date
ping jkddgldb2 date
ping jkddgldb1-priv date
ping jkddgldb2-priv date




HDS lun处理


[root@jkddgldb1 ~]# cd /opt/DynamicLinkManager/bin/
[root@jkddgldb1 bin]# ./dlnkmgr view -lu
Product       : VSP_G1000
SerialNumber  : 50725           
LUs           : 19


iLU    HDevName Device   PathID Status     
000000 sddlmac  /dev/sdb 000000 Online     
000001 sddlmab  /dev/sdc 000001 Online     
000002 sddlmaj  /dev/sdd 000002 Online     
000003 sddlmaa  /dev/sde 000003 Online     
000004 sddlmag  /dev/sdf 000004 Online     
000005 sddlmad  /dev/sdg 000005 Online     
000006 sddlmae  /dev/sdh 000006 Online     
000007 sddlmam  /dev/sdi 000007 Online     
000008 sddlmaf  /dev/sdj 000008 Online     
000009 sddlmal  /dev/sdk 000009 Online     
00000A sddlmai  /dev/sdl 000010 Online     
00000B sddlmak  /dev/sdm 000011 Online     
00000C sddlman  /dev/sdn 000012 Online     
00000D sddlmah  /dev/sdo 000013 Online     
00000E sddlmao  /dev/sdp 000014 Online     
00000F sddlmap  /dev/sdq 000015 Online     
001000 sddlmba  /dev/sdr 000016 Online     
001001 sddlmbb  /dev/sds 000017 Online     
001002 sddlmbc  /dev/sdt 000018 Online     
KAPL01001-I The HDLM command completed normally. Operation name = view(-vstv), completion time = 2016/03/22 18:14:17




[root@jkddgldb1 bin]# cat /etc/udev/rules.d/50.udev.rules 
SUBSYSTEM=="block", KERNEL=="sddlmaa", GROUP="asmadmin", OWNER="grid", MODE="0660"
SUBSYSTEM=="block", KERNEL=="sddlmab", GROUP="asmadmin", OWNER="grid", MODE="0660"
SUBSYSTEM=="block", KERNEL=="sddlmac", GROUP="asmadmin", OWNER="grid", MODE="0660"
SUBSYSTEM=="block", KERNEL=="sddlmad", GROUP="asmadmin", OWNER="grid", MODE="0660"
SUBSYSTEM=="block", KERNEL=="sddlmaf", GROUP="asmadmin", OWNER="grid", MODE="0660"
SUBSYSTEM=="block", KERNEL=="sddlmag", GROUP="asmadmin", OWNER="grid", MODE="0660"
SUBSYSTEM=="block", KERNEL=="sddlmae", GROUP="asmadmin", OWNER="grid", MODE="0660"
SUBSYSTEM=="block", KERNEL=="sddlmah", GROUP="asmadmin", OWNER="grid", MODE="0660"
SUBSYSTEM=="block", KERNEL=="sddlmaj", GROUP="asmadmin", OWNER="grid", MODE="0660"
SUBSYSTEM=="block", KERNEL=="sddlmak", GROUP="asmadmin", OWNER="grid", MODE="0660"
SUBSYSTEM=="block", KERNEL=="sddlmam", GROUP="asmadmin", OWNER="grid", MODE="0660"
SUBSYSTEM=="block", KERNEL=="sddlmao", GROUP="asmadmin", OWNER="grid", MODE="0660"
SUBSYSTEM=="block", KERNEL=="sddlmal", GROUP="asmadmin", OWNER="grid", MODE="0660"
SUBSYSTEM=="block", KERNEL=="sddlmai", GROUP="asmadmin", OWNER="grid", MODE="0660"
SUBSYSTEM=="block", KERNEL=="sddlman", GROUP="asmadmin", OWNER="grid", MODE="0660"
SUBSYSTEM=="block", KERNEL=="sddlmap", GROUP="asmadmin", OWNER="grid", MODE="0660"
SUBSYSTEM=="block", KERNEL=="sddlmba", GROUP="asmadmin", OWNER="grid", MODE="0660"
SUBSYSTEM=="block", KERNEL=="sddlmbb", GROUP="asmadmin", OWNER="grid", MODE="0660"
SUBSYSTEM=="block", KERNEL=="sddlmbc", GROUP="asmadmin", OWNER="grid", MODE="0660"


start_udev


此时安装并配置grid的时候就能看到可用磁盘了




系统重启后集群不来了
现象是
[grid@jkddgldb2 ~]$ crsctl check crs
CRS-4638: Oracle High Availability Services is online
CRS-4535: Cannot communicate with Cluster Ready Services
CRS-4530: Communications failure contacting Cluster Synchronization Services daemon
CRS-4534: Cannot communicate with Event Manage
[root@jkddgldb1 bin]# ./crsctl stop crs
CRS-2796: The command may not proceed when Cluster Ready Services is not running
CRS-4687: Shutdown command has completed with errors.
CRS-4000: Command Stop failed, or completed with errors.
[root@jkddgldb1 bin]# ./crsctl start crs
CRS-4640: Oracle High Availability Services is already active
CRS-4000: Command Start failed, or completed with errors.




原来em3 地址没有了
[root@jkddgldb1 ~]# ifconfig  -a | more
bond0     Link encap:Ethernet  HWaddr 14:18:77:3C:E9:CC  
          inet addr:10.64.51.6  Bcast:10.64.51.15  Mask:255.255.255.240
          inet6 addr: fe80::1618:77ff:fe3c:e9cc/64 Scope:Link
          UP BROADCAST RUNNING MASTER MULTICAST  MTU:1500  Metric:1
          RX packets:240 errors:0 dropped:0 overruns:0 frame:0
          TX packets:135 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0 
          RX bytes:25737 (25.1 KiB)  TX bytes:20238 (19.7 KiB)


em1       Link encap:Ethernet  HWaddr 14:18:77:3C:E9:CC  
          UP BROADCAST RUNNING SLAVE MULTICAST  MTU:1500  Metric:1
          RX packets:138 errors:0 dropped:0 overruns:0 frame:0
          TX packets:135 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000 
          RX bytes:12988 (12.6 KiB)  TX bytes:20238 (19.7 KiB)
          Interrupt:35 


em2       Link encap:Ethernet  HWaddr 14:18:77:3C:E9:CC  
          UP BROADCAST RUNNING SLAVE MULTICAST  MTU:1500  Metric:1
          RX packets:102 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000 
          RX bytes:12749 (12.4 KiB)  TX bytes:0 (0.0 b)
          Interrupt:38 


em3       Link encap:Ethernet  HWaddr 14:18:77:3C:E9:CE  
          BROADCAST MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000 
          RX bytes:0 (0.0 b)  TX bytes:0 (0.0 b)
          Interrupt:34 




处理 系统开机自动启动
[root@jkddgldb2 bin]# vi /etc/rc.local 
#!/bin/sh
#
# This script will be executed *after* all the other init scripts.
# You can put your own initialization stuff in here if you don't
# want to do the full Sys V style init stuff.


touch /var/lock/subsys/local
ifenslave bond0 em1 em2


ifup em3