ITPub博客

首页 > 数据库 > Oracle > Oracle 11g RAC + DG安装详解--01

Oracle 11g RAC + DG安装详解--01

原创 Oracle 作者:pennymeng 时间:2020-07-23 17:20:28 0 删除 编辑

一、准备工作

1. 安装操作系统Centos6.9 ,略。

2. 配置ip 和主机名

ip 配置如下:

节点1

[root@localhost ~]# vi /etc/hosts

#public ip (bond0)

192.168.1.1  TRAC1

192.168.1.2  TRAC2

 

#private ip (em3)

1    TRAC1-priv

10.1.1.2   TRAC2-priv

 

#public vip

192.168.1.3  TRAC1-vip

192.168.1.4  TRAC2-vip

 

#scan ip

192.168.1.5  TRAC

 

节点2

[root@localhost ~]# vi /etc/hosts

#public ip (bond0)

192.168.1.1  TRAC1

192.168.1.2  TRAC2

 

#private ip (em3)

10.1.1.1   TRAC1-priv

10.1.1.2   TRAC2-priv

 

#public vip

192.168.1.3  TRAC1-vip

192.168.1.4  TRAC2-vip

 

#scan ip

192.168.1.5  TRAC

 

3. 配置主机名:

节点1

[root@localhost ~]# hostname TRAC1

[root@localhost ~]# vi /etc/sysconfig/network

NETWORKING=yes

HOSTNAME=TRAC1

 

节点2

[root@localhost ~]# hostname TRAC2

[root@localhost ~]# vi /etc/sysconfig/network

NETWORKING=yes

HOSTNAME=TRAC2

 

检查HOSTS 配置:以上的hostname IP ping 一遍(此时vip ip host 应该是不通的)

 

4. 配置IP

节点1

[root@localhost network-scripts]# vi ifcfg-em3

DEVICE=em3

HWADDR=18:66:DA:EF:22:E2

TYPE=Ethernet

UUID=c3096b1c-94dd-4eb2-88a0-e760eb95f23a

>

NM_CONTROLLED=yes

BOOTPROTO=none

IPADDR=10.1.1.1

NETMASK=255.255.255.0

IPV6INIT=no

USERCTL=no

 

[root@localhost network-scripts]# service network restart

节点2

[root@localhost network-scripts]# vi ifcfg-em3

DEVICE=em3

HWADDR=18:66:DA:EF:30:2A

TYPE=Ethernet

UUID=74c89752-ae33-445b-88de-7125531594f8

>

NM_CONTROLLED=yes

BOOTPROTO=none

IPADDR=10.1.1.2

NETMASK=255.255.255.0

IPV6INIT=no

USERCTL=no

 

[root@localhost network-scripts]# service network restart

 

5. 两节点配置ntp 服务

crontab -l

*/5 * * * * /usr/sbin/ntpdate 服务器IP >> /root/sync_date.log 2>&1

mv /etc/ntp.conf /etc/ntp.conf.old          

 

6. 配置DNS nameserver

cp /etc/resolv.conf /etc/resolv.conf.bak

vi /etc/resolv.conf

nameserver 192.168.1.245

 

 

 

7. 修改内核参数

分别在两节点上执行:

cp /etc/sysctl.conf /etc/sysctl.conf.bak

vi /etc/sysctl.conf 末尾

# 注释掉kernel.shmmax kernel.shmall 两行

kernel.shmall = 11111                              ( 单位:字节  物理内存/4096)

kernel.shmmax = 11111                  (单位:字节  物理内存-1

添加:

fs.aio-max-nr = 1048576

fs.file-max = 6815744

kernel.shmmax = 135221043199

kernel.shmmni = 4096

kernel.shmall = 33012950

kernel.sem = 250 32000 100 128

net.ipv4.ip_local_port_range = 9000 65500

net.core.rmem_default = 262144

net.core.rmem_max = 4194304

net.core.wmem_default = 262144

net.core.wmem_max = 1048576

以上参数为使用oracle yum 自动配置的参数

重新加载生效:

[root@TRAC1 ~]# sysctl -p

 

8. 安装必要的包

两节点分别执行如下三步:

     配置yum

[root@TRAC1 grid]# cd /etc/yum.repos.d/

[root@TRAC1 yum.repos.d]# cp CentOS-Base.repo CentOS-Base.repo.bak

[root@TRAC1 yum.repos.d]# vi /etc/yum.repos.d/CentOS-Base.repo

[base]

name=CentOS-$releasever - Base

baseurl=http://192.168.1.10/centos6.9

enabled=1

gpgcheck=0

gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6

 

[root@TRAC1 yum.repos.d]# yum clean all

Loaded plugins: fastestmirror, refresh-packagekit, security

Cleaning repos: base

Cleaning up Everything

Cleaning up list of fastest mirrors

[root@TRAC1 yum.repos.d]# yum makecache

Loaded plugins: fastestmirror, refresh-packagekit, security

Determining fastest mirrors

Metadata Cache Created

…….

[root@TRAC1 yum.repos.d]#

 

     检查哪些包没安装

for i in binutils compat-gcc-34 compat-libstdc++-296 control-center \

  gcc gcc-c++ glibc glibc-common glibc-devel libaio libgcc elfutils-libelf-devel \

  libstdc++ libstdc++-devel libXp make openmotif22 setarch \

  compat-libstdc++-33 libaio-devel sysstat unixODBC unixODBC-devel \

compat-libcap1 ksh tigervnc-server

do

  rpm -q $i &>/dev/null || F="$F $i"

done ;echo $F;unset F

 

     使用yum 安装这些包

[root@TRAC1 yum.repos.d]# yum install -y compat-gcc-34 compat-libstdc++-296 gcc gcc-c++ elfutils-libelf-devel libstdc++-devel libXp openmotif22 setarch compat-libstdc++-33 libaio-devel unixODBC unixODBC-devel compat-libcap1 ksh tigervnc-server

 

9. 修改两节点的oracle 用户限制

cp /etc/security/limits.conf /etc/security/limits.conf.bak

vi /etc/security/limits.conf 末尾添加:

oracle   soft   nofile   2047

oracle   hard   nofile   65536

oracle   soft   nproc    2047

oracle   hard   nproc    16384

oracle   soft   stack    10240

oracle   hard   stack    32768

grid   soft   nofile   2047

grid   hard   nofile   65536

grid   soft   nproc    2047

grid   hard   nproc    16384

grid   soft   stack    10240

grid   hard   stack    32768

 

10. 修改两节点的/etc/pam.d/login

cp /etc/pam.d/login /etc/pam.d/login.bak

vi /etc/pam.d/login 末尾添加:

session    required     /lib64/security/pam_limits.so

 

11. 关闭两节点的防火墙和selinux

[root@ ~]#

service iptables stop

chkconfig iptables off

setenforce 0

getenforce

cp /etc/selinux/config /etc/selinux/config.bak

vi /etc/selinux/config 确保以下内容

SELINUX=disabled

[root@ ~]# cat /etc/selinux/config

# This file controls the state of SELinux on the system.

# SELINUX= can take one of these three values:

#       enforcing - SELinux security policy is enforced.

#       permissive - SELinux prints warnings instead of enforcing.

#       disabled - SELinux is fully disabled.

SELINUX=disabled

# SELINUXTYPE= type of policy in use. Possible values are:

#       targeted - Only targeted network daemons are protected.

#       strict - Full SELinux protection.

SELINUXTYPE=targeted

 

12. 更改 安装所有者的 ulimit 设置

两节点都要执行:

vi /etc/profile 末尾添加:

if [ /$USER = "oracle" ] || [ $USER = "grid" ]; then

    if [ /$SHELL = "/bin/ksh" ]; then

        ulimit -p 16384

        ulimit -n 65536

    else

        ulimit -u 16384 -n 65536

    fi

    umask 022

fi

 

13. 两节点建立必要的组和用户

[root@ ~]#

groupadd -g 501 oinstall

groupadd -g 502 dba

groupadd -g 503 oper

groupadd -g 504 asmadmin

groupadd -g 505 asmdba

groupadd -g 506 asmoper

 

useradd -u 501 -g oinstall -G dba,oper,asmdba oracle

useradd -u 502 -g oinstall -G asmadmin,asmoper,asmdba grid

echo "123456"|passwd --stdin oracle

echo "123456"|passwd --stdin grid

 

用户和组的说明:

14. 两节点建立安装目录

[root@ ~]#

chown -R oracle:oinstall /home/oracle

chown -R grid:oinstall /home/grid

---Grid/oracle Infrastructure BASE:

mkdir -p /home/app/{oracle,grid}

---Grid Infrastructure HOME:

mkdir -p /home/app/grid/{base,crs}

chown -R grid:oinstall /home/app/grid

chown -R oracle:oinstall /home/app/oracle

chmod 755 /home/oracle

chmod 755 /home/grid

chmod -R 775 /home/app

 

 

15. 两节点设置oracle grid 用户的环境变量

su - oracle

vi .bash_profile 末尾添加:

export ORACLE_BASE=/home/app/oracle

export ORACLE_HOME=/home/app/oracle/product/11.2.0

export ORACLE_SID=TRAC1

export PATH=$PATH:$ORACLE_HOME/bin

export LD_LIBRARY_PATH=$ORACLE_HOME/lib:/lib:/usr/lib

export NLS_LANG=AMERICAN_AMERICA.UTF8

 

su - grid

vi .bash_profile 末尾添加:

export ORACLE_BASE=/home/app/grid/base

export ORACLE_HOME=/home/app/grid/crs

export ORACLE_SID=+ASM1

export PATH=$PATH:$ORACLE_HOME/bin

export LD_LIBRARY_PATH=$ORACLE_HOME/lib:/lib:/usr/lib

export NLS_LANG=AMERICAN_AMERICA.UTF8

 

16. 两节点安装asm

①查看本机内核版本:

[root@rac1 u01]# uname -rm

2.6.18-274.el5 x86_64

②到以下网址下载对应操作系统版本的三个asm 包:

     把下载下来的asm 包上传到两个节点,并安装,注意按照以下安装顺序

cd /home/software/rpm

[root@TRAC1 rpm]# rpm -ivh oracleasm-support-2.1.8-1.el6.x86_64.rpm

warning: oracleasm-support-2.1.8-1.el6.x86_64.rpm: Header V3 RSA/SHA256 Signature, key ID ec551f03: NOKEY

Preparing...                ########################################### [100%]

   1:oracleasm-support      ########################################### [100%]

[root@TRAC1 rpm]# rpm -ivh kmod-oracleasm-2.0.8-15.el6_9.x86_64.rpm

warning: kmod-oracleasm-2.0.8-15.el6_9.x86_64.rpm: Header V3 RSA/SHA1 Signature, key ID c105b9de: NOKEY

Preparing...                ########################################### [100%]

   1:kmod-oracleasm         ########################################### [100%]

[root@TRAC1 rpm]# rpm -ivh oracleasmlib-2.0.4-1.el6.x86_64.rpm

warning: oracleasmlib-2.0.4-1.el6.x86_64.rpm: Header V3 RSA/SHA256 Signature, key ID ec551f03: NOKEY

Preparing...                ########################################### [100%]

   1:oracleasmlib           ########################################### [100%]

[root@TRAC1 rpm]#

 

16. 两节点分别配置asm

[root@ ~]# /etc/init.d/oracleasm configure

Configuring the Oracle ASM library driver.

 

This will configure the on-boot properties of the Oracle ASM library

driver.  The following questions will determine whether the driver is

loaded on boot and what permissions it will have.  The current values

will be shown in brackets ('[]').  Hitting <ENTER> without typing an

answer will keep that current value.  Ctrl-C will abort.

 

Default user to own the driver interface []: grid

Default group to own the driver interface []: asmadmin

Start Oracle ASM library driver on boot (y/n) [n]: y

Scan for Oracle ASM disks on boot (y/n) [y]: y

Writing Oracle ASM library driver configuration: done

Initializing the Oracle ASMLib driver:                     [  OK  ]

Scanning the system for Oracle ASMLib disks:             [  OK  ]

 

17. 共享磁盘( 重新) 分区

 

18. 创建asm 磁盘:

以下在任意一个节点执行就行了:

 

19. 扫描磁盘asm 磁盘,检查有哪些asm 磁盘:

每个节点都要执行:

 

20. ssh 双机互信

oracle/grid 用户均需要做

节点1

ssh-keygen -t rsa

ssh-keygen -t dsa 

ssh-copy-id -i .ssh/id_dsa.pub TRAC2

ssh-copy-id -i .ssh/id_rsa.pub TRAC2

 

节点2

ssh-keygen -t rsa

ssh-keygen -t dsa 

ssh-copy-id -i .ssh/id_dsa.pub TRAC1

ssh-copy-id -i .ssh/id_rsa.pub TRAC1

cat .ssh/*.pub >> .ssh/authorized_keys

 

节点1

cat .ssh/*.pub >> .ssh/authorized_keys

 

ssh TRAC1 date && ssh TRAC1-priv date && ssh TRAC2 date && ssh TRAC2-priv date

 

 

21. NFS 上建立disk (临时测试使用NFS

mount -t nfs -o rw,bg,hard,nointr,rsize=32768,wsize=32768,tcp,noac,vers=3,timeo=600,actimeo=0  188.188.42.71:/data/oracle_test /data

mount -t nfs 166.188.30.19:/data/nfs /nfs

 

dd if=/dev/zero of=/data/oradata/votdisk1 bs=8192 count=100000

 

cd /data/oradata/

chmod 660 *

chown grid:asmadmin *

 

22. 存储配置

1. 多路径安装

yum install -y device-mapper-multi*

chkconfig --level 2345 multipathd on

service multipathd start

multipath -ll

cp /usr/share/doc/device-mapper-multipath-0.4.9/multipath.conf /etc/multipath.conf

service multipathd restart

2.multipath.conf 配置

节点 1 2 需要同步一致

vi /etc/multipath.conf

# This is a basic configuration file with some examples, for device mapper

# multipath.

# For a complete list of the default configuration values, see

# /usr/share/doc/device-mapper-multipath-0.4.9/multipath.conf.defaults

# For a list of configuration options with descriptions, see

# /usr/share/doc/device-mapper-multipath-0.4.9/multipath.conf.annotated

#

# REMEMBER: After updating multipath.conf, you must run

#

# service multipathd reload

#

# for the changes to take effect in multipathd

 

## By default, devices with vendor = "IBM" and product = "S/390.*" are

## blacklisted. To enable mulitpathing on these devies, uncomment the

## following lines.

#blacklist_exceptions {

#       device {

#               vendor  "IBM"

#               product "S/390.*"

#       }

#}

 

## Use user friendly names, instead of using WWIDs as names.

#defaults {

#       user_friendly_names yes

#}

##

## Here is an example of how to configure some standard options.

##

#

defaults {

        udev_dir                /dev

        polling_interval        5

        path_selector           "round-robin 0"

        path_grouping_policy    multibus

        getuid_callout          "/lib/udev/scsi_id --whitelisted --device=/dev/%n"

        prio                    alua

        path_checker            readsector0

        rr_min_io               100

        max_fds                 8192

        rr_weight               priorities

        failback                immediate

        no_path_retry           fail

        user_friendly_names     yes

}

##

## The wwid line in the following blacklist section is shown as an example

## of how to blacklist devices by wwid.  The 2 devnode lines are the

## compiled in default blacklist. If you want to blacklist entire types

## of devices, such as all scsi devices, you should use a devnode line.

## However, if you want to blacklist specific devices, you should use

## a wwid line.  Since there is no guarantee that a specific device will

## not change names on reboot (from /dev/sda to /dev/sdb for example)

## devnode lines are not recommended for blacklisting specific devices.

##

#blacklist {

#       wwid 26353900f02796769

#       devnode "^(ram|raw|loop|fd|md|dm-|sr|scd|st)[0-9]*"

#       devnode "^hd[a-z]"

#}

#multipaths {

#       multipath {

#               wwid                    3600508b4000156d700012000000b0000

#               alias                   yellow

#               path_grouping_policy    multibus

#               path_checker            readsector0

#               path_selector           "round-robin 0"

#               failback                manual

#               rr_weight               priorities

#               no_path_retry           5

#       }

#       multipath {

#               wwid                    1DEC_____321816758474

#               alias                   red

#       }

#}

#devices {

#       device {

#               vendor                  "COMPAQ  "

#               product                 "HSV110 (C)COMPAQ"

#               path_grouping_policy    multibus

#               getuid_callout          "/lib/udev/scsi_id --whitelisted --device=/dev/%n"

#               path_checker            readsector0

#               path_selector           "round-robin 0"

#               hardware_handler        "0"

#               failback                15

#               rr_weight               priorities

#               no_path_retry           queue

#       }

#       device {

#               vendor                  "COMPAQ  "

#               product                 "MSA1000         "

#               path_grouping_policy    multibus

#       }

#}

multipaths {

       multipath {

               wwid                    36000d310013faa000000000000000070

               alias                   racsas1

       }

       multipath {

               wwid                    36000d310013faa00000000000000007a

               alias                   racsas2

       }

       multipath {

               wwid                    36000d310013faa00000000000000007b

               alias                   racsas3

       }

       multipath {

               wwid                    36000d310013faa00000000000000007c

               alias                   racsas4

       }

       multipath {

               wwid                    36000d310013faa00000000000000007d

               alias                   racsas5

       }

       multipath {

               wwid                    36000d310013faa00000000000000008a

               alias                   racssd1

       }

           multipath {

               wwid                    36000d310013faa000000000000000086

               alias                   racssd2

       }

       multipath {

               wwid                    36000d310013faa000000000000000087

               alias                   racssd3

       }

       multipath {

               wwid                    36000d310013faa000000000000000088

               alias                   racssd4

       }

       multipath {

               wwid                    36000d310013faa000000000000000089

               alias                   racssd5

       }

       multipath {

               wwid                    36000d310038170000000000000000007

               alias                   backup1

       }

       multipath {

               wwid                    36000d31003817200000000000000000a

               alias                   dgdata1

       }

}

 

3. 多路径磁盘分区

仅节点2

[root@TRAC2 ~]# parted /dev/mapper/racsas1

(parted) mkpart primary 0% 100%

(parted) select /dev/mapper/racsas2 mklabel gpt   mkpart primary 0% 100%

 

(parted) select /dev/mapper/racsas3 mklabel gpt   mkpart primary 0% 100%

 

(parted) select /dev/mapper/racsas4 mklabel gpt   mkpart primary 0% 100%

(parted) select /dev/mapper/racssd1

(parted) mklabel gpt                                                     

(parted) mkpart primary 0 10G

(parted) mkpart primary                                                  

File system type?  [ext2]? ext3                                          

Start?                                                                   

Start? 10G                                                               

End? 20G   

(parted) mkpart primary                                                   

File system type?  [ext2]? ext3                                          

Start? 20G

End? 30G                                                                 

(parted) mkpart primary

File system type?  [ext2]? ext3                                           

Start? 30G                                                               

End? 100%   

(parted) select /dev/mapper/racssd2

Using /dev/mapper/racssd2

(parted) mklabel gpt                                                      

(parted) mkpart primary 0% 100%                                          

(parted) select /dev/mapper/racssd3 mklabel gpt  mkpart primary 0% 100% 

Using /dev/mapper/racssd3

(parted) select /dev/mapper/racssd4 mklabel gpt  mkpart primary 0% 100% 

Using /dev/mapper/racssd4

 

节点1

[root@TRAC1 ~]# service multipathd restart

ok

Stopping multipathd daemon: [  OK  ]

Starting multipathd daemon: [  OK  ]

 

4.udev 配置

节点1 2

[root@TRAC2 ~]# ll /dev/mapper/

lrwxrwxrwx 1 root root      8 Jun 13 14:21 racsas1 -> ../dm-13

lrwxrwxrwx 1 root root      8 Jun 13 14:21 racsas1p1 -> ../dm-16

lrwxrwxrwx 1 root root      7 Jun 13 14:21 racsas2 -> ../dm-5

lrwxrwxrwx 1 root root      8 Jun 13 14:21 racsas2p1 -> ../dm-17

lrwxrwxrwx 1 root root      7 Jun 13 14:21 racsas3 -> ../dm-4

lrwxrwxrwx 1 root root      8 Jun 13 14:21 racsas3p1 -> ../dm-18

lrwxrwxrwx 1 root root      7 Jun 13 14:21 racsas4 -> ../dm-3

lrwxrwxrwx 1 root root      8 Jun 13 14:21 racsas4p1 -> ../dm-19

lrwxrwxrwx 1 root root      8 Jun 13 14:21 racssd1 -> ../dm-14

lrwxrwxrwx 1 root root      8 Jun 13 14:21 racssd1p1 -> ../dm-20

lrwxrwxrwx 1 root root      8 Jun 13 14:21 racssd1p2 -> ../dm-21

lrwxrwxrwx 1 root root      8 Jun 13 14:21 racssd1p3 -> ../dm-22

lrwxrwxrwx 1 root root      8 Jun 13 14:21 racssd1p4 -> ../dm-23

lrwxrwxrwx 1 root root      7 Jun 13 14:21 racssd2 -> ../dm-7

lrwxrwxrwx 1 root root      8 Jun 13 14:21 racssd2p1 -> ../dm-24

lrwxrwxrwx 1 root root      7 Jun 13 14:21 racssd3 -> ../dm-2

lrwxrwxrwx 1 root root      8 Jun 13 14:21 racssd3p1 -> ../dm-25

lrwxrwxrwx 1 root root      7 Jun 13 14:21 racssd4 -> ../dm-6

lrwxrwxrwx 1 root root      8 Jun 13 14:21 racssd4p1 -> ../dm-26

 

[root@TRAC2 ~]# for i in `seq 16 26`;do printf "%s %s %s\n" "$i" "$(ll /dev/VolGroup/|grep dm-$i|awk '{print $9}')" "$(udevadm info --query=all --name=/dev/dm-$i | grep -i dm_uuid)";done

16  E: DM_UUID=part1-mpath-36000d310013fb0000000000000000015  - ARCH

17  E: DM_UUID=part1-mpath-36000d310013fb0000000000000000010  -SASDATA1

18  E: DM_UUID=part1-mpath-36000d310013fb0000000000000000012  -SASDATA2

19  E: DM_UUID=part1-mpath-36000d310013fb0000000000000000013  -SASDATA3

20  E: DM_UUID=part1-mpath-36000d310013fb0000000000000000016  -VOT1

21  E: DM_UUID=part2-mpath-36000d310013fb0000000000000000016  -VOT2

22  E: DM_UUID=part3-mpath-36000d310013fb0000000000000000016  -VOT3

23  E: DM_UUID=part4-mpath-36000d310013fb0000000000000000016  -FRA

24  E: DM_UUID=part1-mpath-36000d310013fb000000000000000000c  -SSDDATA1

25  E: DM_UUID=part1-mpath-36000d310013fb000000000000000000d  -SSDDATA2

26  E: DM_UUID=part1-mpath-36000d310013fb0000000000000000014  -SSDDATA3

 

[root@TRAC2 ~]# cd /etc/udev/rules.d/

vi /etc/udev/rules.d/99-oracle-asmdevices.rules

KERNEL=="dm-*",ENV{DM_UUID}=="part1-mpath-36000d310013fb0000000000000000015",NAME="asmdisk_arch",OWNER="grid",GROUP="asmadmin",MODE="0660"

KERNEL=="dm-*",ENV{DM_UUID}=="part1-mpath-36000d310013fb0000000000000000010",NAME="asmdisk_sasdata1",OWNER="grid",GROUP="asmadmin",MODE="0660"

KERNEL=="dm-*",ENV{DM_UUID}=="part1-mpath-36000d310013fb0000000000000000012",NAME="asmdisk_sasdata2",OWNER="grid",GROUP="asmadmin",MODE="0660"

KERNEL=="dm-*",ENV{DM_UUID}=="part1-mpath-36000d310013fb0000000000000000013",NAME="asmdisk_sasdata3",OWNER="grid",GROUP="asmadmin",MODE="0660"

KERNEL=="dm-*",ENV{DM_UUID}=="part1-mpath-36000d310013fb0000000000000000016",NAME="asmdisk_vot1",OWNER="grid",GROUP="asmadmin",MODE="0660"

KERNEL=="dm-*",ENV{DM_UUID}=="part2-mpath-36000d310013fb0000000000000000016",NAME="asmdisk_vot2",OWNER="grid",GROUP="asmadmin",MODE="0660"

KERNEL=="dm-*",ENV{DM_UUID}=="part3-mpath-36000d310013fb0000000000000000016",NAME="asmdisk_vot3",OWNER="grid",GROUP="asmadmin",MODE="0660"

KERNEL=="dm-*",ENV{DM_UUID}=="part4-mpath-36000d310013fb0000000000000000016",NAME="asmdisk_fra",OWNER="grid",GROUP="asmadmin",MODE="0660"

KERNEL=="dm-*",ENV{DM_UUID}=="part1-mpath-36000d310013fb000000000000000000c",NAME="asmdisk_ssddata1",OWNER="grid",GROUP="asmadmin",MODE="0660"

KERNEL=="dm-*",ENV{DM_UUID}=="part1-mpath-36000d310013fb000000000000000000d",NAME="asmdisk_ssddata2",OWNER="grid",GROUP="asmadmin",MODE="0660"

KERNEL=="dm-*",ENV{DM_UUID}=="part1-mpath-36000d310013fb0000000000000000014",NAME="asmdisk_ssddata3",OWNER="grid",GROUP="asmadmin",MODE="0660"

 

[root@TRAC2 rules.d]# udevadm control --reload-rules

[root@TRAC2 rules.d]# start_udev

 

[root@TRAC2 rules.d]# scp /etc/udev/rules.d/99-oracle-asmdevices.rules root@192.168.1.6:/etc/udev/rules.d/99-oracle-asmdevices.rules

 

节点1

[root@TRAC1 ~]# udevadm control --reload-rules

[root@TRAC1 ~]# start_udev

Starting udev: [  OK  ]

[root@TRAC1 ~]#

 

[root@TRAC1 ~]# ll /dev/mapper/

total 0

lrwxrwxrwx 1 root root      7 Jun 13 14:35 racsas1 -> ../dm-4

lrwxrwxrwx 1 root root     15 Jun 13 14:35 racsas1p1 -> ../asmdisk_arch

lrwxrwxrwx 1 root root      7 Jun 13 14:35 racsas2 -> ../dm-5

lrwxrwxrwx 1 root root     19 Jun 13 14:35 racsas2p1 -> ../asmdisk_sasdata1

lrwxrwxrwx 1 root root      7 Jun 13 14:35 racsas3 -> ../dm-8

lrwxrwxrwx 1 root root     19 Jun 13 14:35 racsas3p1 -> ../asmdisk_sasdata2

lrwxrwxrwx 1 root root      7 Jun 13 14:35 racsas4 -> ../dm-3

lrwxrwxrwx 1 root root     19 Jun 13 14:35 racsas4p1 -> ../asmdisk_sasdata3

lrwxrwxrwx 1 root root      8 Jun 13 14:35 racssd1 -> ../dm-11

lrwxrwxrwx 1 root root     15 Jun 13 14:35 racssd1p1 -> ../asmdisk_vot1

lrwxrwxrwx 1 root root     15 Jun 13 14:35 racssd1p2 -> ../asmdisk_vot2

lrwxrwxrwx 1 root root     15 Jun 13 14:35 racssd1p3 -> ../asmdisk_vot3

lrwxrwxrwx 1 root root     14 Jun 13 14:35 racssd1p4 -> ../asmdisk_fra

lrwxrwxrwx 1 root root      7 Jun 13 14:35 racssd2 -> ../dm-6

lrwxrwxrwx 1 root root     19 Jun 13 14:35 racssd2p1 -> ../asmdisk_ssddata1

lrwxrwxrwx 1 root root      7 Jun 13 14:35 racssd3 -> ../dm-2

lrwxrwxrwx 1 root root     19 Jun 13 14:35 racssd3p1 -> ../asmdisk_ssddata2

lrwxrwxrwx 1 root root      8 Jun 13 14:35 racssd4 -> ../dm-10

lrwxrwxrwx 1 root root     19 Jun 13 14:35 racssd4p1 -> ../asmdisk_ssddata3

 

5. 添加新磁盘步骤

1.Dell storage management 增加新卷并映射给主机

2. 主机通过下面命令扫描磁盘,节点1 2 均需

for i in `ls /sys/class/scsi_host/`; do echo "- - -" >> /sys/class/scsi_host/$i/scan ; done

3. 通过下面命令查看磁盘的wwid

scsi_id --whitelisted --replace-whitespace /dev/sd*

4. 修改/etc/multipath.conf 添加新磁盘,节点1 2 需要同步一致

5. 重新加载多路径聚合

service multipathd reload

multipath -ll 查看多路径聚合是否成功

6. 修改/etc/udev/rules.d/99-oracle-asmdevices.rules ,节点1 2 需要同步一致

7. 更新udev 磁盘映射

udevadm control --reload-rules

start_udev

ll /dev/mapper/

8. 增加磁盘到asm diskgroup

 

select name,total_mb,free_mb from v$asm_diskgroup;     查看目前diskgroup 大小

alter diskgroup SASDATA add disk ‘/dev/asm_sas6’ rebalance power 10;     增加磁盘到diskgroup

select * from v$asm_operation;          等待operation 完成,再查看diskgroup 大小

注意:每次只能一个operation 存在

 

 

[root@TRAC1 tmp]# su - grid

[grid@TRAC1 ~]$ sqlplus / as sysasm

 

SQL*Plus: Release 11.2.0.4.0 Production on Tue Jun 20 10:26:37 2017

 

Copyright (c) 1982, 2013, Oracle.  All rights reserved.

 

 

Connected to:

Oracle Database 11g Enterprise Edition Release 11.2.0.4.0 - 64bit Production

With the Real Application Clusters and Automatic Storage Management options

SQL> col name format A20

SQL> select name,total_mb,free_mb from v$asm_diskgroup;    

 

NAME                   TOTAL_MB    FREE_MB

-------------------- ---------- ----------

SASDATA                 8388608    6758368

SSDDATA                 8388608    7194758

VOT_MIRROR                42915      42087

VOT                       42915      41989

SQL>  alter diskgroup SASDATA add disk '/dev/asm_sas6'  rebalance power 10;

Diskgroup altered.

SQL>  col ERROR_CODE format a10

SQL> select * from v$asm_operation;

 

GROUP_NUMBER OPERATION       STATE             POWER     ACTUAL      SOFAR   EST_WORK   EST_RATE EST_MINUTES ERROR_CODE

------------ --------------- ------------ ---------- ---------- ---------- ---------- ---------- ----------- ----------

           1 REBAL           RUN                  10         10      13997     326080      13339          23

 

6. 扩展磁盘步骤

1. Dell storage management 扩展现有卷

2. 查看现有多路径关系

[root@TRAC1 ~]# multipath -l racsas6

racsas6 (36000d310013faa0000000000000000a5) dm-20 COMPELNT,Compellent Vol

size=2.0T features='1 queue_if_no_path' hwhandler='0' wp=rw

`-+- policy='round-robin 0' prio=0 status=active

  |- 1:0:5:11  sdaq 66:160 active undef unknown

  |- 4:0:0:11  sdaw 67:0   active undef unknown

  |- 1:0:14:11 sdat 66:208 active undef unknown

  `- 4:0:14:11 sday 67:32  active undef unknown

[root@TRAC1 ~]# multipath -l racssd6

racssd6 (36000d310013faa0000000000000000a8) dm-21 COMPELNT,Compellent Vol

size=2.0T features='1 queue_if_no_path' hwhandler='0' wp=rw

`-+- policy='round-robin 0' prio=0 status=active

  |- 1:0:3:12  sdac 65:192 active undef unknown

  |- 4:0:2:12  sdax 67:16  active undef unknown

  |- 1:0:15:12 sdau 66:224 active undef unknown

  `- 4:0:15:12 sdba 67:64  active undef unknown

[root@TRAC1 ~]# multipath -l backup1

backup1 (36000d310038170000000000000000007) dm-1 COMPELNT,Compellent Vol

size=4.0T features='1 queue_if_no_path' hwhandler='0' wp=rw

`-+- policy='round-robin 0' prio=0 status=active

  |- 1:0:11:1  sde  8:64   active undef unknown

  `- 4:0:8:1   sdc  8:32   active undef unknown

3. 重新扫描磁盘

multipath -l racsas6|grep sd|awk '{print "echo 1 > /sys/block/"$3"/device/rescan"}'

multipath -l racssd6|grep sd|awk '{print "echo 1 > /sys/block/"$3"/device/rescan"}'

multipath -l backup1|grep sd|awk '{print "echo 1 > /sys/block/"$3"/device/rescan"}'

执行上面打印的命令

4.service multipathd reload 重载多路径

5.multipath -l backup1 查看大小

backup1 (36000d310038170000000000000000007) dm-1 COMPELNT,Compellent Vol

size=10T features='1 queue_if_no_path' hwhandler='0' wp=rw

`-+- policy='round-robin 0' prio=0 status=active

  |- 4:0:11:1  sdd  8:48   active undef unknown

  `- 1:0:11:1  sde  8:64   active undef unknown

6. ASM 磁盘扩展

[root@TRAC1 ~]# parted /dev/mapper/backup1

GNU Parted 2.1

Using /dev/mapper/backup1

Welcome to GNU Parted! Type 'help' to view a list of commands.

(parted) print                                                           

Model: Linux device-mapper (multipath) (dm)

Disk /dev/mapper/backup1: 11.0TB

Sector size (logical/physical): 512B/4096B

Partition Table: gpt

 

Number  Start   End     Size    File system  Name     Flags

 1      2097kB  4398GB  4398GB  ext4         primary

 

(parted) unit s                                                          

(parted) print                                                            

Model: Linux device-mapper (multipath) (dm)

Disk /dev/mapper/backup1: 21474836480s

Sector size (logical/physical): 512B/4096B

Partition Table: gpt

 

Number  Start  End          Size         File system  Name     Flags

 1      4096s  8589930495s  8589926400s  ext4         primary

 

(parted) rm 1                                                            

(parted) mkpart primary 4096s  100%

(parted) print                                                           

Model: Linux device-mapper (multipath) (dm)

Disk /dev/mapper/backup1: 21474836480s

Sector size (logical/physical): 512B/4096B

Partition Table: gpt

 

Number  Start  End           Size          File system  Name     Flags

 1      4096s  21474832383s  21474828288s  ext4         primary

 

[root@TRAC1 ~]# e2fsck -f /dev/mapper/backup1p1

e2fsck 1.41.12 (17-May-2010)

Pass 1: Checking inodes, blocks, and sizes

Pass 2: Checking directory structure

Pass 3: Checking directory connectivity

Pass 4: Checking reference counts

Pass 5: Checking group summary information

/dev/mapper/backup1p1: 27/268435456 files (22.2% non-contiguous), 509820413/1073740800 blocks

ASM 磁盘扩展

[root@TRAC1 ~]# resize2fs /dev/mapper/backup1p1

resize2fs 1.41.12 (17-May-2010)

Resizing the filesystem on /dev/mapper/backup1p1 to 2684353536 (4k) blocks.

The filesystem on /dev/mapper/backup1p1 is now 2684353536 blocks long.

7.ASM 磁盘扩展

alter diskgroup SASDATA resize disk SASDATA_0004;

注意:ASM 单盘不支持超过2T

 

7. 删除磁盘步骤-ASM

1. 查看现有磁盘

SQL> col group_name format a20;

SQL> col disk_name format a20;

SQL> col path format a20;

SQL> set linesize 200;

SQL> SELECT b.NAME as group_name, a.name as disk_name, a.PATH,a.total_mb,a.free_mb

  2    FROM V$asm_disk a, v$asm_diskgroup b

  3   where a.group_number = b.group_number;

 

GROUP_NAME           DISK_NAME            PATH                   TOTAL_MB    FREE_MB

-------------------- -------------------- -------------------- ---------- ----------

VOT_MIRROR           VOT_MIRROR_0002      /dev/asm_sas1p3           14306      14029

VOT_MIRROR           VOT_MIRROR_0001      /dev/asm_sas1p2           14304      14028

VOT_MIRROR           VOT_MIRROR_0000      /dev/asm_sas1p1           14305      14030

SASDATA              SASDATA_0003         /dev/asm_sas5           2097152    1766484

SASDATA              SASDATA_0001         /dev/asm_sas3           2097152    1766485

SASDATA              SASDATA_0000         /dev/asm_sas2           2097152    1766484

SASDATA              SASDATA_0002         /dev/asm_sas4           2097152    1766485

VOT                  VOT_0000             /dev/asm_ssd1p3           14306      13997

VOT                  VOT_0002             /dev/asm_ssd1p1           14305      13996

VOT                  VOT_0001             /dev/asm_ssd1p2           14304      13996

SSDDATA              SSDDATA_0003         /dev/asm_ssd5           2097152    1858377

 

GROUP_NAME           DISK_NAME            PATH                   TOTAL_MB    FREE_MB

-------------------- -------------------- -------------------- ---------- ----------

SSDDATA              SSDDATA_0001         /dev/asm_ssd3           2097152    1858378

SSDDATA              SSDDATA_0002         /dev/asm_ssd4           2097152    1858377

SSDDATA              SSDDATA_0000         /dev/asm_ssd2           2097152    1858377

SSDDATA              SSDDATA_0004         /dev/asm_ssd6           2097152    1858381

SASDATA              SASDATA_0004         /dev/asm_sas6           2097152    1766490

 

16 rows selected.

2. 确定要删除的磁盘

SQL> alter diskgroup SASDATA drop disk SASDATA_0004 rebalance power 10;

SQL> alter diskgroup SSDDATA drop disk SSDDATA_0004 rebalance power 10;

 

select * from v$asm_operation; 注意:每次只能一个operation 存在

3.

3. 修改/etc/multipath.conf /etc/udev/rules.d/99-oracle-asmdevices.rules, 去掉对应的磁盘,节点1 2 需要同步

4.dell storage management 里面删除对应的卷和映射


来自 “ ITPUB博客 ” ,链接:http://blog.itpub.net/69976387/viewspace-2706571/,如需转载,请注明出处,否则将追究法律责任。

请登录后发表评论 登录
全部评论
Oracle 11g OCP, Oracle 11g OCM, MySQL 5.7 OCP, A member of OCMU Oracle User Group, Certificate of Aptech Certified System Master

注册时间:2020-06-03

  • 博文量
    18
  • 访问量
    10706