Objective
To add 1x100GB & 1x200GB mountpoints to existing GPFS cluster.
Overview
Current mountpoints:
[root@kpgmeddev01 ~]# df -h
Filesystem Size Used Avail Use% Mounted on
/dev/mapper/vg00-root
3.0G 2.3G 479M 84% /
/dev/mapper/vg00-tmp 3.0G 145M 2.7G 6% /tmp
/dev/mapper/vg00-var 3.0G 834M 2.0G 30% /var
/dev/mapper/vg00-itm 992M 34M 908M 4% /Tivoli/ITM
/dev/mapper/vg00-fusion
496M 248M 223M 53% /ibm/fusion
/dev/mapper/vg00-usr 4.9G 2.4G 2.3G 52% /usr
/dev/sda1 99M 44M 51M 47% /boot
tmpfs 7.8G 0 7.8G 0% /dev/shm
/dev/mapper/vg01-opt--dcs
5.0G 1.1G 3.9G 22% /opt/dcs
/dev/mapper/vg01-opt--dcs6
9.9G 1.2G 8.8G 12% /opt/dcs_6.0
/dev/mapper/vg01-var--dcs
9.9G 190M 9.7G 2% /var/dcs_6.0
/dev/dcsappl01 500G 3.4G 497G 1% /dcs/appl01
/dev/dcsdata01 1000G 2.3G 998G 1% /dcs/data01
/dev/dcsdata02 1000G 3.1G 997G 1% /dcs/data02
/dev/dcsdata03 1000G 8.4G 992G 1% /dcs/data03
/dev/dcsdata04 1000G 2.8G 998G 1% /dcs/data04
/dev/dcsdata05 1000G 1.5G 999G 1% /dcs/data05
/dev/dcsdata06 500G 1.4G 499G 1% /dcs/data06
Highlighted ones are GPFS mountpoints
Current NSD:
[root@kpgmeddev01 ~]# mmlsnsd
File system Disk name NSD servers
---------------------------------------------------------------------------
dcsappl01 nsdappl01 (directly attached)
dcsdata01 nsddata01 (directly attached)
dcsdata02 nsddata02 (directly attached)
dcsdata03 nsddata03 (directly attached)
dcsdata04 nsddata04 (directly attached)
dcsdata05 nsddata05 (directly attached)
dcsdata06 nsddata06 (directly attached)
(free disk) nsd_tb1 (directly attached)
(free disk) nsd_tb2 (directly attached)
(free disk) nsd_tb3 (directly attached)
Disks:
3x100GB disks are multipathed,
[root@kpgmeddev01 ~]# multipath -l mpath11
mpath11 (36005076801910332f800000000000244) dm-18 IBM,2145
[size=100G][features=1 queue_if_no_path][hwhandler=0][rw]
\_ round-robin 0 [prio=0][active]
\_ 2:0:1:10 sday 67:32 [active][undef]
\_ 1:0:1:10 sdy 65:128 [active][undef]
\_ round-robin 0 [prio=0][enabled]
\_ 2:0:0:10 sdal 66:80 [active][undef]
\_ 1:0:0:10 sdl 8:176 [active][undef]
[root@kpgmeddev01 ~]# multipath -l mpath12
mpath12 (36005076801910332f800000000000245) dm-19 IBM,2145
[size=100G][features=1 queue_if_no_path][hwhandler=0][rw]
\_ round-robin 0 [prio=0][active]
\_ 2:0:0:11 sdam 66:96 [active][undef]
\_ 1:0:0:11 sdm 8:192 [active][undef]
\_ round-robin 0 [prio=0][enabled]
\_ 2:0:1:11 sdaz 67:48 [active][undef]
\_ 1:0:1:11 sdz 65:144 [active][undef]
[root@kpgmeddev01 ~]# multipath -l mpath13
mpath13 (36005076801910332f800000000000246) dm-7 IBM,2145
[size=100G][features=1 queue_if_no_path][hwhandler=0][rw]
\_ round-robin 0 [prio=0][active]
\_ 1:0:1:12 sdaa 65:160 [active][undef]
\_ 2:0:1:12 sdba 67:64 [active][undef]
\_ round-robin 0 [prio=0][enabled]
\_ 2:0:0:12 sdan 66:112 [active][undef]
\_ 1:0:0:12 sdn 8:208 [active][undef]
Step-by-Step
- Create NSD:
[root@kpgmeddev01 ~]# cat gpfs-test/adddisk.txt
/dev/dm-7:::dataAndMetadata::nsdtest01:
/dev/dm-18:::dataAndMetadata::nsdtest02:
/dev/dm-19:::dataAndMetadata::nsdtest03:
[root@kpgmeddev01 ~]# mmcrnsd -F gpfs-test/adddisk.txt
mmcrnsd: Processing disk dm-7
mmcrnsd: Processing disk dm-18
mmcrnsd: Processing disk dm-19
mmcrnsd: Propagating the cluster configuration data to all
affected nodes. This is an asynchronous process.
[root@kpgmeddev01 ~]# mmlsnsd
File system Disk name NSD servers
---------------------------------------------------------------------------
dcsappl01 nsdappl01 (directly attached)
dcsdata01 nsddata01 (directly attached)
dcsdata02 nsddata02 (directly attached)
dcsdata03 nsddata03 (directly attached)
dcsdata04 nsddata04 (directly attached)
dcsdata05 nsddata05 (directly attached)
dcsdata06 nsddata06 (directly attached)
(free disk) nsd_tb1 (directly attached)
(free disk) nsd_tb2 (directly attached)
(free disk) nsd_tb3 (directly attached)
(free disk) nsdtest01 (directly attached)
(free disk) nsdtest02 (directly attached)
(free disk) nsdtest03 (directly attached)
- Create New Filesystem
[root@kpgmeddev01 ~]# mmcrfs test100g nsdtest01 -T /test100g
The following disks of test100g will be formatted on node kpgmeddev02:
nsdtest01: size 104857600 KB
Formatting file system ...
Disks up to size 883 GB can be added to storage pool 'system'.
Creating Inode File
Creating Allocation Maps
Clearing Inode Allocation Map
Clearing Block Allocation Map
Formatting Allocation Map for storage pool 'system'
Completed creation of file system /dev/test100g.
mmcrfs: Propagating the cluster configuration data to all
affected nodes. This is an asynchronous process.
[root@kpgmeddev01 ~]# mmcrfs test200g "nsdtest02;nsdtest03" -T /test200g
The following disks of test200g will be formatted on node kpgmeddev02:
nsdtest02: size 104857600 KB
nsdtest03: size 104857600 KB
Formatting file system ...
Disks up to size 877 GB can be added to storage pool 'system'.
Creating Inode File
Creating Allocation Maps
Clearing Inode Allocation Map
Clearing Block Allocation Map
Formatting Allocation Map for storage pool 'system'
Completed creation of file system /dev/test200g.
mmcrfs: Propagating the cluster configuration data to all
affected nodes. This is an asynchronous process.
[root@kpgmeddev01 ~]# mmlsnsd
File system Disk name NSD servers
---------------------------------------------------------------------------
dcsappl01 nsdappl01 (directly attached)
dcsdata01 nsddata01 (directly attached)
dcsdata02 nsddata02 (directly attached)
dcsdata03 nsddata03 (directly attached)
dcsdata04 nsddata04 (directly attached)
dcsdata05 nsddata05 (directly attached)
dcsdata06 nsddata06 (directly attached)
test100g nsdtest01 (directly attached)
test200g nsdtest02 (directly attached)
test200g nsdtest03 (directly attached)
(free disk) nsd_tb1 (directly attached)
(free disk) nsd_tb2 (directly attached)
(free disk) nsd_tb3 (directly attached)
[root@kpgmeddev01 ~]# mmlsfs test100g
flag value description
---- ---------------- -----------------------------------------------------
-f 8192 Minimum fragment size in bytes
-i 512 Inode size in bytes
-I 16384 Indirect block size in bytes
-m 1 Default number of metadata replicas
-M 2 Maximum number of metadata replicas
-r 1 Default number of data replicas
-R 2 Maximum number of data replicas
-j cluster Block allocation type
-D nfs4 File locking semantics in effect
-k all ACL semantics in effect
-a 1048576 Estimated average file size
-n 32 Estimated number of nodes that will mount file system
-B 262144 Block size
-Q none Quotas enforced
none Default quotas enabled
-F 102912 Maximum number of inodes
-V 11.05 (3.3.0.2) File system version
-u yes Support for large LUNs?
-z no Is DMAPI enabled?
-L 4194304 Logfile size
-E yes Exact mtime mount option
-S no Suppress atime mount option
-K whenpossible Strict replica allocation option
-P system Disk storage pools in file system
-d nsdtest01 Disks in file system
-A yes Automatic mount option
-o none Additional mount options
-T /test100g Default mount point
[root@kpgmeddev01 ~]# mmlsfs test200g
flag value description
---- ---------------- -----------------------------------------------------
-f 8192 Minimum fragment size in bytes
-i 512 Inode size in bytes
-I 16384 Indirect block size in bytes
-m 1 Default number of metadata replicas
-M 2 Maximum number of metadata replicas
-r 1 Default number of data replicas
-R 2 Maximum number of data replicas
-j cluster Block allocation type
-D nfs4 File locking semantics in effect
-k all ACL semantics in effect
-a 1048576 Estimated average file size
-n 32 Estimated number of nodes that will mount file system
-B 262144 Block size
-Q none Quotas enforced
none Default quotas enabled
-F 205824 Maximum number of inodes
-V 11.05 (3.3.0.2) File system version
-u yes Support for large LUNs?
-z no Is DMAPI enabled?
-L 4194304 Logfile size
-E yes Exact mtime mount option
-S no Suppress atime mount option
-K whenpossible Strict replica allocation option
-P system Disk storage pools in file system
-d nsdtest02;nsdtest03 Disks in file system
-A yes Automatic mount option
-o none Additional mount options
-T /test200g Default mount point
- Mount New filesystem
[root@kpgmeddev01 ~]# mmmount test100g
Thu May 3 13:19:34 MYT 2012: mmmount: Mounting file systems ...
[root@kpgmeddev01 ~]# mmmount test200g
Thu May 3 13:19:42 MYT 2012: mmmount: Mounting file systems ...
- Verify
[root@kpgmeddev01 ~]# mmdf test100g
disk disk size failure holds holds free KB free KB
name in KB group metadata data in full blocks in fragments
--------------- ------------- -------- -------- ----- -------------------- -------------------
Disks in storage pool: system (Maximum disk size allowed is 883 GB)
nsdtest01 104857600 -1 yes yes 104643840 (100%) 440 ( 0%)
------------- -------------------- -------------------
(pool total) 104857600 104643840 (100%) 440 ( 0%)
============= ==================== ===================
(total) 104857600 104643840 (100%) 440 ( 0%)
Inode Information
-----------------
Number of used inodes: 4038
Number of free inodes: 98874
Number of allocated inodes: 102912
Maximum number of inodes: 102912
[root@kpgmeddev01 ~]# mmdf test200g
disk disk size failure holds holds free KB free KB
name in KB group metadata data in full blocks in fragments
--------------- ------------- -------- -------- ----- -------------------- -------------------
Disks in storage pool: system (Maximum disk size allowed is 877 GB)
nsdtest02 104857600 -1 yes yes 104723712 (100%) 488 ( 0%)
nsdtest03 104857600 -1 yes yes 104723968 (100%) 456 ( 0%)
------------- -------------------- -------------------
(pool total) 209715200 209447680 (100%) 944 ( 0%)
============= ==================== ===================
(total) 209715200 209447680 (100%) 944 ( 0%)
Inode Information
-----------------
Number of used inodes: 4038
Number of free inodes: 201786
Number of allocated inodes: 205824
Maximum number of inodes: 205824
[root@kpgmeddev01 ~]# df -h
Filesystem Size Used Avail Use% Mounted on
/dev/mapper/vg00-root
3.0G 2.4G 463M 84% /
/dev/mapper/vg00-tmp 3.0G 145M 2.7G 6% /tmp
/dev/mapper/vg00-var 3.0G 630M 2.2G 23% /var
/dev/mapper/vg00-itm 992M 34M 908M 4% /Tivoli/ITM
/dev/mapper/vg00-fusion
496M 248M 223M 53% /ibm/fusion
/dev/mapper/vg00-usr 4.9G 2.5G 2.2G 53% /usr
/dev/sda1 99M 44M 51M 47% /boot
tmpfs 7.8G 0 7.8G 0% /dev/shm
/dev/mapper/vg01-opt--dcs
5.0G 1.1G 3.9G 22% /opt/dcs
/dev/mapper/vg01-opt--dcs6
9.9G 1.2G 8.8G 12% /opt/dcs_6.0
/dev/mapper/vg01-var--dcs
9.9G 190M 9.7G 2% /var/dcs_6.0
/dev/dcsappl01 500G 3.4G 497G 1% /dcs/appl01
/dev/dcsdata01 1000G 2.3G 998G 1% /dcs/data01
/dev/dcsdata02 1000G 3.1G 997G 1% /dcs/data02
/dev/dcsdata03 1000G 8.4G 992G 1% /dcs/data03
/dev/dcsdata04 1000G 2.8G 998G 1% /dcs/data04
/dev/dcsdata05 1000G 1.5G 999G 1% /dcs/data05
/dev/dcsdata06 500G 1.4G 499G 1% /dcs/data06
/dev/test100g 100G 209M 100G 1% /test100g
/dev/test200g 200G 262M 200G 1% /test200g
Delete Everything
Unmount the filesystems,
[root@kpgmeddev01 ~]# mmumount /test100g
Thu May 3 13:23:15 MYT 2012: mmumount: Unmounting file systems ...
[root@kpgmeddev01 ~]# mmumount /test200g
Thu May 3 13:23:18 MYT 2012: mmumount: Unmounting file systems ...
Delete the filesystems,
[root@kpgmeddev01 ~]# mmdelfs test100g
All data on following disks of test100g will be destroyed:
nsdtest01
Completed deletion of file system /dev/test100g.
mmdelfs: Propagating the cluster configuration data to all
affected nodes. This is an asynchronous process.
[root@kpgmeddev01 ~]# mmdelfs test200g
All data on following disks of test200g will be destroyed:
nsdtest02
nsdtest03
Completed deletion of file system /dev/test200g.
mmdelfs: Propagating the cluster configuration data to all
affected nodes. This is an asynchronous process.
Delete NSD,
[root@kpgmeddev01 ~]# mmdelnsd "nsdtest01;nsdtest02;nsdtest03"
mmdelnsd: Processing disk nsdtest01
mmdelnsd: Processing disk nsdtest02
mmdelnsd: Processing disk nsdtest03
mmdelnsd: Propagating the cluster configuration data to all
affected nodes. This is an asynchronous process.
Verify those NSD were deleted,
[root@kpgmeddev01 ~]# mmlsnsd
File system Disk name NSD servers
---------------------------------------------------------------------------
dcsappl01 nsdappl01 (directly attached)
dcsdata01 nsddata01 (directly attached)
dcsdata02 nsddata02 (directly attached)
dcsdata03 nsddata03 (directly attached)
dcsdata04 nsddata04 (directly attached)
dcsdata05 nsddata05 (directly attached)
dcsdata06 nsddata06 (directly attached)
(free disk) nsd_tb1 (directly attached)
(free disk) nsd_tb2 (directly attached)
(free disk) nsd_tb3 (directly attached)
Comment(s) via Facebook
Tagged ibm-gpfs