HACMP Procedures

The following provides an example script for configuring HACMP in a two-node mutual failover cluster with multiple resource groups and multiple volume groups.

This script assumes the appropriate entries and modifications have been made to the following files:

• /etc/hosts
• /.rhosts
• /usr/es/sbin/cluster/etc/rhosts
• /usr/es/sbin/cluster/netmon.cf

#!/usr/bin/ksh93
#################################################################
NODE1="pppapora03"
NODE2="pppapora04"

RG1="etapdtu0"
RG1APPLS="${RG1}_oradb01 ${RG1}_oradb02"
RG1VGS="${RG1}01vg ${RG1}02vg"

RG2="gappdtu0"
RG2APPLS="${RG2}_oradb01"
RG2VGS="${RG2}01vg"

RG3="mrkpdtu0"
RG3APPLS="${RG3}_oradb01"
RG3VGS="${RG3}01vg"

RG4="znmpdtu0"
RG4APPLS="${RG4}_oradb01"
RG4VGS="${RG4}01vg"

################################################################
# Remove Existing Cluster: pppaporatu03
/usr/es/sbin/cluster/utilities/clrmclstr

# Create Cluster
/usr/es/sbin/cluster/utilities/clmodclstr -n ${NODE1%??}tu01 -p "${NODE1}-boot.triparadigm.com ${NODE2}-boot.triparadigm.com"

################################################################
# Create Service Addresses
/usr/es/sbin/cluster/utilities/clrmnode -a ${NODE1}.triparadigm.com
/usr/es/sbin/cluster/utilities/clrmnode -a ${NODE2}.triparadigm.com
/usr/es/sbin/cluster/utilities/claddnode -T service -B ${NODE1}.triparadigm.com -w net_ether_01
/usr/es/sbin/cluster/utilities/claddnode -T service -B ${NODE2}.triparadigm.com -w net_ether_01

################################################################
# Create Application Servers
for APPLS in ${RG1APPLS} ${RG2APPLS} ${RG3APPLS} ${RG4APPLS}
do
/usr/es/sbin/cluster/utilities/clrmserv ${APPLS}
/usr/es/sbin/cluster/utilities/claddserv -s ${APPLS} -b "/usr/local/hascripts/${APPLS}_start.sh" -e "/usr/local/hascripts/${APPLS}_stop.sh"
done

################################################################
# Create persistant addresses
/usr/es/sbin/cluster/utilities/clrmnode -a ${NODE1}-pers
/usr/es/sbin/cluster/utilities/clrmnode -a ${NODE2}-pers

/usr/es/sbin/cluster/utilities/claddnode -a ${NODE1}-pers.triparadigm.com :ether :net_ether_01 : :persistent : : -n ${NODE1}

/usr/es/sbin/cluster/utilities/claddnode -a ${NODE2}-pers.triparadigm.com :ether :net_ether_01 : :persistent : : -n ${NODE2}

################################################################
# Create resource groups
/usr/es/sbin/cluster/utilities/clrmgrp -g ${NODE1}
/usr/es/sbin/cluster/utilities/claddgrp -g ${NODE1} -n "${NODE1} ${NODE2}" -S OHN -O FNPN -B FBHPN

/usr/es/sbin/cluster/utilities/clrmgrp -g ${NODE2}
/usr/es/sbin/cluster/utilities/claddgrp -g ${NODE2} -n "${NODE2} ${NODE1}" -S OHN -O FNPN -B FBHPN

/usr/es/sbin/cluster/utilities/clrmgrp -g ${RG1}
/usr/es/sbin/cluster/utilities/claddgrp -g ${RG1} -n "${NODE1} ${NODE2}" -S OHN -O FNPN -B FBHPN

/usr/es/sbin/cluster/utilities/clrmgrp -g ${RG2}
/usr/es/sbin/cluster/utilities/claddgrp -g ${RG2} -n "${NODE2} ${NODE1}" -S OHN -O FNPN -B FBHPN

/usr/es/sbin/cluster/utilities/clrmgrp -g ${RG3}
/usr/es/sbin/cluster/utilities/claddgrp -g ${RG3} -n "${NODE2} ${NODE1}" -S OHN -O FNPN -B FBHPN

/usr/es/sbin/cluster/utilities/clrmgrp -g ${RG4}
/usr/es/sbin/cluster/utilities/claddgrp -g ${RG4} -n "${NODE2} ${NODE1}" -S OHN -O FNPN -B FBHPN

################################################################
# Add the Service address to the resource group
/usr/es/sbin/cluster/utilities/claddres -g "${NODE1}" \
  FALLBACK_AT= \
  SERVICE_LABEL="${NODE1}" \
  APPLICATIONS= \
  VOLUME_GROUP= \
  FORCED_VARYON="false" \
  VG_AUTO_IMPORT="false" \
  FILESYSTEM= \
  FSCHECK_TOOL="fsck" \
  RECOVERY_METHOD="sequential" \
  FS_BEFORE_IPADDR="false" \
  EXPORT_FILESYSTEM= \
  MOUNT_FILESYSTEM= \
  NFS_NETWORK= \
  SHARED_TAPE_RESOURCES= \
  DISK= \
  AIX_FAST_CONNECT_SERVICES= \
  COMMUNICATION_LINKS= \
  MISC_DATA=

/usr/es/sbin/cluster/utilities/claddres -g "${NODE2}" \
  FALLBACK_AT= \
  SERVICE_LABEL="${NODE2}" \
  APPLICATIONS= \
  VOLUME_GROUP= \
  FORCED_VARYON="false" \
  VG_AUTO_IMPORT="false" \
  FILESYSTEM= \
  FSCHECK_TOOL="fsck" \
  RECOVERY_METHOD="sequential" \
  FS_BEFORE_IPADDR="false" \
  EXPORT_FILESYSTEM= \
  MOUNT_FILESYSTEM= \
  NFS_NETWORK= \
  SHARED_TAPE_RESOURCES= \
  DISK= \
  AIX_FAST_CONNECT_SERVICES= \
  COMMUNICATION_LINKS= \
  MISC_DATA=

################################################################
# Add application servers and volumes to resource groups
/usr/es/sbin/cluster/utilities/claddres -g "${RG1}" \
  FALLBACK_AT= \
  SERVICE_LABEL="${NODE1}-rg01" \
  APPLICATIONS="${RG1APPLS}"  \
  VOLUME_GROUP="${RG1VGS}" \
  FORCED_VARYON="false"  \
  VG_AUTO_IMPORT="false"  \
  FILESYSTEM=  \
  FSCHECK_TOOL="fsck" \
  RECOVERY_METHOD="sequential"  \
  FS_BEFORE_IPADDR="false"  \
  EXPORT_FILESYSTEM= \
  MOUNT_FILESYSTEM=  \
  NFS_NETWORK=  \
  SHARED_TAPE_RESOURCES=  \
  DISK= \
  AIX_FAST_CONNECT_SERVICES=  \
  COMMUNICATION_LINKS=  \
  MISC_DATA=

/usr/es/sbin/cluster/utilities/claddres -g "${RG2}" \
  FALLBACK_AT= \
  SERVICE_LABEL="${NODE1}-rg02" \
  APPLICATIONS="${RG2APPLS}" \
  VOLUME_GROUP="${RG2VGS}" \
  FORCED_VARYON="false" \
  VG_AUTO_IMPORT="false" \
  FILESYSTEM= \
  FSCHECK_TOOL="fsck" \
  RECOVERY_METHOD="sequential" \
  FS_BEFORE_IPADDR="false" \
  EXPORT_FILESYSTEM= \
  MOUNT_FILESYSTEM= \
  NFS_NETWORK= \
  SHARED_TAPE_RESOURCES= \
  DISK= \
  AIX_FAST_CONNECT_SERVICES= \
  COMMUNICATION_LINKS= \
  MISC_DATA=

/usr/es/sbin/cluster/utilities/claddres -g "${RG3}" \
  FALLBACK_AT= \
  SERVICE_LABEL="${NODE2}-rg03" \
  APPLICATIONS="${RG3APPLS}" \
  VOLUME_GROUP="${RG3VGS}" \
  FORCED_VARYON="false" \
  VG_AUTO_IMPORT="false" \
  FILESYSTEM= \
  FSCHECK_TOOL="fsck" \
  RECOVERY_METHOD="sequential" \
  FS_BEFORE_IPADDR="false" \
  EXPORT_FILESYSTEM= \
  MOUNT_FILESYSTEM= \
  NFS_NETWORK= \
  SHARED_TAPE_RESOURCES= \
  DISK= \
  AIX_FAST_CONNECT_SERVICES= \
  COMMUNICATION_LINKS= \
  MISC_DATA=

/usr/es/sbin/cluster/utilities/claddres -g "${RG4}" \
  FALLBACK_AT= \
  SERVICE_LABEL="${NODE2}-rg04" \
  APPLICATIONS="${RG4APPLS}" \
  VOLUME_GROUP="${RG4VGS}" \
  FORCED_VARYON="false" \
  VG_AUTO_IMPORT="false" \
  FILESYSTEM= \
  FSCHECK_TOOL="fsck" \
  RECOVERY_METHOD="sequential" \
  FS_BEFORE_IPADDR="false" \
  EXPORT_FILESYSTEM= \
  MOUNT_FILESYSTEM= \
  NFS_NETWORK= \
  SHARED_TAPE_RESOURCES= \
  DISK= \
  AIX_FAST_CONNECT_SERVICES= \
  COMMUNICATION_LINKS= \
  MISC_DATA=

The Automated HACMP configuration is almost complete, at this point manually configure the concurrent disk heartbeat.

Manual HACMP configuration

EXAMPLE: P3 EDNA HACMP 590 Configuration

1. Verify Appropriate Network Configuration :
   • To Add the service address to boot interface initially before HACMP
   • configuration is completed run the following command.

     ifconfig en4 10.61.171.60 netmask 255.255.255.0 alias
     

   • Add Default Route for system.

     mkdev -l inet0
     mkdev -l en4
     

   • After HACMP Configuration completed verify network configuration.

     netstat -in
     Name  Mtu   Network     Address
     en4   1500  link#2      12.f3.f0.0.61.f4
     en4   1500  192.168.171 192.168.171.60   ** Boot address
     en4   1500  10.61.171  10.61.171.60    ** Service Address
     en4   1500  10.61.171  10.61.171.61    ** Persistence Address
     en5   1500  link#3      12.f3.f0.0.63.52
     en5   1500  10.61.172  10.61.172.44    ** Man Address
     lo0   16896 link#1
     lo0   16896 127         127.0.0.1
     lo0   16896 ::1
           

2. Remove Old HACMP configurations if they exist :

       smitty hacmp -->
           Extended Configuration -->
               Extended Topology Configuration -->
                    Configure an HACMP Cluster -->
                       Remove an HACMP Cluster
   

3. Configure Various HACMP Required Files :

   /etc/hosts  :
   ## HACMP ip adress labels ##
   10.61.171.68   pppapedn07.triparadigm.com       pppapedn07
   10.61.171.69   pppapedn07-pers.triparadigm.com  pppapedn07-pers
   192.168.171.68  pppapedn07-boot.triparadigm.com  pppapedn07-boot
   10.61.172.48   pppapedn07-man.triparadigm.com   pppapedn07-man
   10.61.171.70   pppapedn08.triparadigm.com       pppapedn08
   10.61.171.71   pppapedn08-pers.triparadigm.com  pppapedn08-pers
   192.168.171.70  pppapedn08-boot.triparadigm.com  pppapedn08-boot
   10.61.172.49   pppapedn08-man.triparadigm.com   pppapedn08-man
   

   /.rhosts :
   mtxapnim01              root
   mtxapnim01.triparadigm.com       root
   pppapnim01.triparadigm.com        root
   pppapnim01              root
   46.61.171.68           root
   pppapedn07.triparadigm.com        root
   pppapedn07        root
   10.61.171.69       root
   pppapedn07-pers.triparadigm.com  root
   pppapedn07-pers        root
   192.168.171.68      root
   pppapedn07-boot.triparadigm.com  root
   pppapedn07-boot        root
   10.61.172.48       root
   pppapedn07-man.triparadigm.com   root
   pppapedn07-man        root
   10.61.171.70       root
   pppapedn08.triparadigm.com    root
   pppapedn08        root
   10.61.171.71       root
   pppapedn08-pers.triparadigm.com  root
   pppapedn08-pers        root
   192.168.171.70      root
   pppapedn08-boot.triparadigm.com    root
   pppapedn08-boot        root
   10.61.172.49       root
   pppapedn08-man.triparadigm.com    root
   pppapedn08-man        root
   

   /usr/es/sbin/cluster/etc/rhosts :
   pppapedn07
   pppapedn07-boot
   pppapedn08
   pppapedn08-boot
   

   • service address and boot address for participating nodes in cluster.

   /usr/es/sbin/cluster/etc/netmon.cf :
   pppapedn07
   pppapedn07-boot
   pppapedn08
   pppapedn08-boot
   

4. Test rsh functionality between nodes.
   • Use boot address to rsh to and from clustered pairs.
5. Add Nodes To Cluster Configuration :

       Initialization and Standard Configuration -->
           Add Nodes to an HACMP Cluster -->
               cluster name "pppapedntu07"
               New Nodes "pppapedn07 pppapedn08"
                   ( Service Names )
   

6. Add Service Lables :

       Initialization and Standard Configuration -->
           Configure Resources to Make Highly Available -->
                   Configure Service IP Labels/Addresses -->    
                       Add a Service IP Label/Address -->
                           IP Label Address "pppapedn07.triparadigm.com"
                               (Service hostname)
                           Network Name "net_ether_01"
                               (Boot Label 192.168.xx.xx)
           ***** Perform Twice, Next pppapedn08.triparadigm.com Service Label.
   

7. Add Persistence Labels :

       Extended Configuration -->
           Extended Topology Configuration -->
               Configure HACMP Persistent Node IP Label/Addresses -->
                   Add a Persistent Node IP Label/Address -->
                       Select Node "pppapedn07"
                       Network Name "net_ether_01"
                           ( Persistence Network Label )
                       Node IP Label/Address "pppapedn07-pers.triparadigm.com"
                           ( Persistence Node Label )
           ***** Perform Twice, Next pppapedn08.triparadigm.com Persistence Label.
   

8. Synchronize Cluster configuration :

       Extended Configuration -->
           Extended Verification and Synchronization -->
               Automatically correct errors    found "yes"
               ( Correct Timestamp for HACMP file. )
       ***** Accept other defaults and synchronize cluster.
   

9. Create Volume Groups and Filesystems On Systems:
10. Create Resource Groups :

        Extended Configuration -->
            Extended Resource Configuration -->
                HACMP Extended Resource Group Configuration -->
                    Add a Resource Group -->
                    Enter RG Name for Service Labels "pppapedn07"
                    Participating Nodes "pppapedn07 pppapedn08"
                    ( Priority Order of Nodes for the group )
                     *****  Accept defaults for additional fields.
        *****  Perform Twice if Mutual Failover else Once if 1-way.
                                    Enter RG Name for Volume groups "ednpdtu7"
                        Participating Nodes "pppapedn07 pppapedn08"
                    ( Priority Order of Nodes for the group )
        *****  Perform Twice if Mutual Failover else Once if 1-way.
    

11. Add Application Server :

        ***** Perform these steps for application servers only.
          Initialization and Standard Configuration -->
            Configure Resources to Make Highly Available -->
                Configure Application Servers -->
                    Add an Application Server -->
                    Enter APP SERVER RG Name "ednpdtu7_appl01"
                    Enter Start Script "/usr/local/hascripts/ednpdtu7_appl01_start.sh"
                    Enter Stop Script  "/usr/local/hascripts/ednpdtu7_appl01_stop.sh"
                    ( Application Start and Stop Scripts )
    

12. Configure Attributes for New Resource Groups :

        Extended Configuration -->
            Extended Resource Configuration -->
                Extended Resource Group Configuration -->
                    Change/Show Resources and Attributes for a Resource Group -->
                    Select RG "pppapedn07"
                    Enter Service IP Labels "pppapedn07"
                    ( Failover of Serve IP for RG )
        *****  Perform Twice if Mutual Failover else Once if 1-way.
                    Select RG "ednpdtu7"
                    Enter Application Server "ednpdtu7_appl01"
                    Enter Volume Groups "ednpdtu700vg"
                    ( Failover of VG's )
    

13. Synchronize Cluster configuration :

        Extended Configuration -->
            Extended Verification and Synchronization -->
            ***** Accept defaults and synchronize cluster.
    

14. Discover HACMP-related Information from Clustered Nodes :

        Discover HACMP-related Information from Configured Nodes
        ( Collect disk info for Disk Heartbeat Configuration etc ........ )
    

15. Disk Heartbeat Configuration :

        Extended Configuration -->
            Extended Topology Configuration -->
                Configure HACMP Communication Interfaces/Devices -->
                    Add Communication Interfaces/Devices -->
                    Select "Add Discovered Communication Interface and Devices"
                    Select "Communication Devices"
                    Select 1 Disk from each Node /LPAR "pppapedn07 pppapedn08"
        *****  Configure Disk Heartbeat for all VG in cluster.
    

16. Synchronize Cluster configuration :

       Extended Configuration -->
            Extended Verification and Synchronization -->
            ***** Accept defaults and synchronize cluster.