Notes: STE Disk Recovery
- Background: When the CV-STE lost a disk (one of the disks which had the users/groups account), Alexis provided his notes on recovering the disk
- This is a collection of those notes, with minimal organization by AndyHale
STE CV LVM Commands
For maintenance (on console)
- bring up the network interface
LVM disk management
- display the volume groups
- to remove the inconsistent metadata
LVM management sources:
Disk configuration notes
CV: [root@gns /]# cat /etc/fstab
# If you see this comment it means something wrong happened
# The fstab file should be re-written when using updateSystemFromCache
LABEL=root / ext3 defaults 1 1
none /dev/pts devpts gid=5,mode=620 0 0
none /dev/shm tmpfs defaults 0 0
LABEL=swap1 swap swap defaults 0 0
LABEL=swap2 swap swap defaults 0 0
/dev/cdrom /media/cdrom udf,iso9660 noauto,owner,ro 0 0
none /proc proc defaults 0 0
none /sys sysfs defaults 0 0
#10.12.97.112:/export/ibis /mnt/ibis nfs defaults,noauto 0 0
LABEL=data1 /mnt/gns/data1 ext3 defaults 1 2
#LABEL=data2 /mnt/gns/data2 ext3 defaults 1 2
LABEL=data3 /mnt/gns/data3 ext3 defaults 1 2
LABEL=data4 /mnt/gns/data4 ext3 defaults 1 2
LABEL=data5 /mnt/gns/data5 ext3 defaults 1 2
CV: [root@gns /]# cat /etc/mtab
/dev/sda1 / ext3 rw 0 0
none /proc proc rw 0 0
none /sys sysfs rw 0 0
none /dev/pts devpts rw,gid=5,mode=620 0 0
usbfs /proc/bus/usb usbfs rw 0 0
none /dev/shm tmpfs rw 0 0
/dev/sda4 /mnt/gns/data1 ext3 rw 0 0
/dev/sdb1 /mnt/gns/data3 ext3 rw 0 0
/dev/sde1 /mnt/gns/data4 ext3 rw 0 0
/dev/sdf1 /mnt/gns/data5 ext3 rw 0 0
none /proc/sys/fs/binfmt_misc binfmt_misc rw 0 0
sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw 0 0
nfsd /proc/fs/nfsd nfsd rw 0 0
CV: [root@gns /]# ll /dev/disk/by-label
total 0
lrwxrwxrwx 1 root root 10 Jun 22 18:39 data1 -> ../../sda4
lrwxrwxrwx 1 root root 10 Jun 22 18:39 data3 -> ../../sdb1
lrwxrwxrwx 1 root root 10 Jun 22 18:39 data4 -> ../../sde1
lrwxrwxrwx 1 root root 10 Jun 22 18:39 data5 -> ../../sdf1
lrwxrwxrwx 1 root root 10 Jun 22 18:39 root -> ../../sda1
lrwxrwxrwx 1 root root 10 Jun 22 18:39 swap1 -> ../../sda2
lrwxrwxrwx 1 root root 10 Jun 22 18:39 swap2 -> ../../sda3
CV: [root@gns /]# ll /dev/disk/by-id/
total 0
lrwxrwxrwx 1 root root 9 Jun 22 18:39 scsi-3500000e110f311d6 -> ../../sdc
lrwxrwxrwx 1 root root 9 Jun 22 18:39 scsi-3500000e11105e9ef -> ../../sdd
lrwxrwxrwx 1 root root 9 Jun 22 18:39 scsi-35000cca002e6c5b2 -> ../../sda
lrwxrwxrwx 1 root root 10 Jun 22 18:39 scsi-35000cca002e6c5b2-part1 -> ../../sda1
lrwxrwxrwx 1 root root 10 Jun 22 18:39 scsi-35000cca002e6c5b2-part2 -> ../../sda2
lrwxrwxrwx 1 root root 10 Jun 22 18:39 scsi-35000cca002e6c5b2-part3 -> ../../sda3
lrwxrwxrwx 1 root root 10 Jun 22 18:39 scsi-35000cca002e6c5b2-part4 -> ../../sda4
lrwxrwxrwx 1 root root 9 Jun 22 18:39 scsi-35000cca002e71898 -> ../../sdb
lrwxrwxrwx 1 root root 10 Jun 22 18:39 scsi-35000cca002e71898-part1 -> ../../sdb1
lrwxrwxrwx 1 root root 9 Jun 22 18:39 scsi-35000cca00404f02f -> ../../sdf
lrwxrwxrwx 1 root root 10 Jun 22 18:39 scsi-35000cca00404f02f-part1 -> ../../sdf1
lrwxrwxrwx 1 root root 9 Jun 22 18:39 scsi-35000cca004050cb7 -> ../../sde
lrwxrwxrwx 1 root root 10 Jun 22 18:39 scsi-35000cca004050cb7-part1 -> ../../sde1
CV: [root@gns /]# ll /dev/disk/by-path/
total 0
lrwxrwxrwx 1 root root 9 Jun 22 18:39 pci-0000:00:1f.1-ide-0:0 -> ../../hda
lrwxrwxrwx 1 root root 9 Jun 22 18:39 pci-0000:02:05.0-scsi-0:0:0:0 -> ../../sda
lrwxrwxrwx 1 root root 10 Jun 22 18:39 pci-0000:02:05.0-scsi-0:0:0:0-part1 -> ../../sda1
lrwxrwxrwx 1 root root 10 Jun 22 18:39 pci-0000:02:05.0-scsi-0:0:0:0-part2 -> ../../sda2
lrwxrwxrwx 1 root root 10 Jun 22 18:39 pci-0000:02:05.0-scsi-0:0:0:0-part3 -> ../../sda3
lrwxrwxrwx 1 root root 10 Jun 22 18:39 pci-0000:02:05.0-scsi-0:0:0:0-part4 -> ../../sda4
lrwxrwxrwx 1 root root 9 Jun 22 18:39 pci-0000:02:05.0-scsi-0:0:1:0 -> ../../sdb
lrwxrwxrwx 1 root root 10 Jun 22 18:39 pci-0000:02:05.0-scsi-0:0:1:0-part1 -> ../../sdb1
lrwxrwxrwx 1 root root 9 Jun 22 18:39 pci-0000:02:05.0-scsi-0:0:2:0 -> ../../sdc
lrwxrwxrwx 1 root root 9 Jun 22 18:39 pci-0000:02:05.0-scsi-0:0:3:0 -> ../../sdd
lrwxrwxrwx 1 root root 9 Jun 22 18:39 pci-0000:02:05.0-scsi-0:0:4:0 -> ../../sde
lrwxrwxrwx 1 root root 10 Jun 22 18:39 pci-0000:02:05.0-scsi-0:0:4:0-part1 -> ../../sde1
lrwxrwxrwx 1 root root 9 Jun 22 18:39 pci-0000:02:05.0-scsi-0:0:5:0 -> ../../sdf
lrwxrwxrwx 1 root root 10 Jun 22 18:39 pci-0000:02:05.0-scsi-0:0:5:0-part1 -> ../../sdf1
CV: [root@gns /]# ll /dev/disk/by-uuid/
total 0
lrwxrwxrwx 1 root root 10 Jun 22 18:39 41421651-89bf-427f-ae7b-f92ea1dd133f -> ../../sda1
lrwxrwxrwx 1 root root 10 Jun 22 18:39 735f4ee0-e2ae-44f3-9453-237790c5436f -> ../../sda4
lrwxrwxrwx 1 root root 10 Jun 22 18:39 b78081af-756b-467a-9c28-b058183ca90d -> ../../sde1
lrwxrwxrwx 1 root root 10 Jun 22 18:39 e8e3cf6c-5421-4dc4-9850-43de600ddd22 -> ../../sdb1
lrwxrwxrwx 1 root root 10 Jun 22 18:39 e959463e-79db-4f70-9c35-610963bca5e4 -> ../../sdf1
CV: [root@gns /]# mount
/dev/sda1 on / type ext3 (rw)
none on /proc type proc (rw)
none on /sys type sysfs (rw)
none on /dev/pts type devpts (rw,gid=5,mode=620)
usbfs on /proc/bus/usb type usbfs (rw)
none on /dev/shm type tmpfs (rw)
/dev/sda4 on /mnt/gns/data1 type ext3 (rw)
/dev/sdb1 on /mnt/gns/data3 type ext3 (rw)
/dev/sde1 on /mnt/gns/data4 type ext3 (rw)
/dev/sdf1 on /mnt/gns/data5 type ext3 (rw)
none on /proc/sys/fs/binfmt_misc type binfmt_misc (rw)
sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw)
nfsd on /proc/fs/nfsd type nfsd (rw)
/dev/mapper/vg01-lv01 on /mnt/gns/data2 type ext3 (rw)
CV: [root@gns /]# ll /usr/
total 248
lrwxrwxrwx 1 root root 19 Sep 16 2014 alma -> /mnt/gns/data5/alma
lrwxrwxrwx 1 root root 21 Apr 25 2013 alma64 -> /mnt/gns/data4/alma64
lrwxrwxrwx 1 root root 24 Sep 16 2014 almabuild -> /mnt/gns/data5/almabuild
lrwxrwxrwx 1 root root 22 May 31 2013 almadev -> /mnt/gns/data4/almadev
drwxr-xr-x 2 root root 4096 Mar 5 17:17 bin
drwxr-xr-x 3 root root 4096 Mar 13 20:37 boot
drwxr-xr-x 11 root root 4040 Jun 22 18:40 dev
drwxr-xr-x 90 root root 12288 Jun 22 18:40 etc
lrwxrwxrwx 1 root root 21 Jan 25 2011 groups -> /mnt/gns/data2/groups
drwxr-xr-x 13 root root 12288 Mar 5 17:17 lib
drwx------ 2 root root 16384 Jan 25 2011 lost+found
drwxr-xr-x 3 root root 4096 Dec 16 2014 media
drwxr-xr-x 2 root root 4096 Nov 2 2010 misc
drwxr-xr-x 10 root root 4096 Dec 9 2014 mnt
drwxr-xr-x 6 root root 4096 Mar 14 03:43 node
drwxr-xr-x 5 root root 4096 Nov 21 2014 opt
dr-xr-xr-x 142 root root 0 Jun 22 18:39 proc
drwxr-xr-x 18 root root 4096 Jun 22 19:04 root
drwxr-xr-x 2 root root 12288 May 20 14:26 sbin
drwxr-xr-x 2 root root 4096 Oct 1 2009 selinux
drwxr-xr-x 2 root root 4096 Oct 1 2009 srv
drwxr-xr-x 11 root root 0 Jun 22 18:39 sys
drwxr-xr-x 12 root root 4096 Mar 14 07:05 tftpboot
drwxrwxrwt 18 root root 233472 Jun 22 19:04 tmp
lrwxrwxrwx 1 root root 20 Jan 25 2011 users -> /mnt/gns/data2/users
drwxr-xr-x 23 root root 4096 Apr 10 17:58 usr
drwxr-xr-x 24 root root 4096 Jun 21 2011 var
CV: [root@gns /]# lvdisplay
--- Logical volume ---
LV Name /dev/vg01/lv01
VG Name vg01
LV UUID
M174QW-T8Tv-mF9r-AxTZ-qMiq-Ffxp-1H3HkT
LV Write Access read/write
LV Status available
# open 1
LV Size 558.79 GB
Current LE 143050
Segments 1
Allocation inherit
Read ahead sectors auto
- currently set to 512
Block device 253:0
CV: [root@gns ~]# fdisk -l
Disk /dev/sda: 146.8 GB, 146815733760 bytes
255 heads, 63 sectors/track, 17849 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Device Boot Start End Blocks Id System
/dev/sda1 * 1 2611 20972826 83 Linux
/dev/sda2 2612 2872 2096482+ 82 Linux swap / Solaris
/dev/sda3 2873 3133 2096482+ 82 Linux swap / Solaris
/dev/sda4 3134 17849 118206270 83 Linux
Disk /dev/sdb: 146.8 GB, 146815733760 bytes
255 heads, 63 sectors/track, 17849 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Device Boot Start End Blocks Id System
/dev/sdb1 1 17023 136732421+ 83 Linux
Disk /dev/sdc: 300.0 GB, 300000000000 bytes
255 heads, 63 sectors/track, 36472 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Disk /dev/sdc doesn't contain a valid partition table
Disk /dev/sdd: 300.0 GB, 300000000000 bytes
255 heads, 63 sectors/track, 36472 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Disk /dev/sdd doesn't contain a valid partition table
Disk /dev/sde: 300.0 GB, 300000000000 bytes
255 heads, 63 sectors/track, 36472 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Device Boot Start End Blocks Id System
/dev/sde1 1 36472 292961308+ 7 HPFS/NTFS
Disk /dev/sdf: 300.0 GB, 300000000000 bytes
255 heads, 63 sectors/track, 36472 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Device Boot Start End Blocks Id System
/dev/sdf1 1 36472 292961308+ 7 HPFS/NTFS
CV: [root@gns ~]# pvdisplay
--- Physical volume ---
PV Name /dev/sdc
VG Name vg01
PV Size 279.40 GB / not usable 2.29 MB
Allocatable yes (but full)
PE Size (KByte) 4096
Total PE 71525
Free PE 0
Allocated PE 71525
PV UUID
BnAKDY-J4Gy-eBs4-6txe-ZPvw-n3RI-AdpEh6
--- Physical volume ---
PV Name /dev/sdd
VG Name vg01
PV Size 279.40 GB / not usable 2.29 MB
Allocatable yes (but full)
PE Size (KByte) 4096
Total PE 71525
Free PE 0
Allocated PE 71525
PV UUID
Iak2OE-KRFC-eOWT-7L1g-IYfq-cO3B-aqivlF
CV: [root@gns ~]# vgdisplay
--- Volume group ---
VG Name vg01
System ID
Format lvm2
Metadata Areas 2
Metadata Sequence No 9
VG Access read/write
VG Status resizable
MAX LV 0
Cur LV 1
Open LV 1
Max PV 0
Cur PV 2
Act PV 2
VG Size 558.79 GB
PE Size 4.00 MB
Total PE 143050
Alloc PE / Size 143050 / 558.79 GB
Free PE / Size 0 / 0
VG UUID bphut0-kGRr-vgai-eSSo-t5Rl-hS9d-N9x58I
CV: [root@gns ~]# vgscan
Reading all physical volumes. This may take a while...
Found volume group "vg01" using metadata type lvm2
CV: [root@gns ~]# lvscan
ACTIVE '/dev/vg01/lv01' [558.79 GB] inherit
CV: [root@gns ~]# ll /dev/md*
brw-r----- 1 root disk 9, 0 Jun 22 18:39 /dev/md0
CV: [root@gns ~]# pvscan
File descriptor 3 (socket:[37520]) leaked on pvscan invocation. Parent PID 8613: -bash
PV /dev/sdc VG vg01 lvm2 [279.39 GB / 0 free]
PV /dev/sdd VG vg01 lvm2 [279.39 GB / 0 free]
Total: 2 [558.79 GB] / in use: 2 [558.79 GB] / in no VG: 0 [0 ]
CV: [root@gns ~]# pvdisplay
File descriptor 3 (socket:[37520]) leaked on pvdisplay invocation. Parent PID 8613: -bash
--- Physical volume ---
PV Name /dev/sdc
VG Name vg01
PV Size 279.40 GB / not usable 2.29 MB
Allocatable yes (but full)
PE Size (KByte) 4096
Total PE 71525
Free PE 0
Allocated PE 71525
PV UUID
BnAKDY-J4Gy-eBs4-6txe-ZPvw-n3RI-AdpEh6
--- Physical volume ---
PV Name /dev/sdd
VG Name vg01
PV Size 279.40 GB / not usable 2.29 MB
Allocatable yes (but full)
PE Size (KByte) 4096
Total PE 71525
Free PE 0
Allocated PE 71525
PV UUID
Iak2OE-KRFC-eOWT-7L1g-IYfq-cO3B-aqivlF
CV: [root@gns ~]# pvdisplay /dev/sdc
File descriptor 3 (socket:[37520]) leaked on pvdisplay invocation. Parent PID 8613: -bash
--- Physical volume ---
PV Name /dev/sdc
VG Name vg01
PV Size 279.40 GB / not usable 2.29 MB
Allocatable yes (but full)
PE Size (KByte) 4096
Total PE 71525
Free PE 0
Allocated PE 71525
PV UUID
BnAKDY-J4Gy-eBs4-6txe-ZPvw-n3RI-AdpEh6
CV: [root@gns ~]# pvdisplay /dev/sdd
File descriptor 3 (socket:[37520]) leaked on pvdisplay invocation. Parent PID 8613: -bash
--- Physical volume ---
PV Name /dev/sdd
VG Name vg01
PV Size 279.40 GB / not usable 2.29 MB
Allocatable yes (but full)
PE Size (KByte) 4096
Total PE 71525
Free PE 0
Allocated PE 71525
PV UUID
Iak2OE-KRFC-eOWT-7L1g-IYfq-cO3B-aqivlF
ls
CV: [root@gns ~]# lspci
00:00.0 Host bridge: Intel Corporation E7520 Memory Controller Hub (rev 09)
00:02.0 PCI bridge: Intel Corporation E7525/E7520/E7320 PCI Express Port A (rev 09)
00:04.0 PCI bridge: Intel Corporation E7525/E7520 PCI Express Port B (rev 09)
00:05.0 PCI bridge: Intel Corporation E7520 PCI Express Port B1 (rev 09)
00:06.0 PCI bridge: Intel Corporation E7520 PCI Express Port C (rev 09)
00:1d.0 USB Controller: Intel Corporation 82801EB/ER (ICH5/ICH5R) USB UHCI Controller #1 (rev 02)
00:1d.1 USB Controller: Intel Corporation 82801EB/ER (ICH5/ICH5R) USB UHCI Controller #2 (rev 02)
00:1d.2 USB Controller: Intel Corporation 82801EB/ER (ICH5/ICH5R) USB UHCI Controller #3 (rev 02)
00:1d.7 USB Controller: Intel Corporation 82801EB/ER (ICH5/ICH5R) USB2 EHCI Controller (rev 02)
00:1e.0 PCI bridge: Intel Corporation 82801 PCI Bridge (rev c2)
00:1f.0 ISA bridge: Intel Corporation 82801EB/ER (ICH5/ICH5R) LPC Interface Bridge (rev 02)
00:1f.1 IDE interface: Intel Corporation 82801EB/ER (ICH5/ICH5R) IDE Controller (rev 02)
01:00.0 PCI bridge: Intel Corporation 80332 [Dobson] I/O processor (A-Segment Bridge) (rev 06)
01:00.2 PCI bridge: Intel Corporation 80332 [Dobson] I/O processor (B-Segment Bridge) (rev 06)
02:05.0 SCSI storage controller: LSI Logic / Symbios Logic 53c1030 PCI-X Fusion-MPT Dual Ultra320 SCSI (rev 08)
02:05.1 SCSI storage controller: LSI Logic / Symbios Logic 53c1030 PCI-X Fusion-MPT Dual Ultra320 SCSI (rev 08)
05:00.0 PCI bridge: Intel Corporation 6700PXH PCI Express-to-PCI Bridge A (rev 09)
05:00.2 PCI bridge: Intel Corporation 6700PXH PCI Express-to-PCI Bridge B (rev 09)
06:07.0 Ethernet controller: Intel Corporation 82541GI Gigabit Ethernet Controller (rev 05)
07:08.0 Ethernet controller: Intel Corporation 82541GI Gigabit Ethernet Controller (rev 05)
08:00.0 PCI bridge: Intel Corporation 6700PXH PCI Express-to-PCI Bridge A (rev 09)
08:00.2 PCI bridge: Intel Corporation 6700PXH PCI Express-to-PCI Bridge B (rev 09)
0b:0d.0 VGA compatible controller: ATI Technologies Inc Radeon RV100 QY [Radeon 7000/VE]
--
CV: [root@gns ~]# modprobe linear
CV: [root@gns ~]# modprobe raid0
CV: [root@gns ~]# modprobe raid1
CV: [root@gns ~]# cat /proc/mdstat
Personalities : [linear] [raid0] [raid1]
unused devices:
CV STE Disk
- The CV disk was replaced and configured through LVM (from the scratch) same as was before, /dev/vg01/lv01.
- Currently user and group data is located at /mnt/gns/data2 (a bind of /mnt/gns/data5), I haven't switched the configuration back pointing to the current "partition" /dev/vg01/lv01 labeled as data2... yet.
- The 20150621 backup is already synchronized to /dev/vg01/lv01/users/. *Rachel, Rodrigo and J, you current user data is located at /mnt/gns/data5/users, I'll switch the configuration of the user data back to the new disks same as before (back to data2), means that you guys need to rsync/cp/etc what is needed or important from /mnt/gns/data5/users back to /mnt/gns/data2/users, I'll keep the data /mnt/gns/data5/users for a week more or less. So, if tomorrow your stuff isn't there, check your old user data located at /mnt/gns/data5/users.
- the backup is located at quark.cv.nrao.edu:/net/quark/media/ste-bkup/ste-corr/users/quark_backup/20150621.
-- AndyHale - 2015-07-09