Network/HasturRaidUpgrade.md
... ...
@@ -20,8 +20,9 @@ Upgrading from 6x 500GB RAID5 to 4x 2TB RAID6
20 20
21 21
* Samsung disks had SMART disabled by default
22 22
23
+ ```
23 24
# for i sd{b..e} ; do smartctl -s on /dev/$i ; done
24
-
25
+ ```
25 26
26 27
27 28
... ...
@@ -31,11 +32,13 @@ Upgrading from 6x 500GB RAID5 to 4x 2TB RAID6
31 32
* Default cipher and mode is "aes-cbc-essiv:sha256"
32 33
* Default keysize is 128-bits
33 34
35
+ ```
34 36
# for i in sd{b..e} ; do dd_rescue -m 10M /dev/urandom /dev/$i ; done
35 37
# for i in sd{b..e} ; do echo -n $TEST_PW | cryptsetup luksFormat --key-file=- /dev/$i ; done
36 38
# for i in sd{b..e} ; do echo -n $TEST_PW | cryptsetup luksOpen --key-file=- /dev/$i crypt-$i ; done
37 39
# for i in sd{b..e} ; do dd_rescue /dev/zero /dev/mapper/crypt-$i & done
38 40
# for i in sd{b..e} ; do cryptsetup luksClose /dev/$i crypt-$i ; done
41
+ ```
39 42
40 43
41 44
NOTE: dd_rescue doesn't stop when it reaches the end of the output device!
... ...
@@ -52,14 +55,16 @@ NOTE: dd_rescue doesn't stop when it reaches the end of the output device!
52 55
53 56
#### Build and install XTS
54 57
55
- $ echo "CONFIG_CRYPTO_XTS=m" >> .config
56
- $ make modules
57
- # cp crypto/xts.ko /lib/modules/2.6.31-pmp/kernel/crypto/
58
- # depmod -A
59
- # modprobe xts
60
- # mount -o remount,rw /boot
61
- # cp .config /boot/config-2.6.31-pmp
62
- # mount -o remount,ro /boot
58
+```
59
+$ echo "CONFIG_CRYPTO_XTS=m" >> .config
60
+$ make modules
61
+# cp crypto/xts.ko /lib/modules/2.6.31-pmp/kernel/crypto/
62
+# depmod -A
63
+# modprobe xts
64
+# mount -o remount,rw /boot
65
+# cp .config /boot/config-2.6.31-pmp
66
+# mount -o remount,ro /boot
67
+```
63 68
64 69
65 70
... ...
@@ -68,52 +73,62 @@ NOTE: dd_rescue doesn't stop when it reaches the end of the output device!
68 73
69 74
* Update parted to 1.8.8
70 75
* DOS partition
71
- # cat > sfdisk.tempraid0.format
72
- ,1024,fd
73
- ,2048,fd
74
-
75
- ^D
76
- # for dev in {b..e} ; do cat sfdisk.tempraid0.format | sfdisk -uM /dev/sd$dev ; done
76
+ ```
77
+ # cat > sfdisk.tempraid0.format
78
+ ,1024,fd
79
+ ,2048,fd
80
+
81
+ ^D
82
+ # for dev in {b..e} ; do cat sfdisk.tempraid0.format | sfdisk -uM /dev/sd$dev ; done
83
+ ```
77 84
78 85
79 86
80 87
* GPT partition (annoying buggy parted means manual label creation)
81
- # for dev in {b..e} ; do parted /dev/sd$dev mklabel ; done
82
- # for dev in {b..e} ; do parted /dev/sd$dev mkpart primary 0 1024 mkpart primary 1024 3072 ; done
88
+ ```
89
+ # for dev in {b..e} ; do parted /dev/sd$dev mklabel ; done
90
+ # for dev in {b..e} ; do parted /dev/sd$dev mkpart primary 0 1024 mkpart primary 1024 3072 ; done
91
+ ```
83 92
84 93
85 94
86 95
87 96
88 97
* Arrays
89
- # mdadm --create --verbose --metadata=1.2 --level=raid0 --raid-devices=4 /dev/md66 /dev/sd{b,c,d,e}1
90
- # mdadm --create --verbose --assume-clean --metadata=1.2 --level=raid6 --raid-devices=4 /dev/md67 /dev/sd{b,c,d,e}2
91
- # for dev in md66 md67 ; do mkfs.ext4 -m 0 /dev/$dev && mkdir -p /mnt/$dev && mount /dev/$dev /mnt/$dev ; done
98
+ ```
99
+ # mdadm --create --verbose --metadata=1.2 --level=raid0 --raid-devices=4 /dev/md66 /dev/sd{b,c,d,e}1
100
+ # mdadm --create --verbose --assume-clean --metadata=1.2 --level=raid6 --raid-devices=4 /dev/md67 /dev/sd{b,c,d,e}2
101
+ # for dev in md66 md67 ; do mkfs.ext4 -m 0 /dev/$dev && mkdir -p /mnt/$dev && mount /dev/$dev /mnt/$dev ; done
102
+ ```
92 103
93 104
94 105
95 106
96 107
97 108
* Results (some 3908MB, some 3750MB)
98
- # bonnie++ -f -d /mnt/md66 -s 3750 -n 0 -u root
99
- Version 1.03d ------Sequential Output------ --Sequential Input- --Random-
100
- -Per Chr- --Block-- -Rewrite- -Per Chr- --Block-- --Seeks--
101
- Machine Size K/sec %CP K/sec %CP K/sec %CP K/sec %CP K/sec %CP /sec %CP
102
- msdos,3908M,,,374639,44,110712,13,,,317465,19,372.9,0,,,,,,,,,,,,,
103
- msdos,3908M,,,400310,49,107278,14,,,343700,19,366.2,0,,,,,,,,,,,,,
104
- gpt,3750M,,,373408,44,106484,13,,,335955,22,383.3,0,,,,,,,,,,,,,
105
-
106
- gpt-raid0,3750M,,,379201,45,105965,13,,,334808,19,379.7,0,,,,,,,,,,,,,
107
- gpt-raid6,3750M,,,129930,19,47485,6,,,120628,8,360.5,0,,,,,,,,,,,,,
108
-
109
- # bonnie++ -f -d /mnt/md67 -s 3750 -n 0 -u root
109
+ ```
110
+ # bonnie++ -f -d /mnt/md66 -s 3750 -n 0 -u root
111
+ Version 1.03d ------Sequential Output------ --Sequential Input- --Random-
112
+ -Per Chr- --Block-- -Rewrite- -Per Chr- --Block-- --Seeks--
113
+ Machine Size K/sec %CP K/sec %CP K/sec %CP K/sec %CP K/sec %CP /sec %CP
114
+ msdos,3908M,,,374639,44,110712,13,,,317465,19,372.9,0,,,,,,,,,,,,,
115
+ msdos,3908M,,,400310,49,107278,14,,,343700,19,366.2,0,,,,,,,,,,,,,
116
+ gpt,3750M,,,373408,44,106484,13,,,335955,22,383.3,0,,,,,,,,,,,,,
117
+
118
+ gpt-raid0,3750M,,,379201,45,105965,13,,,334808,19,379.7,0,,,,,,,,,,,,,
119
+ gpt-raid6,3750M,,,129930,19,47485,6,,,120628,8,360.5,0,,,,,,,,,,,,,
120
+
121
+ # bonnie++ -f -d /mnt/md67 -s 3750 -n 0 -u root
122
+ ```
110 123
111 124
112 125
113 126
* No obvious difference for RAID0
114 127
115 128
* Stop
116
- # for dev in md66 md67 ; do umount /mnt/$dev ; mdadm --stop /dev/$dev ; done
129
+ ```
130
+ # for dev in md66 md67 ; do umount /mnt/$dev ; mdadm --stop /dev/$dev ; done
131
+ ```
117 132
118 133
119 134
... ...
@@ -215,19 +230,21 @@ NOTE: dd_rescue doesn't stop when it reaches the end of the output device!
215 230
216 231
217 232
* Taking dmcrypt out of the equation evens out IO load. It's misalignment of dmcrypt/ext4, not the Samsungs.
218
- # cat bench.raid0.nocrypt.chunk32.out
219
- hastur,3750M,,,429599,49,98253,12,,,310307,22,320.6,0,,,,,,,,,,,,,
220
- hastur,3750M,,,418548,49,100704,13,,,345801,24,309.8,0,,,,,,,,,,,,,
233
+ ```
234
+ # cat bench.raid0.nocrypt.chunk32.out
235
+ hastur,3750M,,,429599,49,98253,12,,,310307,22,320.6,0,,,,,,,,,,,,,
236
+ hastur,3750M,,,418548,49,100704,13,,,345801,24,309.8,0,,,,,,,,,,,,,
221 237
222 238
223
- # cat bench.raid0.nocrypt.chunk64.out
224
- hastur,3750M,,,426720,51,103288,13,,,304494,19,337.2,0,,,,,,,,,,,,,
225
- hastur,3750M,,,420937,50,105754,13,,,341867,19,333.3,0,,,,,,,,,,,,,
239
+ # cat bench.raid0.nocrypt.chunk64.out
240
+ hastur,3750M,,,426720,51,103288,13,,,304494,19,337.2,0,,,,,,,,,,,,,
241
+ hastur,3750M,,,420937,50,105754,13,,,341867,19,333.3,0,,,,,,,,,,,,,
226 242
227 243
228
- # cat bench.raid0.crypt_align256.chunk64.out
229
- hastur,3750M,,,99446,11,40287,6,,,98194,7,308.0,0,,,,,,,,,,,,,
230
- hastur,3750M,,,98978,12,40615,6,,,100049,7,307.4,0,,,,,,,,,,,,,
244
+ # cat bench.raid0.crypt_align256.chunk64.out
245
+ hastur,3750M,,,99446,11,40287,6,,,98194,7,308.0,0,,,,,,,,,,,,,
246
+ hastur,3750M,,,98978,12,40615,6,,,100049,7,307.4,0,,,,,,,,,,,,,
247
+ ```
231 248
232 249
233 250
... ...
@@ -258,6 +275,7 @@ NOTE: dd_rescue doesn't stop when it reaches the end of the output device!
258 275
* 300GB RAID0 (75GB per disk)
259 276
* 3850GB RAID6 (1925GB per disk)
260 277
278
+ ```
261 279
# for dev in {b..e} ; do parted /dev/sd$dev mklabel ; done
262 280
Warning: The existing disk label on /dev/sdb will be destroyed and all data on this disk will be lost. Do you want to continue?
263 281
Yes/No? y
... ...
@@ -265,6 +283,7 @@ NOTE: dd_rescue doesn't stop when it reaches the end of the output device!
265 283
...
266 284
267 285
# for dev in {b..e} ; do parted /dev/sd$dev -- mkpart primary 0 76800 mkpart primary 76800 -0 ; done
286
+ ```
268 287
269 288
270 289
* Buggy parted cli means creating labels manually.
... ...
@@ -300,7 +319,9 @@ NOTE: dd_rescue doesn't stop when it reaches the end of the output device!
300 319
* Small chunks for: few large I/O requests -> increased bandwidth
301 320
* Big chunks for: many small I/O requests (DB) -> increase IOPS (one disk per request)
302 321
322
+ ```
303 323
# mdadm --create --metadata=1.2 --verbose --chunk 64 --level=raid6 --raid-devices=4 /dev/md6 /dev/sd{b..e}
324
+ ```
304 325
305 326
306 327
... ...
@@ -327,8 +348,10 @@ NOTE: dd_rescue doesn't stop when it reaches the end of the output device!
327 348
* Specifying --hash for luksFormat [is supported][43] but seems largely unnecessary
328 349
* 2048 is safe offset
329 350
351
+ ```
330 352
# cryptsetup -c aes-xts-plain -s 512 --align-payload=2048 luksFormat /dev/md6
331 353
# cryptsetup luksOpen /dev/md6 crypt-md6
354
+ ```
332 355
333 356
334 357
... ...
@@ -347,8 +370,10 @@ NOTE: dd_rescue doesn't stop when it reaches the end of the output device!
347 370
348 371
349 372
* Check alignment
350
- # pvs /dev/sdb2 -o+pe_start
351
- /dev/dm-6 lvm2 -- 3.64T 3.64T 256.00K
373
+ ```
374
+ # pvs /dev/sdb2 -o+pe_start
375
+ /dev/dm-6 lvm2 -- 3.64T 3.64T 256.00K
376
+ ```
352 377
353 378
354 379
... ...
@@ -424,7 +449,9 @@ NOTE: dd_rescue doesn't stop when it reaches the end of the output device!
424 449
425 450
* No reserved space. Stride and stripe-width for 2 data disks and 64k chunk.
426 451
452
+ ```
427 453
# mkfs.ext4 -m 0 -E stride=16,stripe-width=32 /dev/vg-md6/home
454
+ ```
428 455
429 456
430 457
... ...
@@ -433,7 +460,9 @@ NOTE: dd_rescue doesn't stop when it reaches the end of the output device!
433 460
434 461
* No reserved space. 64k per inode. Stride and stripe-width for 2 data disks and 64k chunk.
435 462
463
+ ```
436 464
# mkfs.ext4 -m 0 -i 65536 -E stride=16,stripe-width=32 /dev/vg-md6/media
465
+ ```
437 466
438 467
439 468
... ...
@@ -458,23 +487,27 @@ NOTE: dd_rescue doesn't stop when it reaches the end of the output device!
458 487
459 488
### Start new array in 3-of-4 disk degraded state
460 489
490
+ ```
461 491
# mdadm --assemble --run /dev/md6 /dev/sd{h,i,j}
462 492
# cryptsetup luksOpen /dev/md6 crypt-md6
463 493
# vgchange -a y vg-md6
464 494
# mount /mnt/md6-media
465 495
# mount /mnt/md6-home
496
+ ```
466 497
467 498
468 499
469 500
470 501
### Start old array
471 502
503
+ ```
472 504
# mdadm --assemble /dev/md2
473 505
# mdadm --assemble /dev/md3
474 506
# cryptsetup luksOpen /dev/md2 crypt-md2
475 507
# cryptsetup luksOpen /dev/md3 crypt-md3
476 508
# mount /dev/mapper/crypt-md2
477 509
# mount /dev/mapper/crypt-md3
510
+ ```
478 511
479 512
480 513
... ...
@@ -491,7 +524,9 @@ NOTE: dd_rescue doesn't stop when it reaches the end of the output device!
491 524
492 525
### Samba
493 526
494
- # sed -i -e 's%md3/media%md6-media%g' /etc/samba/smb.conf
527
+```
528
+# sed -i -e 's%md3/media%md6-media%g' /etc/samba/smb.conf
529
+```
495 530
496 531
497 532
* Replace valid users lists with groups:
... ...
@@ -512,13 +547,17 @@ NOTE: dd_rescue doesn't stop when it reaches the end of the output device!
512 547
513 548
### Mediatomb
514 549
515
- # vim /etc/mediatomb/config.xml
516
- <home>/mnt/md6-media/metadata/mediatomb</home>
550
+```
551
+# vim /etc/mediatomb/config.xml
552
+ <home>/mnt/md6-media/metadata/mediatomb</home>
553
+```
517 554
518 555
519 556
### mtdaapd
520 557
521
- # sed -i -e 's%md3/media%md6-media%g' /etc/mtdaapd.conf
558
+```
559
+# sed -i -e 's%md3/media%md6-media%g' /etc/mtdaapd.conf
560
+```
522 561
523 562
524 563
... ...
@@ -527,8 +566,10 @@ NOTE: dd_rescue doesn't stop when it reaches the end of the output device!
527 566
528 567
### mediasys
529 568
530
- # for mntpt in /export/mediasys/media/{movies,videos,music,tvshows,photos}/{library,meta} ; do umount $mntpt ; done
531
- # for mntpt in $(grep '^[^#]*md6-media.*bind' /etc/fstab | cut -d' ' -f1) ; do mount $mntpt ; done
569
+```
570
+# for mntpt in /export/mediasys/media/{movies,videos,music,tvshows,photos}/{library,meta} ; do umount $mntpt ; done
571
+# for mntpt in $(grep '^[^#]*md6-media.*bind' /etc/fstab | cut -d' ' -f1) ; do mount $mntpt ; done
572
+```
532 573
533 574
534 575
... ...
@@ -539,7 +580,9 @@ NOTE: dd_rescue doesn't stop when it reaches the end of the output device!
539 580
540 581
### Add 4th new disk
541 582
542
- # mdadm --manage /dev/md6 --add /dev/sdb
583
+```
584
+# mdadm --manage /dev/md6 --add /dev/sdb
585
+```
543 586
544 587
545 588
... ...
@@ -549,11 +592,13 @@ NOTE: dd_rescue doesn't stop when it reaches the end of the output device!
549 592
## Benchmark
550 593
551 594
* Post sync. Read performance doesn't look right
552
- # bonnie++ -q -f -x 3 -s 3750 -n 0 -u root -d /mnt/md6-media
553
- name,file_size,putc,putc_cpu,put_block,put_block_cpu,rewrite,rewrite_cpu,getc,getc_cpu,get_block,get_block_cpu,seeks,seeks_cpu,num_files,seq_create,seq_create_cpu,seq_stat,seq_stat_cpu,seq_del,seq_del_cpu,ran_create,ran_create_cpu,ran_stat,ran_stat_cpu,ran_del,ran_del_cpu
554
- hastur,3750M,,,80818,11,24548,4,,,56356,5,305.6,0,,,,,,,,,,,,,
555
- hastur,3750M,,,86100,12,24631,4,,,59027,5,305.8,0,,,,,,,,,,,,,
556
- hastur,3750M,,,87435,11,24239,4,,,59217,5,312.1,0,,,,,,,,,,,,,
595
+ ```
596
+ # bonnie++ -q -f -x 3 -s 3750 -n 0 -u root -d /mnt/md6-media
597
+ name,file_size,putc,putc_cpu,put_block,put_block_cpu,rewrite,rewrite_cpu,getc,getc_cpu,get_block,get_block_cpu,seeks,seeks_cpu,num_files,seq_create,seq_create_cpu,seq_stat,seq_stat_cpu,seq_del,seq_del_cpu,ran_create,ran_create_cpu,ran_stat,ran_stat_cpu,ran_del,ran_del_cpu
598
+ hastur,3750M,,,80818,11,24548,4,,,56356,5,305.6,0,,,,,,,,,,,,,
599
+ hastur,3750M,,,86100,12,24631,4,,,59027,5,305.8,0,,,,,,,,,,,,,
600
+ hastur,3750M,,,87435,11,24239,4,,,59217,5,312.1,0,,,,,,,,,,,,,
601
+ ```
557 602
558 603
[39]: http://en.opensuse.org/Encrypted_Root_File_System
559 604
[41]: http://www.gnu.org/software/parted/faq.shtml