{"id":275,"date":"2016-09-19T22:22:16","date_gmt":"2016-09-19T12:22:16","guid":{"rendered":"https:\/\/icicimov.com\/blog\/?p=275"},"modified":"2017-01-02T22:25:03","modified_gmt":"2017-01-02T11:25:03","slug":"adding-iscsi-shared-volume-to-proxmox-to-support-live-migration","status":"publish","type":"post","link":"https:\/\/icicimov.com\/blog\/?p=275","title":{"rendered":"Adding iSCSI shared volume to Proxmox to support Live Migration"},"content":{"rendered":"

We will use Multipath<\/code> for link HA and improved performance. Install the needed packages first:<\/p>\n

root@proxmox01:~# apt-get install open-iscsi multipath-tools\n<\/code><\/pre>\n
Then we discover the target and login:<\/p>\n
root@proxmox01:~# systemctl start open-iscsi.service\n\nroot@proxmox01:~# iscsiadm -m discovery -t st -p 192.168.0.180\n192.168.0.180:3260,1 iqn.2016-02.local.virtual:virtual.vg1\n10.20.1.180:3260,1 iqn.2016-02.local.virtual:virtual.vg1\n\nroot@proxmox01:~# iscsiadm -m node --login\n\nroot@proxmox01:~# iscsiadm -m node\n10.20.1.180:3260,1 iqn.2016-02.local.virtual:virtual.vg1\n192.168.0.180:3260,1 iqn.2016-02.local.virtual:virtual.vg1\n\nroot@proxmox02:~# iscsiadm -m session -P 1\nTarget: iqn.2016-02.local.virtual:virtual.vg1 (non-flash)\n    Current Portal: 10.20.1.180:3260,1\n    Persistent Portal: 10.20.1.180:3260,1\n        **********\n        Interface:\n        **********\n        Iface Name: default\n        Iface Transport: tcp\n        Iface Initiatorname: iqn.1993-08.org.debian:01:674b46a9745\n        Iface IPaddress: 10.20.1.186\n        Iface HWaddress: \n        Iface Netdev: \n        SID: 1\n        iSCSI Connection State: LOGGED IN\n        iSCSI Session State: LOGGED_IN\n        Internal iscsid Session State: NO CHANGE\n    Current Portal: 192.168.0.180:3260,1\n    Persistent Portal: 192.168.0.180:3260,1\n        **********\n        Interface:\n        **********\n        Iface Name: default\n        Iface Transport: tcp\n        Iface Initiatorname: iqn.1993-08.org.debian:01:674b46a9745\n        Iface IPaddress: 192.168.0.186\n        Iface HWaddress: \n        Iface Netdev: \n        SID: 2\n        iSCSI Connection State: LOGGED IN\n        iSCSI Session State: LOGGED_IN\n        Internal iscsid Session State: NO CHANGE\n\nroot@proxmox01:~# lsscsi\n[0:0:0:0]    cd\/dvd  QEMU     QEMU DVD-ROM     1.4.  \/dev\/sr0\n[2:0:0:0]    disk    SCST_FIO VDISK-LUN01       311  \/dev\/sdb\n[3:0:0:0]    disk    SCST_FIO VDISK-LUN01       311  \/dev\/sda\n<\/code><\/pre>\n
We can see two new SCSI block devices have been introduced to the system, \/dev\/sda<\/code> and \/dev\/sdb<\/code>. Next is Multipathing<\/code> setup. First we find the WWID of the new device:<\/p>\n
root@proxmox01:~# \/lib\/udev\/scsi_id -g -d \/dev\/sda\n23238363932313833\n\nroot@proxmox01:~# \/lib\/udev\/scsi_id -g -d \/dev\/sdb\n23238363932313833\n<\/code><\/pre>\n
that we then use in the Multipath<\/code> config file \/etc\/multipath.conf<\/code> that we create:<\/p>\n
defaults {\n    user_friendly_names    yes\n        polling_interval        2\n        path_selector           \"round-robin 0\"\n        path_grouping_policy    multibus\n        path_checker            readsector0\n        getuid_callout          \"\/lib\/udev\/scsi_id -g -u -d \/dev\/%n\"\n        rr_min_io               100\n        failback                immediate\n        no_path_retry           queue\n}\nblacklist {\n        wwid .*\n}\nblacklist_exceptions {\n        wwid \"23238363932313833\"\n    property \"(ID_SCSI_VPD|ID_WWN|ID_SERIAL)\"\n}\nmultipaths {\n  multipath {\n        wwid \"23238363932313833\"\n        alias mylun\n  }\n}\n<\/code><\/pre>\n
and after loading Multipath kernel modules:<\/p>\n
root@proxmox02:~# modprobe -v dm_multipath\ninsmod \/lib\/modules\/4.2.6-1-pve\/kernel\/drivers\/scsi\/device_handler\/scsi_dh.ko\ninsmod \/lib\/modules\/4.2.6-1-pve\/kernel\/drivers\/md\/dm-multipath.ko\n\nroot@proxmox02:~# modprobe -v dm_round_robin\ninsmod \/lib\/modules\/4.2.6-1-pve\/kernel\/drivers\/md\/dm-round-robin.ko\n<\/code><\/pre>\n
and restarting Multipath service\/daemon:<\/p>\n
root@proxmox01:~# systemctl stop multipath-tools.service\nroot@proxmox01:~# systemctl start multipath-tools.service\nroot@proxmox02:~# systemctl status -l multipath-tools.service\n   multipath-tools.service - LSB: multipath daemon\n   Loaded: loaded (\/etc\/init.d\/multipath-tools)\n   Active: active (running) since Fri 2016-03-04 17:40:44 AEDT; 6s ago\n  Process: 8177 ExecStop=\/etc\/init.d\/multipath-tools stop (code=exited, status=0\/SUCCESS)\n  Process: 8191 ExecStart=\/etc\/init.d\/multipath-tools start (code=exited, status=0\/SUCCESS)\n   CGroup: \/system.slice\/multipath-tools.service\n           \u2514\u25008195 \/sbin\/multipathd\n\nMar 04 17:40:44 proxmox02 multipath-tools[8191]: Starting multipath daemon: multipathd.\nMar 04 17:40:44 proxmox02 multipathd[8195]: sda: using deprecated getuid callout\nMar 04 17:40:44 proxmox02 multipathd[8195]: sdb: using deprecated getuid callout\nMar 04 17:40:44 proxmox02 multipathd[8195]: mylun: load table [0 41934848 multipath 1 queue_if_no_path 0 1 1 round-robin 0 2 1 8:0 1 8:16 1]\nMar 04 17:40:44 proxmox02 multipathd[8195]: mylun: event checker started\nMar 04 17:40:44 proxmox02 multipathd[8195]: path checkers start up\n<\/code><\/pre>\n
we can see the multipath device mylun<\/code> created:<\/p>\n
root@proxmox01:~# multipath -ll\nmylun (23238363932313833) dm-1 SCST_FIO,VDISK-LUN01\nsize=20G features='1 queue_if_no_path' hwhandler='0' wp=rw\n`-+- policy='round-robin 0' prio=1 status=active\n  |- 2:0:0:0 sdb 8:16 active ready running\n  `- 3:0:0:0 sda 8:0  active ready running\n\nroot@proxmox01:~# dmsetup ls\npve-swap    (251:3)\npve-root    (251:0)\nmylun    (251:1)\npve-data    (251:4)\nvg_drbd0-vm--107--disk--1    (251:5)\nvg_proxmox-lv_proxmox    (251:2)\n<\/code><\/pre>\n
Next we create the volume group on one node only:<\/p>\n
root@proxmox01:~# pvcreate \/dev\/mapper\/mylun\nroot@proxmox01:~# vgcreate vg_iscsi \/dev\/mapper\/mylun\n<\/code><\/pre>\n
Finally, using the PVE web GUI, we add new LVM storage using the newly created vg_iscsi<\/code> volume group in the Datacenter<\/code> and set it as Active<\/code> and Shared<\/code>.<\/p>\n
At the end, in order to log in and establish the sessions to the iSCSI targets on reboot we set the startup mode to automatic<\/code> in the iscsi initiator<\/code> config file \/etc\/iscsi\/iscsid.conf<\/code>:<\/p>\n
[...]\nnode.startup = automatic\n[...]\n<\/code><\/pre>\n
We can see all the shared VG’s and volumes created in the cluster on both nodes for all 3 storage types we have setup above:<\/p>\n
root@proxmox01:~# vgs\n  VG         #PV #LV #SN Attr   VSize  VFree\n  pve          1   3   0 wz--n- 31.87g  3.87g\n  vg_drbd0     1   1   0 wz--n-  9.31g  3.31g\n  vg_drbd1     1   1   0 wz--n- 11.68g  5.68g\n  vg_iscsi     1   1   0 wz--n- 19.99g 13.99g\n  vg_proxmox   1   1   0 wz--n- 20.00g     0\nroot@proxmox01:~# lvs\n  LV            VG         Attr       LSize  Pool Origin Data%  Meta%  Move Log Cpy%Sync Convert\n  data          pve        -wi-ao---- 16.38g                                                   \n  root          pve        -wi-ao----  7.75g                                                   \n  swap          pve        -wi-ao----  3.88g                                                   \n  vm-107-disk-1 vg_drbd0   -wi-ao----  6.00g                                                   \n  vm-106-disk-1 vg_drbd1   -wi-------  6.00g                                                   \n  vm-108-disk-1 vg_iscsi   -wi-------  6.00g                                                   \n  lv_proxmox    vg_proxmox -wi-ao---- 20.00g\n\nroot@proxmox02:~# vgs\n  VG         #PV #LV #SN Attr   VSize  VFree\n  pve          1   3   0 wz--n- 31.87g  3.87g\n  vg_drbd0     1   1   0 wz--n-  9.31g  3.31g\n  vg_drbd1     1   1   0 wz--n- 11.68g  5.68g\n  vg_iscsi     1   1   0 wz--n- 19.99g 13.99g\n  vg_proxmox   1   1   0 wz--n- 20.00g     0\nroot@proxmox02:~# lvs\n  LV            VG         Attr       LSize  Pool Origin Data%  Meta%  Move Log Cpy%Sync Convert\n  data          pve        -wi-ao---- 16.38g                                                   \n  root          pve        -wi-ao----  7.75g                                                   \n  swap          pve        -wi-ao----  3.88g                                                   \n  vm-107-disk-1 vg_drbd0   -wi-------  6.00g                                                   \n  vm-106-disk-1 vg_drbd1   -wi-ao----  6.00g                                                   \n  vm-108-disk-1 vg_iscsi   -wi-ao----  6.00g                                                   \n  lv_proxmox    vg_proxmox -wi-ao---- 20.00g\n<\/code><\/pre>\n
[serialposts]<\/p>\n","protected":false},"excerpt":{"rendered":"
We will use Multipath for link HA and improved performance. Install the needed packages first: root@proxmox01:~# apt-get install open-iscsi multipath-tools Then we discover the target and login: root@proxmox01:~# systemctl start open-iscsi.service root@proxmox01:~# iscsiadm -m discovery -t st -p 192.168.0.180 192.168.0.180:3260,1…<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[17,9,22,16,13],"tags":[26,25,18,24,23],"class_list":["post-275","post","type-post","status-publish","format-standard","hentry","category-cluster","category-high-availability","category-kvm","category-storage","category-virtualization","tag-cluster","tag-high-availability","tag-iscsi","tag-kvm","tag-proxmox"],"_links":{"self":[{"href":"https:\/\/icicimov.com\/blog\/index.php?rest_route=\/wp\/v2\/posts\/275","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/icicimov.com\/blog\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/icicimov.com\/blog\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/icicimov.com\/blog\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/icicimov.com\/blog\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=275"}],"version-history":[{"count":1,"href":"https:\/\/icicimov.com\/blog\/index.php?rest_route=\/wp\/v2\/posts\/275\/revisions"}],"predecessor-version":[{"id":276,"href":"https:\/\/icicimov.com\/blog\/index.php?rest_route=\/wp\/v2\/posts\/275\/revisions\/276"}],"wp:attachment":[{"href":"https:\/\/icicimov.com\/blog\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=275"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/icicimov.com\/blog\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=275"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/icicimov.com\/blog\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=275"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}