From: Mike Christie <mchristi@redhat.com> Date: Thu, 18 Feb 2010 22:11:46 -0500 Subject: [scsi] be2iscsi: fix eh bugs and enable new hw support Message-id: <1266531106-2609-1-git-send-email-mchristi@redhat.com> Patchwork-id: 23356 O-Subject: [PATCH 1/1 RHEL 5.5] fix be2iscsi eh bugs and enable Bugzilla: 564145 RH-Acked-by: Rob Evers <revers@redhat.com> From: Mike Christie <mchristi@redhat.com> This is for BZ 564145. This patches fixes the scsi eh callouts for the driver. They abort path was sleepging with a spin lock and the device reset code was not cleaning up all the cmds in firmware. It also enables their newer card by setting it up in the pci device table. The driver is new in RHEL 5.5, so there is no chance for regression. Emulex and ServerEngines are also doing all the testing so it will not invalidate that work. The patch to add support for their newer card is in the scsi maintianer's tree http://git.kernel.org/?p=linux/kernel/git/jejb/scsi-misc-2.6.git;a=commitdiff;h=f98c96b0b6572b5491e954148509b20f08f31491;hp=b3925514f6f6878e3f671fb6990874e731a26d02 The other patches are not upstream. They have been sent to the scsi list here: http://marc.info/?l=linux-scsi&m=126645774916225&w=2 http://marc.info/?l=linux-scsi&m=126645779116279&w=2 http://marc.info/?l=linux-scsi&m=126645768716156&w=2 http://marc.info/?l=linux-scsi&m=126645782516317&w=2 http://marc.info/?l=linux-scsi&m=126645790316368&w=2 My card does not work so well. I was able to test the scsi eh callout code, because with my busted card IO ends up timing out normally. We are relying on SE and Emulex for the testing. diff --git a/drivers/scsi/be2iscsi/be.h b/drivers/scsi/be2iscsi/be.h index 9ba9d77..00df7d7 100644 --- a/drivers/scsi/be2iscsi/be.h +++ b/drivers/scsi/be2iscsi/be.h @@ -28,6 +28,9 @@ #define MCC_Q_LEN 128 #define MCC_CQ_LEN 256 #define MAX_MCC_CMD 16 +/* BladeEngine Generation numbers */ +#define BE_GEN2 2 +#define BE_GEN3 3 struct be_dma_mem { void *va; diff --git a/drivers/scsi/be2iscsi/be_cmds.c b/drivers/scsi/be2iscsi/be_cmds.c index b103349..42ab379 100644 --- a/drivers/scsi/be2iscsi/be_cmds.c +++ b/drivers/scsi/be2iscsi/be_cmds.c @@ -33,19 +33,13 @@ void be_mcc_notify(struct beiscsi_hba *phba) unsigned int alloc_mcc_tag(struct beiscsi_hba *phba) { unsigned int tag = 0; - unsigned int num = 0; -mcc_tag_rdy: if (phba->ctrl.mcc_tag_available) { tag = phba->ctrl.mcc_tag[phba->ctrl.mcc_alloc_index]; phba->ctrl.mcc_tag[phba->ctrl.mcc_alloc_index] = 0; phba->ctrl.mcc_numtag[tag] = 0; - } else { - udelay(100); - num++; - if (num < mcc_timeout) - goto mcc_tag_rdy; } + if (tag) { phba->ctrl.mcc_tag_available--; if (phba->ctrl.mcc_alloc_index == (MAX_MCC_CMD - 1)) diff --git a/drivers/scsi/be2iscsi/be_iscsi.c b/drivers/scsi/be2iscsi/be_iscsi.c index 2aa598f..b77de14 100644 --- a/drivers/scsi/be2iscsi/be_iscsi.c +++ b/drivers/scsi/be2iscsi/be_iscsi.c @@ -494,7 +494,7 @@ static int beiscsi_open_conn(struct iscsi_endpoint *ep, tag = mgmt_open_connection(phba, dst_addr, beiscsi_ep); if (!tag) { SE_DEBUG(DBG_LVL_1, - "mgmt_invalidate_connection Failed for cid=%d \n", + "mgmt_open_connection Failed for cid=%d \n", beiscsi_ep->ep_cid); } else { wait_event_interruptible(phba->ctrl.mcc_wait[tag], @@ -707,7 +707,7 @@ void beiscsi_conn_stop(struct iscsi_cls_conn *cls_conn, int flag) SE_DEBUG(DBG_LVL_8, "In beiscsi_conn_stop ep_cid = %d\n", beiscsi_ep->ep_cid); tag = mgmt_invalidate_connection(phba, beiscsi_ep, - beiscsi_ep->ep_cid, 1, + beiscsi_ep->ep_cid, 0, savecfg_flag); if (!tag) { SE_DEBUG(DBG_LVL_1, diff --git a/drivers/scsi/be2iscsi/be_main.c b/drivers/scsi/be2iscsi/be_main.c index 013836d..8772f6e 100644 --- a/drivers/scsi/be2iscsi/be_main.c +++ b/drivers/scsi/be2iscsi/be_main.c @@ -78,16 +78,132 @@ static int beiscsi_host_reset(struct scsi_cmnd *sc) return SUCCESS; } +static int beiscsi_eh_abort(struct scsi_cmnd *sc) +{ + struct iscsi_cls_session *cls_session; + struct iscsi_task *aborted_task = (struct iscsi_task *)sc->SCp.ptr; + struct beiscsi_io_task *aborted_io_task; + struct iscsi_conn *conn; + struct beiscsi_conn *beiscsi_conn; + struct beiscsi_hba *phba; + struct iscsi_session *session; + struct invalidate_command_table *inv_tbl; + unsigned int cid, tag, num_invalidate; + + cls_session = starget_to_session(scsi_target(sc->device)); + session = cls_session->dd_data; + + spin_lock_bh(&session->lock); + if (!aborted_task || !aborted_task->sc) { + /* we raced */ + spin_unlock_bh(&session->lock); + return SUCCESS; + } + + aborted_io_task = aborted_task->dd_data; + if (!aborted_io_task->scsi_cmnd) { + /* raced or invalid command */ + spin_unlock_bh(&session->lock); + return SUCCESS; + } + spin_unlock_bh(&session->lock); + conn = aborted_task->conn; + beiscsi_conn = conn->dd_data; + phba = beiscsi_conn->phba; + + /* invalidate iocb */ + cid = beiscsi_conn->beiscsi_conn_cid; + inv_tbl = phba->inv_tbl; + memset(inv_tbl, 0x0, sizeof(*inv_tbl)); + inv_tbl->cid = cid; + inv_tbl->icd = aborted_io_task->psgl_handle->sgl_index; + num_invalidate = 1; + tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate, cid); + if (!tag) { + shost_printk(KERN_WARNING, phba->shost, + "mgmt_invalidate_icds could not be" + " submitted\n"); + return FAILED; + } else { + wait_event_interruptible(phba->ctrl.mcc_wait[tag], + phba->ctrl.mcc_numtag[tag]); + free_mcc_tag(&phba->ctrl, tag); + } + + return iscsi2_eh_abort(sc); +} + +static int beiscsi_eh_device_reset(struct scsi_cmnd *sc) +{ + struct iscsi_task *abrt_task; + struct beiscsi_io_task *abrt_io_task; + struct iscsi_conn *conn; + struct beiscsi_conn *beiscsi_conn; + struct beiscsi_hba *phba; + struct iscsi_session *session; + struct iscsi_cls_session *cls_session; + struct invalidate_command_table *inv_tbl; + unsigned int cid, tag, i, num_invalidate; + int rc = FAILED; + + /* invalidate iocbs */ + cls_session = starget_to_session(scsi_target(sc->device)); + session = cls_session->dd_data; + spin_lock_bh(&session->lock); + if (!session->leadconn || session->state != ISCSI_STATE_LOGGED_IN) + goto unlock; + + conn = session->leadconn; + beiscsi_conn = conn->dd_data; + phba = beiscsi_conn->phba; + cid = beiscsi_conn->beiscsi_conn_cid; + inv_tbl = phba->inv_tbl; + memset(inv_tbl, 0x0, sizeof(*inv_tbl) * BE2_CMDS_PER_CXN); + num_invalidate = 0; + for (i = 0; i < conn->session->cmds_max; i++) { + abrt_task = conn->session->cmds[i]; + abrt_io_task = abrt_task->dd_data; + if (!abrt_task->sc || abrt_task->state == ISCSI_TASK_FREE) + continue; + + if (abrt_task->sc->device->lun != abrt_task->sc->device->lun) + continue; + + inv_tbl->cid = cid; + inv_tbl->icd = abrt_io_task->psgl_handle->sgl_index; + num_invalidate++; + inv_tbl++; + } + spin_unlock_bh(&session->lock); + inv_tbl = phba->inv_tbl; + + tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate, cid); + if (!tag) { + shost_printk(KERN_WARNING, phba->shost, + "mgmt_invalidate_icds could not be" + " submitted\n"); + return FAILED; + } else { + wait_event_interruptible(phba->ctrl.mcc_wait[tag], + phba->ctrl.mcc_numtag[tag]); + free_mcc_tag(&phba->ctrl, tag); + } + + return iscsi2_eh_device_reset(sc); +unlock: + spin_unlock_bh(&session->lock); + return rc; +} + static struct scsi_host_template beiscsi_sht = { .module = THIS_MODULE, .name = "ServerEngines 10Gbe open-iscsi Initiator Driver", .proc_name = DRV_NAME, .queuecommand = iscsi2_queuecommand, - .eh_abort_handler = iscsi2_eh_abort, .change_queue_depth = iscsi2_change_queue_depth, .slave_configure = beiscsi_slave_configure, - .eh_device_reset_handler = iscsi2_eh_device_reset, - .eh_host_reset_handler = beiscsi_host_reset, + .eh_abort_handler = beiscsi_eh_abort, + .eh_device_reset_handler = beiscsi_eh_device_reset, .sg_tablesize = BEISCSI_SGLIST_ELEMENTS, .can_queue = BE2_IO_DEPTH, .this_id = -1, @@ -98,10 +214,10 @@ static struct scsi_host_template beiscsi_sht = { static DEFINE_PCI_DEVICE_TABLE(beiscsi_pci_id_table) = { { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) }, + { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) }, { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) }, { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) }, { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID3) }, - { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID4) }, { 0 } }; MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table); @@ -161,6 +277,7 @@ static int beiscsi_map_pci_bars(struct beiscsi_hba *phba, struct pci_dev *pcidev) { u8 __iomem *addr; + int pcicfg_reg; addr = ioremap_nocache(pci_resource_start(pcidev, 2), pci_resource_len(pcidev, 2)); @@ -177,8 +294,13 @@ static int beiscsi_map_pci_bars(struct beiscsi_hba *phba, phba->db_va = addr; phba->db_pa.u.a64.address = pci_resource_start(pcidev, 4); - addr = ioremap_nocache(pci_resource_start(pcidev, 1), - pci_resource_len(pcidev, 1)); + if (phba->generation == BE_GEN2) + pcicfg_reg = 1; + else + pcicfg_reg = 0; + + addr = ioremap_nocache(pci_resource_start(pcidev, pcicfg_reg), + pci_resource_len(pcidev, pcicfg_reg)); if (addr == NULL) goto pci_map_err; phba->ctrl.pcicfg = addr; @@ -957,26 +1079,24 @@ static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn, case HWH_TYPE_IO: case HWH_TYPE_IO_RD: if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == - ISCSI_OP_NOOP_OUT) { + ISCSI_OP_NOOP_OUT) be_complete_nopin_resp(beiscsi_conn, task, psol); - } else + else be_complete_io(beiscsi_conn, task, psol); break; case HWH_TYPE_LOGOUT: - be_complete_logout(beiscsi_conn, task, psol); + if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGOUT) + be_complete_logout(beiscsi_conn, task, psol); + else + be_complete_tmf(beiscsi_conn, task, psol); break; - case HWH_TYPE_LOGIN: SE_DEBUG(DBG_LVL_1, "\t\t No HWH_TYPE_LOGIN Expected in hwi_complete_cmd" "- Solicited path \n"); break; - case HWH_TYPE_TMF: - be_complete_tmf(beiscsi_conn, task, psol); - break; - case HWH_TYPE_NOP: be_complete_nopin_resp(beiscsi_conn, task, psol); break; @@ -3084,10 +3204,16 @@ static unsigned char hwi_enable_intr(struct beiscsi_hba *phba) reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; SE_DEBUG(DBG_LVL_8, "reg =x%08x addr=%p \n", reg, addr); iowrite32(reg, addr); - for (i = 0; i <= phba->num_cpus; i++) { - eq = &phwi_context->be_eq[i].q; + if (!phba->msix_enabled) { + eq = &phwi_context->be_eq[0].q; SE_DEBUG(DBG_LVL_8, "eq->id=%d \n", eq->id); hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1); + } else { + for (i = 0; i <= phba->num_cpus; i++) { + eq = &phwi_context->be_eq[i].q; + SE_DEBUG(DBG_LVL_8, "eq->id=%d \n", eq->id); + hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1); + } } } else shost_printk(KERN_WARNING, phba->shost, @@ -3362,7 +3488,6 @@ static int beiscsi_alloc_pdu(struct iscsi_task *task, uint8_t opcode) wrb_index << 16) | (unsigned int) (io_task->psgl_handle->sgl_index)); io_task->pwrb_handle->pio_handle = task; - io_task->cmd_bhs->iscsi_hdr.itt = itt; return 0; @@ -3448,11 +3573,11 @@ static int beiscsi_iotask(struct iscsi_task *task, struct scatterlist *sg, AMAP_SET_BITS(struct amap_pdu_data_out, final_bit, &io_task->cmd_bhs->iscsi_data_pdu, 1); AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, - INI_WR_CMD); + INI_WR_CMD); AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1); } else { AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, - INI_RD_CMD); + INI_RD_CMD); AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0); } memcpy(&io_task->cmd_bhs->iscsi_data_pdu. @@ -3487,19 +3612,13 @@ static int beiscsi_iotask(struct iscsi_task *task, struct scatterlist *sg, static int beiscsi_mtask(struct iscsi_task *task) { - struct beiscsi_io_task *aborted_io_task, *io_task = task->dd_data; + struct beiscsi_io_task *io_task = task->dd_data; struct iscsi_conn *conn = task->conn; struct beiscsi_conn *beiscsi_conn = conn->dd_data; struct beiscsi_hba *phba = beiscsi_conn->phba; - struct iscsi_session *session; struct iscsi_wrb *pwrb = NULL; - struct hwi_controller *phwi_ctrlr; - struct hwi_wrb_context *pwrb_context; - struct wrb_handle *pwrb_handle; unsigned int doorbell = 0; - unsigned int i, cid; - struct iscsi_task *aborted_task; - unsigned int tag; + unsigned int cid; cid = beiscsi_conn->beiscsi_conn_cid; pwrb = io_task->pwrb_handle->pwrb; @@ -3521,7 +3640,7 @@ static int beiscsi_mtask(struct iscsi_task *task) break; case ISCSI_OP_NOOP_OUT: AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, - INI_RD_CMD); + INI_RD_CMD); if (task->hdr->ttt == ISCSI_RESERVED_TAG) AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0); else @@ -3535,35 +3654,8 @@ static int beiscsi_mtask(struct iscsi_task *task) hwi_write_buffer(pwrb, task); break; case ISCSI_OP_SCSI_TMFUNC: - session = conn->session; - i = ((struct iscsi_tm *)task->hdr)->rtt; - phwi_ctrlr = phba->phwi_ctrlr; - pwrb_context = &phwi_ctrlr->wrb_context[cid - - phba->fw_config.iscsi_cid_start]; - pwrb_handle = pwrb_context->pwrb_handle_basestd[be32_to_cpu(i) - >> 16]; - aborted_task = pwrb_handle->pio_handle; - if (!aborted_task) - return 0; - - aborted_io_task = aborted_task->dd_data; - if (!aborted_io_task->scsi_cmnd) - return 0; - - tag = mgmt_invalidate_icds(phba, - aborted_io_task->psgl_handle->sgl_index, - cid); - if (!tag) { - shost_printk(KERN_WARNING, phba->shost, - "mgmt_invalidate_icds could not be" - " submitted\n"); - } else { - wait_event_interruptible(phba->ctrl.mcc_wait[tag], - phba->ctrl.mcc_numtag[tag]); - free_mcc_tag(&phba->ctrl, tag); - } AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb, - INI_TMF_CMD); + INI_TMF_CMD); AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0); hwi_write_buffer(pwrb, task); break; @@ -3596,17 +3688,12 @@ static int beiscsi_mtask(struct iscsi_task *task) static int beiscsi_task_xmit(struct iscsi_task *task) { - struct iscsi_conn *conn = task->conn; struct beiscsi_io_task *io_task = task->dd_data; struct scsi_cmnd *sc = task->sc; - struct beiscsi_conn *beiscsi_conn = conn->dd_data; struct scatterlist *sg; int num_sg; unsigned int writedir = 0, xferlen = 0; - SE_DEBUG(DBG_LVL_4, "\n cid=%d In beiscsi_task_xmit task=%p conn=%p \t" - "beiscsi_conn=%p \n", beiscsi_conn->beiscsi_conn_cid, - task, conn, beiscsi_conn); if (!sc) return beiscsi_mtask(task); @@ -3710,6 +3797,20 @@ static int __devinit beiscsi_dev_probe(struct pci_dev *pcidev, goto disable_pci; } + switch (pcidev->device) { + case BE_DEVICE_ID1: + case OC_DEVICE_ID1: + case OC_DEVICE_ID2: + phba->generation = BE_GEN2; + break; + case BE_DEVICE_ID2: + case OC_DEVICE_ID3: + phba->generation = BE_GEN3; + break; + default: + phba->generation = 0; + } + if (enable_msix) num_cpus = find_num_cpus(); else diff --git a/drivers/scsi/be2iscsi/be_main.h b/drivers/scsi/be2iscsi/be_main.h index 49415a0..ae85b93 100644 --- a/drivers/scsi/be2iscsi/be_main.h +++ b/drivers/scsi/be2iscsi/be_main.h @@ -40,11 +40,14 @@ #define DRV_DESC BE_NAME " " "Driver" #define BE_VENDOR_ID 0x19A2 +/* DEVICE ID's for BE2 */ #define BE_DEVICE_ID1 0x212 #define OC_DEVICE_ID1 0x702 #define OC_DEVICE_ID2 0x703 + +/* DEVICE ID's for BE3 */ +#define BE_DEVICE_ID2 0x222 #define OC_DEVICE_ID3 0x712 -#define OC_DEVICE_ID4 0x222 #define BE2_IO_DEPTH 1024 #define BE2_MAX_SESSIONS 256 @@ -259,6 +262,11 @@ struct hba_parameters { unsigned int num_sge; }; +struct invalidate_command_table { + unsigned short icd; + unsigned short cid; +} __packed; + struct beiscsi_hba { struct hba_parameters params; struct hwi_controller *phwi_ctrlr; @@ -330,6 +338,8 @@ struct beiscsi_hba { struct workqueue_struct *wq; /* The actuak work queue */ struct work_struct work_cqs; /* The work being queued */ struct be_ctrl_info ctrl; + unsigned int generation; + struct invalidate_command_table inv_tbl[128]; }; struct beiscsi_session { diff --git a/drivers/scsi/be2iscsi/be_mgmt.c b/drivers/scsi/be2iscsi/be_mgmt.c index dd55312..5fe95e4 100644 --- a/drivers/scsi/be2iscsi/be_mgmt.c +++ b/drivers/scsi/be2iscsi/be_mgmt.c @@ -85,6 +85,7 @@ unsigned char mgmt_check_supported_fw(struct be_ctrl_info *ctrl, } nonemb_cmd.size = sizeof(struct be_mgmt_controller_attributes); req = nonemb_cmd.va; + memset(req, 0, sizeof(*req)); spin_lock(&ctrl->mbox_lock); memset(wrb, 0, sizeof(*wrb)); be_wrb_hdr_prepare(wrb, sizeof(*req), false, 1); @@ -144,14 +145,15 @@ unsigned char mgmt_epfw_cleanup(struct beiscsi_hba *phba, unsigned short chute) } unsigned char mgmt_invalidate_icds(struct beiscsi_hba *phba, - unsigned int icd, unsigned int cid) + struct invalidate_command_table *inv_tbl, + unsigned int num_invalidate, unsigned int cid) { struct be_dma_mem nonemb_cmd; struct be_ctrl_info *ctrl = &phba->ctrl; struct be_mcc_wrb *wrb; struct be_sge *sge; struct invalidate_commands_params_in *req; - unsigned int tag = 0; + unsigned int i, tag = 0; spin_lock(&ctrl->mbox_lock); tag = alloc_mcc_tag(phba); @@ -171,6 +173,7 @@ unsigned char mgmt_invalidate_icds(struct beiscsi_hba *phba, } nonemb_cmd.size = sizeof(struct invalidate_commands_params_in); req = nonemb_cmd.va; + memset(req, 0, sizeof(*req)); wrb = wrb_from_mccq(phba); sge = nonembedded_sgl(wrb); wrb->tag0 |= tag; @@ -181,9 +184,12 @@ unsigned char mgmt_invalidate_icds(struct beiscsi_hba *phba, sizeof(*req)); req->ref_handle = 0; req->cleanup_type = CMD_ISCSI_COMMAND_INVALIDATE; - req->icd_count = 0; - req->table[req->icd_count].icd = icd; - req->table[req->icd_count].cid = cid; + for (i = 0; i < num_invalidate; i++) { + req->table[i].icd = inv_tbl->icd; + req->table[i].cid = inv_tbl->cid; + req->icd_count++; + inv_tbl++; + } sge->pa_hi = cpu_to_le32(upper_32_bits(nonemb_cmd.dma)); sge->pa_lo = cpu_to_le32(nonemb_cmd.dma & 0xFFFFFFFF); sge->len = cpu_to_le32(nonemb_cmd.size); diff --git a/drivers/scsi/be2iscsi/be_mgmt.h b/drivers/scsi/be2iscsi/be_mgmt.h index ecead6a..3d316b8 100644 --- a/drivers/scsi/be2iscsi/be_mgmt.h +++ b/drivers/scsi/be2iscsi/be_mgmt.h @@ -94,7 +94,8 @@ unsigned char mgmt_upload_connection(struct beiscsi_hba *phba, unsigned short cid, unsigned int upload_flag); unsigned char mgmt_invalidate_icds(struct beiscsi_hba *phba, - unsigned int icd, unsigned int cid); + struct invalidate_command_table *inv_tbl, + unsigned int num_invalidate, unsigned int cid); struct iscsi_invalidate_connection_params_in { struct be_cmd_req_hdr hdr; @@ -116,11 +117,6 @@ union iscsi_invalidate_connection_params { struct iscsi_invalidate_connection_params_out response; } __packed; -struct invalidate_command_table { - unsigned short icd; - unsigned short cid; -} __packed; - struct invalidate_commands_params_in { struct be_cmd_req_hdr hdr; unsigned int ref_handle;