3  * Copyright (c) 2013-2014 Mellanox Technologies. All rights reserved.
15  *	- Redistributions of source code must retain the above
19  *	- Redistributions in binary form must reproduce the above
50 	struct iser_fr_pool *fr_pool = &ib_conn->fr_pool;  in iser_reg_desc_get_fr()
54 	spin_lock_irqsave(&fr_pool->lock, flags);  in iser_reg_desc_get_fr()
55 	desc = list_first_entry(&fr_pool->list,  in iser_reg_desc_get_fr()
57 	list_del(&desc->list);  in iser_reg_desc_get_fr()
58 	spin_unlock_irqrestore(&fr_pool->lock, flags);  in iser_reg_desc_get_fr()
67 	struct iser_fr_pool *fr_pool = &ib_conn->fr_pool;  in iser_reg_desc_put_fr()
70 	spin_lock_irqsave(&fr_pool->lock, flags);  in iser_reg_desc_put_fr()
71 	list_add(&desc->list, &fr_pool->list);  in iser_reg_desc_put_fr()
72 	spin_unlock_irqrestore(&fr_pool->lock, flags);  in iser_reg_desc_put_fr()
82 	iser_task->dir[iser_dir] = 1;  in iser_dma_map_task_data()
83 	dev = iser_task->iser_conn->ib_conn.device->ib_device;  in iser_dma_map_task_data()
85 	data->dma_nents = ib_dma_map_sg(dev, data->sg, data->size, dma_dir);  in iser_dma_map_task_data()
86 	if (unlikely(data->dma_nents == 0)) {  in iser_dma_map_task_data()
88 		return -EINVAL;  in iser_dma_map_task_data()
95 			      enum dma_data_direction dir)  in iser_dma_unmap_task_data()  argument
99 	dev = iser_task->iser_conn->ib_conn.device->ib_device;  in iser_dma_unmap_task_data()
100 	ib_dma_unmap_sg(dev, data->sg, data->size, dir);  in iser_dma_unmap_task_data()
107 	struct scatterlist *sg = mem->sg;  in iser_reg_dma()
109 	reg->sge.lkey = device->pd->local_dma_lkey;  in iser_reg_dma()
115 	if (device->pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY)  in iser_reg_dma()
116 		reg->rkey = device->pd->unsafe_global_rkey;  in iser_reg_dma()
118 		reg->rkey = 0;  in iser_reg_dma()
119 	reg->sge.addr = sg_dma_address(&sg[0]);  in iser_reg_dma()
120 	reg->sge.length = sg_dma_len(&sg[0]);  in iser_reg_dma()
123 		 " length=0x%x\n", reg->sge.lkey, reg->rkey,  in iser_reg_dma()
124 		 reg->sge.addr, reg->sge.length);  in iser_reg_dma()
132 	struct iser_mem_reg *reg = &iser_task->rdma_reg[cmd_dir];  in iser_unreg_mem_fastreg()
136 	desc = reg->mem_h;  in iser_unreg_mem_fastreg()
143 	 * SCSI-Response is received. And the signature MR is not checked if  in iser_unreg_mem_fastreg()
148 	if (unlikely(desc->sig_protected)) {  in iser_unreg_mem_fastreg()
149 		desc->sig_protected = false;  in iser_unreg_mem_fastreg()
150 		ib_check_mr_status(desc->rsc.sig_mr, IB_MR_CHECK_SIG_STATUS,  in iser_unreg_mem_fastreg()
153 	iser_reg_desc_put_fr(&iser_task->iser_conn->ib_conn, reg->mem_h);  in iser_unreg_mem_fastreg()
154 	reg->mem_h = NULL;  in iser_unreg_mem_fastreg()
160 	domain->sig_type = IB_SIG_TYPE_T10_DIF;  in iser_set_dif_domain()
161 	domain->sig.dif.pi_interval = scsi_prot_interval(sc);  in iser_set_dif_domain()
162 	domain->sig.dif.ref_tag = t10_pi_ref_tag(scsi_cmd_to_rq(sc));  in iser_set_dif_domain()
167 	domain->sig.dif.apptag_check_mask = 0xffff;  in iser_set_dif_domain()
168 	domain->sig.dif.app_escape = true;  in iser_set_dif_domain()
169 	domain->sig.dif.ref_escape = true;  in iser_set_dif_domain()
170 	if (sc->prot_flags & SCSI_PROT_REF_INCREMENT)  in iser_set_dif_domain()
171 		domain->sig.dif.ref_remap = true;  in iser_set_dif_domain()
180 		sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE;  in iser_set_sig_attrs()
181 		iser_set_dif_domain(sc, &sig_attrs->wire);  in iser_set_sig_attrs()
182 		sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC;  in iser_set_sig_attrs()
186 		sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE;  in iser_set_sig_attrs()
187 		iser_set_dif_domain(sc, &sig_attrs->mem);  in iser_set_sig_attrs()
188 		sig_attrs->mem.sig.dif.bg_type = sc->prot_flags & SCSI_PROT_IP_CHECKSUM ?  in iser_set_sig_attrs()
193 		iser_set_dif_domain(sc, &sig_attrs->wire);  in iser_set_sig_attrs()
194 		sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC;  in iser_set_sig_attrs()
195 		iser_set_dif_domain(sc, &sig_attrs->mem);  in iser_set_sig_attrs()
196 		sig_attrs->mem.sig.dif.bg_type = sc->prot_flags & SCSI_PROT_IP_CHECKSUM ?  in iser_set_sig_attrs()
202 		return -EINVAL;  in iser_set_sig_attrs()
212 	if (sc->prot_flags & SCSI_PROT_REF_CHECK)  in iser_set_prot_checks()
214 	if (sc->prot_flags & SCSI_PROT_GUARD_CHECK)  in iser_set_prot_checks()
224 	inv_wr->opcode = IB_WR_LOCAL_INV;  in iser_inv_rkey()
225 	inv_wr->wr_cqe = cqe;  in iser_inv_rkey()
226 	inv_wr->ex.invalidate_rkey = mr->rkey;  in iser_inv_rkey()
227 	inv_wr->send_flags = 0;  in iser_inv_rkey()
228 	inv_wr->num_sge = 0;  in iser_inv_rkey()
229 	inv_wr->next = next_wr;  in iser_inv_rkey()
239 	struct iser_tx_desc *tx_desc = &iser_task->desc;  in iser_reg_sig_mr()
240 	struct ib_cqe *cqe = &iser_task->iser_conn->ib_conn.reg_cqe;  in iser_reg_sig_mr()
241 	struct ib_mr *mr = rsc->sig_mr;  in iser_reg_sig_mr()
242 	struct ib_sig_attrs *sig_attrs = mr->sig_attrs;  in iser_reg_sig_mr()
243 	struct ib_reg_wr *wr = &tx_desc->reg_wr;  in iser_reg_sig_mr()
247 	ret = iser_set_sig_attrs(iser_task->sc, sig_attrs);  in iser_reg_sig_mr()
251 	iser_set_prot_checks(iser_task->sc, &sig_attrs->check_mask);  in iser_reg_sig_mr()
253 	if (rsc->mr_valid)  in iser_reg_sig_mr()
254 		iser_inv_rkey(&tx_desc->inv_wr, mr, cqe, &wr->wr);  in iser_reg_sig_mr()
256 	ib_update_fast_reg_key(mr, ib_inc_rkey(mr->rkey));  in iser_reg_sig_mr()
258 	ret = ib_map_mr_sg_pi(mr, mem->sg, mem->dma_nents, NULL,  in iser_reg_sig_mr()
259 			      sig_mem->sg, sig_mem->dma_nents, NULL, SZ_4K);  in iser_reg_sig_mr()
262 			 mem->dma_nents + sig_mem->dma_nents);  in iser_reg_sig_mr()
267 	wr->wr.next = &tx_desc->send_wr;  in iser_reg_sig_mr()
268 	wr->wr.opcode = IB_WR_REG_MR_INTEGRITY;  in iser_reg_sig_mr()
269 	wr->wr.wr_cqe = cqe;  in iser_reg_sig_mr()
270 	wr->wr.num_sge = 0;  in iser_reg_sig_mr()
271 	wr->wr.send_flags = 0;  in iser_reg_sig_mr()
272 	wr->mr = mr;  in iser_reg_sig_mr()
273 	wr->key = mr->rkey;  in iser_reg_sig_mr()
274 	wr->access = IB_ACCESS_LOCAL_WRITE |  in iser_reg_sig_mr()
277 	rsc->mr_valid = 1;  in iser_reg_sig_mr()
279 	sig_reg->sge.lkey = mr->lkey;  in iser_reg_sig_mr()
280 	sig_reg->rkey = mr->rkey;  in iser_reg_sig_mr()
281 	sig_reg->sge.addr = mr->iova;  in iser_reg_sig_mr()
282 	sig_reg->sge.length = mr->length;  in iser_reg_sig_mr()
285 		 sig_reg->sge.lkey, sig_reg->rkey, sig_reg->sge.addr,  in iser_reg_sig_mr()
286 		 sig_reg->sge.length);  in iser_reg_sig_mr()
296 	struct iser_tx_desc *tx_desc = &iser_task->desc;  in iser_fast_reg_mr()
297 	struct ib_cqe *cqe = &iser_task->iser_conn->ib_conn.reg_cqe;  in iser_fast_reg_mr()
298 	struct ib_mr *mr = rsc->mr;  in iser_fast_reg_mr()
299 	struct ib_reg_wr *wr = &tx_desc->reg_wr;  in iser_fast_reg_mr()
302 	if (rsc->mr_valid)  in iser_fast_reg_mr()
303 		iser_inv_rkey(&tx_desc->inv_wr, mr, cqe, &wr->wr);  in iser_fast_reg_mr()
305 	ib_update_fast_reg_key(mr, ib_inc_rkey(mr->rkey));  in iser_fast_reg_mr()
307 	n = ib_map_mr_sg(mr, mem->sg, mem->dma_nents, NULL, SZ_4K);  in iser_fast_reg_mr()
308 	if (unlikely(n != mem->dma_nents)) {  in iser_fast_reg_mr()
310 			 n, mem->dma_nents);  in iser_fast_reg_mr()
311 		return n < 0 ? n : -EINVAL;  in iser_fast_reg_mr()
314 	wr->wr.next = &tx_desc->send_wr;  in iser_fast_reg_mr()
315 	wr->wr.opcode = IB_WR_REG_MR;  in iser_fast_reg_mr()
316 	wr->wr.wr_cqe = cqe;  in iser_fast_reg_mr()
317 	wr->wr.send_flags = 0;  in iser_fast_reg_mr()
318 	wr->wr.num_sge = 0;  in iser_fast_reg_mr()
319 	wr->mr = mr;  in iser_fast_reg_mr()
320 	wr->key = mr->rkey;  in iser_fast_reg_mr()
321 	wr->access = IB_ACCESS_LOCAL_WRITE  |  in iser_fast_reg_mr()
325 	rsc->mr_valid = 1;  in iser_fast_reg_mr()
327 	reg->sge.lkey = mr->lkey;  in iser_fast_reg_mr()
328 	reg->rkey = mr->rkey;  in iser_fast_reg_mr()
329 	reg->sge.addr = mr->iova;  in iser_fast_reg_mr()
330 	reg->sge.length = mr->length;  in iser_fast_reg_mr()
333 		 reg->sge.lkey, reg->rkey, reg->sge.addr, reg->sge.length);  in iser_fast_reg_mr()
345 	struct iser_device *device = task->iser_conn->ib_conn.device;  in iser_reg_data_sg()
350 	return iser_fast_reg_mr(task, mem, &desc->rsc, reg);  in iser_reg_data_sg()
354 			 enum iser_data_dir dir,  in iser_reg_mem_fastreg()  argument
357 	struct ib_conn *ib_conn = &task->iser_conn->ib_conn;  in iser_reg_mem_fastreg()
358 	struct iser_data_buf *mem = &task->data[dir];  in iser_reg_mem_fastreg()
359 	struct iser_mem_reg *reg = &task->rdma_reg[dir];  in iser_reg_mem_fastreg()
364 	use_dma_key = mem->dma_nents == 1 && (all_imm || !iser_always_reg) &&  in iser_reg_mem_fastreg()
365 		      scsi_get_prot_op(task->sc) == SCSI_PROT_NORMAL;  in iser_reg_mem_fastreg()
369 		reg->mem_h = desc;  in iser_reg_mem_fastreg()
372 	if (scsi_get_prot_op(task->sc) == SCSI_PROT_NORMAL) {  in iser_reg_mem_fastreg()
377 		err = iser_reg_sig_mr(task, mem, &task->prot[dir],  in iser_reg_mem_fastreg()
378 				      &desc->rsc, reg);  in iser_reg_mem_fastreg()
382 		desc->sig_protected = true;  in iser_reg_mem_fastreg()