1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Driver for IBM Power 842 compression accelerator
4 *
5 * Copyright (C) IBM Corporation, 2012
6 *
7 * Authors: Robert Jennings <rcj@linux.vnet.ibm.com>
8 * Seth Jennings <sjenning@linux.vnet.ibm.com>
9 */
10
11 #include <asm/vio.h>
12 #include <asm/hvcall.h>
13 #include <asm/vas.h>
14
15 #include "nx-842.h"
16 #include "nx_csbcpb.h" /* struct nx_csbcpb */
17
18 MODULE_LICENSE("GPL");
19 MODULE_AUTHOR("Robert Jennings <rcj@linux.vnet.ibm.com>");
20 MODULE_DESCRIPTION("842 H/W Compression driver for IBM Power processors");
21 MODULE_ALIAS_CRYPTO("842");
22 MODULE_ALIAS_CRYPTO("842-nx");
23
24 /*
25 * Coprocessor type specific capabilities from the hypervisor.
26 */
27 struct hv_nx_cop_caps {
28 __be64 descriptor;
29 __be64 req_max_processed_len; /* Max bytes in one GZIP request */
30 __be64 min_compress_len; /* Min compression size in bytes */
31 __be64 min_decompress_len; /* Min decompression size in bytes */
32 } __packed __aligned(0x1000);
33
34 /*
35 * Coprocessor type specific capabilities.
36 */
37 struct nx_cop_caps {
38 u64 descriptor;
39 u64 req_max_processed_len; /* Max bytes in one GZIP request */
40 u64 min_compress_len; /* Min compression in bytes */
41 u64 min_decompress_len; /* Min decompression in bytes */
42 };
43
44 static u64 caps_feat;
45 static struct nx_cop_caps nx_cop_caps;
46
47 static struct nx842_constraints nx842_pseries_constraints = {
48 .alignment = DDE_BUFFER_ALIGN,
49 .multiple = DDE_BUFFER_LAST_MULT,
50 .minimum = DDE_BUFFER_LAST_MULT,
51 .maximum = PAGE_SIZE, /* dynamic, max_sync_size */
52 };
53
check_constraints(unsigned long buf,unsigned int * len,bool in)54 static int check_constraints(unsigned long buf, unsigned int *len, bool in)
55 {
56 if (!IS_ALIGNED(buf, nx842_pseries_constraints.alignment)) {
57 pr_debug("%s buffer 0x%lx not aligned to 0x%x\n",
58 in ? "input" : "output", buf,
59 nx842_pseries_constraints.alignment);
60 return -EINVAL;
61 }
62 if (*len % nx842_pseries_constraints.multiple) {
63 pr_debug("%s buffer len 0x%x not multiple of 0x%x\n",
64 in ? "input" : "output", *len,
65 nx842_pseries_constraints.multiple);
66 if (in)
67 return -EINVAL;
68 *len = round_down(*len, nx842_pseries_constraints.multiple);
69 }
70 if (*len < nx842_pseries_constraints.minimum) {
71 pr_debug("%s buffer len 0x%x under minimum 0x%x\n",
72 in ? "input" : "output", *len,
73 nx842_pseries_constraints.minimum);
74 return -EINVAL;
75 }
76 if (*len > nx842_pseries_constraints.maximum) {
77 pr_debug("%s buffer len 0x%x over maximum 0x%x\n",
78 in ? "input" : "output", *len,
79 nx842_pseries_constraints.maximum);
80 if (in)
81 return -EINVAL;
82 *len = nx842_pseries_constraints.maximum;
83 }
84 return 0;
85 }
86
87 /* I assume we need to align the CSB? */
88 #define WORKMEM_ALIGN (256)
89
90 struct nx842_workmem {
91 /* scatterlist */
92 char slin[4096];
93 char slout[4096];
94 /* coprocessor status/parameter block */
95 struct nx_csbcpb csbcpb;
96
97 char padding[WORKMEM_ALIGN];
98 } __aligned(WORKMEM_ALIGN);
99
100 /* Macros for fields within nx_csbcpb */
101 /* Check the valid bit within the csbcpb valid field */
102 #define NX842_CSBCBP_VALID_CHK(x) (x & BIT_MASK(7))
103
104 /* CE macros operate on the completion_extension field bits in the csbcpb.
105 * CE0 0=full completion, 1=partial completion
106 * CE1 0=CE0 indicates completion, 1=termination (output may be modified)
107 * CE2 0=processed_bytes is source bytes, 1=processed_bytes is target bytes */
108 #define NX842_CSBCPB_CE0(x) (x & BIT_MASK(7))
109 #define NX842_CSBCPB_CE1(x) (x & BIT_MASK(6))
110 #define NX842_CSBCPB_CE2(x) (x & BIT_MASK(5))
111
112 /* The NX unit accepts data only on 4K page boundaries */
113 #define NX842_HW_PAGE_SIZE (4096)
114 #define NX842_HW_PAGE_MASK (~(NX842_HW_PAGE_SIZE-1))
115
116 struct ibm_nx842_counters {
117 atomic64_t comp_complete;
118 atomic64_t comp_failed;
119 atomic64_t decomp_complete;
120 atomic64_t decomp_failed;
121 atomic64_t swdecomp;
122 atomic64_t comp_times[32];
123 atomic64_t decomp_times[32];
124 };
125
126 static struct nx842_devdata {
127 struct vio_dev *vdev;
128 struct device *dev;
129 struct ibm_nx842_counters *counters;
130 unsigned int max_sg_len;
131 unsigned int max_sync_size;
132 unsigned int max_sync_sg;
133 } __rcu *devdata;
134 static DEFINE_SPINLOCK(devdata_mutex);
135
136 #define NX842_COUNTER_INC(_x) \
137 static inline void nx842_inc_##_x( \
138 const struct nx842_devdata *dev) { \
139 if (dev) \
140 atomic64_inc(&dev->counters->_x); \
141 }
142 NX842_COUNTER_INC(comp_complete);
143 NX842_COUNTER_INC(comp_failed);
144 NX842_COUNTER_INC(decomp_complete);
145 NX842_COUNTER_INC(decomp_failed);
146 NX842_COUNTER_INC(swdecomp);
147
148 #define NX842_HIST_SLOTS 16
149
ibm_nx842_incr_hist(atomic64_t * times,unsigned int time)150 static void ibm_nx842_incr_hist(atomic64_t *times, unsigned int time)
151 {
152 int bucket = fls(time);
153
154 if (bucket)
155 bucket = min((NX842_HIST_SLOTS - 1), bucket - 1);
156
157 atomic64_inc(×[bucket]);
158 }
159
160 /* NX unit operation flags */
161 #define NX842_OP_COMPRESS 0x0
162 #define NX842_OP_CRC 0x1
163 #define NX842_OP_DECOMPRESS 0x2
164 #define NX842_OP_COMPRESS_CRC (NX842_OP_COMPRESS | NX842_OP_CRC)
165 #define NX842_OP_DECOMPRESS_CRC (NX842_OP_DECOMPRESS | NX842_OP_CRC)
166 #define NX842_OP_ASYNC (1<<23)
167 #define NX842_OP_NOTIFY (1<<22)
168 #define NX842_OP_NOTIFY_INT(x) ((x & 0xff)<<8)
169
nx842_get_desired_dma(struct vio_dev * viodev)170 static unsigned long nx842_get_desired_dma(struct vio_dev *viodev)
171 {
172 /* No use of DMA mappings within the driver. */
173 return 0;
174 }
175
176 struct nx842_slentry {
177 __be64 ptr; /* Real address (use __pa()) */
178 __be64 len;
179 };
180
181 /* pHyp scatterlist entry */
182 struct nx842_scatterlist {
183 int entry_nr; /* number of slentries */
184 struct nx842_slentry *entries; /* ptr to array of slentries */
185 };
186
187 /* Does not include sizeof(entry_nr) in the size */
nx842_get_scatterlist_size(struct nx842_scatterlist * sl)188 static inline unsigned long nx842_get_scatterlist_size(
189 struct nx842_scatterlist *sl)
190 {
191 return sl->entry_nr * sizeof(struct nx842_slentry);
192 }
193
nx842_build_scatterlist(unsigned long buf,int len,struct nx842_scatterlist * sl)194 static int nx842_build_scatterlist(unsigned long buf, int len,
195 struct nx842_scatterlist *sl)
196 {
197 unsigned long entrylen;
198 struct nx842_slentry *entry;
199
200 sl->entry_nr = 0;
201
202 entry = sl->entries;
203 while (len) {
204 entry->ptr = cpu_to_be64(nx842_get_pa((void *)buf));
205 entrylen = min_t(int, len,
206 LEN_ON_SIZE(buf, NX842_HW_PAGE_SIZE));
207 entry->len = cpu_to_be64(entrylen);
208
209 len -= entrylen;
210 buf += entrylen;
211
212 sl->entry_nr++;
213 entry++;
214 }
215
216 return 0;
217 }
218
nx842_validate_result(struct device * dev,struct cop_status_block * csb)219 static int nx842_validate_result(struct device *dev,
220 struct cop_status_block *csb)
221 {
222 /* The csb must be valid after returning from vio_h_cop_sync */
223 if (!NX842_CSBCBP_VALID_CHK(csb->valid)) {
224 dev_err(dev, "%s: cspcbp not valid upon completion.\n",
225 __func__);
226 dev_dbg(dev, "valid:0x%02x cs:0x%02x cc:0x%02x ce:0x%02x\n",
227 csb->valid,
228 csb->crb_seq_number,
229 csb->completion_code,
230 csb->completion_extension);
231 dev_dbg(dev, "processed_bytes:%d address:0x%016lx\n",
232 be32_to_cpu(csb->processed_byte_count),
233 (unsigned long)be64_to_cpu(csb->address));
234 return -EIO;
235 }
236
237 /* Check return values from the hardware in the CSB */
238 switch (csb->completion_code) {
239 case 0: /* Completed without error */
240 break;
241 case 64: /* Compression ok, but output larger than input */
242 dev_dbg(dev, "%s: output size larger than input size\n",
243 __func__);
244 break;
245 case 13: /* Output buffer too small */
246 dev_dbg(dev, "%s: Out of space in output buffer\n",
247 __func__);
248 return -ENOSPC;
249 case 65: /* Calculated CRC doesn't match the passed value */
250 dev_dbg(dev, "%s: CRC mismatch for decompression\n",
251 __func__);
252 return -EINVAL;
253 case 66: /* Input data contains an illegal template field */
254 case 67: /* Template indicates data past the end of the input stream */
255 dev_dbg(dev, "%s: Bad data for decompression (code:%d)\n",
256 __func__, csb->completion_code);
257 return -EINVAL;
258 default:
259 dev_dbg(dev, "%s: Unspecified error (code:%d)\n",
260 __func__, csb->completion_code);
261 return -EIO;
262 }
263
264 /* Hardware sanity check */
265 if (!NX842_CSBCPB_CE2(csb->completion_extension)) {
266 dev_err(dev, "%s: No error returned by hardware, but "
267 "data returned is unusable, contact support.\n"
268 "(Additional info: csbcbp->processed bytes "
269 "does not specify processed bytes for the "
270 "target buffer.)\n", __func__);
271 return -EIO;
272 }
273
274 return 0;
275 }
276
277 /**
278 * nx842_pseries_compress - Compress data using the 842 algorithm
279 *
280 * Compression provide by the NX842 coprocessor on IBM Power systems.
281 * The input buffer is compressed and the result is stored in the
282 * provided output buffer.
283 *
284 * Upon return from this function @outlen contains the length of the
285 * compressed data. If there is an error then @outlen will be 0 and an
286 * error will be specified by the return code from this function.
287 *
288 * @in: Pointer to input buffer
289 * @inlen: Length of input buffer
290 * @out: Pointer to output buffer
291 * @outlen: Length of output buffer
292 * @wmem: ptr to buffer for working memory, size determined by
293 * nx842_pseries_driver.workmem_size
294 *
295 * Returns:
296 * 0 Success, output of length @outlen stored in the buffer at @out
297 * -ENOMEM Unable to allocate internal buffers
298 * -ENOSPC Output buffer is to small
299 * -EIO Internal error
300 * -ENODEV Hardware unavailable
301 */
nx842_pseries_compress(const unsigned char * in,unsigned int inlen,unsigned char * out,unsigned int * outlen,void * wmem)302 static int nx842_pseries_compress(const unsigned char *in, unsigned int inlen,
303 unsigned char *out, unsigned int *outlen,
304 void *wmem)
305 {
306 struct nx842_devdata *local_devdata;
307 struct device *dev = NULL;
308 struct nx842_workmem *workmem;
309 struct nx842_scatterlist slin, slout;
310 struct nx_csbcpb *csbcpb;
311 int ret = 0;
312 unsigned long inbuf, outbuf;
313 struct vio_pfo_op op = {
314 .done = NULL,
315 .handle = 0,
316 .timeout = 0,
317 };
318 unsigned long start = get_tb();
319
320 inbuf = (unsigned long)in;
321 if (check_constraints(inbuf, &inlen, true))
322 return -EINVAL;
323
324 outbuf = (unsigned long)out;
325 if (check_constraints(outbuf, outlen, false))
326 return -EINVAL;
327
328 rcu_read_lock();
329 local_devdata = rcu_dereference(devdata);
330 if (!local_devdata || !local_devdata->dev) {
331 rcu_read_unlock();
332 return -ENODEV;
333 }
334 dev = local_devdata->dev;
335
336 /* Init scatterlist */
337 workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN);
338 slin.entries = (struct nx842_slentry *)workmem->slin;
339 slout.entries = (struct nx842_slentry *)workmem->slout;
340
341 /* Init operation */
342 op.flags = NX842_OP_COMPRESS_CRC;
343 csbcpb = &workmem->csbcpb;
344 memset(csbcpb, 0, sizeof(*csbcpb));
345 op.csbcpb = nx842_get_pa(csbcpb);
346
347 if ((inbuf & NX842_HW_PAGE_MASK) ==
348 ((inbuf + inlen - 1) & NX842_HW_PAGE_MASK)) {
349 /* Create direct DDE */
350 op.in = nx842_get_pa((void *)inbuf);
351 op.inlen = inlen;
352 } else {
353 /* Create indirect DDE (scatterlist) */
354 nx842_build_scatterlist(inbuf, inlen, &slin);
355 op.in = nx842_get_pa(slin.entries);
356 op.inlen = -nx842_get_scatterlist_size(&slin);
357 }
358
359 if ((outbuf & NX842_HW_PAGE_MASK) ==
360 ((outbuf + *outlen - 1) & NX842_HW_PAGE_MASK)) {
361 /* Create direct DDE */
362 op.out = nx842_get_pa((void *)outbuf);
363 op.outlen = *outlen;
364 } else {
365 /* Create indirect DDE (scatterlist) */
366 nx842_build_scatterlist(outbuf, *outlen, &slout);
367 op.out = nx842_get_pa(slout.entries);
368 op.outlen = -nx842_get_scatterlist_size(&slout);
369 }
370
371 dev_dbg(dev, "%s: op.in %lx op.inlen %ld op.out %lx op.outlen %ld\n",
372 __func__, (unsigned long)op.in, (long)op.inlen,
373 (unsigned long)op.out, (long)op.outlen);
374
375 /* Send request to pHyp */
376 ret = vio_h_cop_sync(local_devdata->vdev, &op);
377
378 /* Check for pHyp error */
379 if (ret) {
380 dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",
381 __func__, ret, op.hcall_err);
382 ret = -EIO;
383 goto unlock;
384 }
385
386 /* Check for hardware error */
387 ret = nx842_validate_result(dev, &csbcpb->csb);
388 if (ret)
389 goto unlock;
390
391 *outlen = be32_to_cpu(csbcpb->csb.processed_byte_count);
392 dev_dbg(dev, "%s: processed_bytes=%d\n", __func__, *outlen);
393
394 unlock:
395 if (ret)
396 nx842_inc_comp_failed(local_devdata);
397 else {
398 nx842_inc_comp_complete(local_devdata);
399 ibm_nx842_incr_hist(local_devdata->counters->comp_times,
400 (get_tb() - start) / tb_ticks_per_usec);
401 }
402 rcu_read_unlock();
403 return ret;
404 }
405
406 /**
407 * nx842_pseries_decompress - Decompress data using the 842 algorithm
408 *
409 * Decompression provide by the NX842 coprocessor on IBM Power systems.
410 * The input buffer is decompressed and the result is stored in the
411 * provided output buffer. The size allocated to the output buffer is
412 * provided by the caller of this function in @outlen. Upon return from
413 * this function @outlen contains the length of the decompressed data.
414 * If there is an error then @outlen will be 0 and an error will be
415 * specified by the return code from this function.
416 *
417 * @in: Pointer to input buffer
418 * @inlen: Length of input buffer
419 * @out: Pointer to output buffer
420 * @outlen: Length of output buffer
421 * @wmem: ptr to buffer for working memory, size determined by
422 * nx842_pseries_driver.workmem_size
423 *
424 * Returns:
425 * 0 Success, output of length @outlen stored in the buffer at @out
426 * -ENODEV Hardware decompression device is unavailable
427 * -ENOMEM Unable to allocate internal buffers
428 * -ENOSPC Output buffer is to small
429 * -EINVAL Bad input data encountered when attempting decompress
430 * -EIO Internal error
431 */
nx842_pseries_decompress(const unsigned char * in,unsigned int inlen,unsigned char * out,unsigned int * outlen,void * wmem)432 static int nx842_pseries_decompress(const unsigned char *in, unsigned int inlen,
433 unsigned char *out, unsigned int *outlen,
434 void *wmem)
435 {
436 struct nx842_devdata *local_devdata;
437 struct device *dev = NULL;
438 struct nx842_workmem *workmem;
439 struct nx842_scatterlist slin, slout;
440 struct nx_csbcpb *csbcpb;
441 int ret = 0;
442 unsigned long inbuf, outbuf;
443 struct vio_pfo_op op = {
444 .done = NULL,
445 .handle = 0,
446 .timeout = 0,
447 };
448 unsigned long start = get_tb();
449
450 /* Ensure page alignment and size */
451 inbuf = (unsigned long)in;
452 if (check_constraints(inbuf, &inlen, true))
453 return -EINVAL;
454
455 outbuf = (unsigned long)out;
456 if (check_constraints(outbuf, outlen, false))
457 return -EINVAL;
458
459 rcu_read_lock();
460 local_devdata = rcu_dereference(devdata);
461 if (!local_devdata || !local_devdata->dev) {
462 rcu_read_unlock();
463 return -ENODEV;
464 }
465 dev = local_devdata->dev;
466
467 workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN);
468
469 /* Init scatterlist */
470 slin.entries = (struct nx842_slentry *)workmem->slin;
471 slout.entries = (struct nx842_slentry *)workmem->slout;
472
473 /* Init operation */
474 op.flags = NX842_OP_DECOMPRESS_CRC;
475 csbcpb = &workmem->csbcpb;
476 memset(csbcpb, 0, sizeof(*csbcpb));
477 op.csbcpb = nx842_get_pa(csbcpb);
478
479 if ((inbuf & NX842_HW_PAGE_MASK) ==
480 ((inbuf + inlen - 1) & NX842_HW_PAGE_MASK)) {
481 /* Create direct DDE */
482 op.in = nx842_get_pa((void *)inbuf);
483 op.inlen = inlen;
484 } else {
485 /* Create indirect DDE (scatterlist) */
486 nx842_build_scatterlist(inbuf, inlen, &slin);
487 op.in = nx842_get_pa(slin.entries);
488 op.inlen = -nx842_get_scatterlist_size(&slin);
489 }
490
491 if ((outbuf & NX842_HW_PAGE_MASK) ==
492 ((outbuf + *outlen - 1) & NX842_HW_PAGE_MASK)) {
493 /* Create direct DDE */
494 op.out = nx842_get_pa((void *)outbuf);
495 op.outlen = *outlen;
496 } else {
497 /* Create indirect DDE (scatterlist) */
498 nx842_build_scatterlist(outbuf, *outlen, &slout);
499 op.out = nx842_get_pa(slout.entries);
500 op.outlen = -nx842_get_scatterlist_size(&slout);
501 }
502
503 dev_dbg(dev, "%s: op.in %lx op.inlen %ld op.out %lx op.outlen %ld\n",
504 __func__, (unsigned long)op.in, (long)op.inlen,
505 (unsigned long)op.out, (long)op.outlen);
506
507 /* Send request to pHyp */
508 ret = vio_h_cop_sync(local_devdata->vdev, &op);
509
510 /* Check for pHyp error */
511 if (ret) {
512 dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",
513 __func__, ret, op.hcall_err);
514 goto unlock;
515 }
516
517 /* Check for hardware error */
518 ret = nx842_validate_result(dev, &csbcpb->csb);
519 if (ret)
520 goto unlock;
521
522 *outlen = be32_to_cpu(csbcpb->csb.processed_byte_count);
523
524 unlock:
525 if (ret)
526 /* decompress fail */
527 nx842_inc_decomp_failed(local_devdata);
528 else {
529 nx842_inc_decomp_complete(local_devdata);
530 ibm_nx842_incr_hist(local_devdata->counters->decomp_times,
531 (get_tb() - start) / tb_ticks_per_usec);
532 }
533
534 rcu_read_unlock();
535 return ret;
536 }
537
538 /**
539 * nx842_OF_set_defaults -- Set default (disabled) values for devdata
540 *
541 * @devdata: struct nx842_devdata to update
542 *
543 * Returns:
544 * 0 on success
545 * -ENOENT if @devdata ptr is NULL
546 */
nx842_OF_set_defaults(struct nx842_devdata * devdata)547 static int nx842_OF_set_defaults(struct nx842_devdata *devdata)
548 {
549 if (devdata) {
550 devdata->max_sync_size = 0;
551 devdata->max_sync_sg = 0;
552 devdata->max_sg_len = 0;
553 return 0;
554 } else
555 return -ENOENT;
556 }
557
558 /**
559 * nx842_OF_upd_status -- Check the device info from OF status prop
560 *
561 * The status property indicates if the accelerator is enabled. If the
562 * device is in the OF tree it indicates that the hardware is present.
563 * The status field indicates if the device is enabled when the status
564 * is 'okay'. Otherwise the device driver will be disabled.
565 *
566 * @devdata: struct nx842_devdata to use for dev_info
567 * @prop: struct property point containing the maxsyncop for the update
568 *
569 * Returns:
570 * 0 - Device is available
571 * -ENODEV - Device is not available
572 */
nx842_OF_upd_status(struct nx842_devdata * devdata,struct property * prop)573 static int nx842_OF_upd_status(struct nx842_devdata *devdata,
574 struct property *prop)
575 {
576 const char *status = (const char *)prop->value;
577
578 if (!strncmp(status, "okay", (size_t)prop->length))
579 return 0;
580 if (!strncmp(status, "disabled", (size_t)prop->length))
581 return -ENODEV;
582 dev_info(devdata->dev, "%s: unknown status '%s'\n", __func__, status);
583
584 return -EINVAL;
585 }
586
587 /**
588 * nx842_OF_upd_maxsglen -- Update the device info from OF maxsglen prop
589 *
590 * Definition of the 'ibm,max-sg-len' OF property:
591 * This field indicates the maximum byte length of a scatter list
592 * for the platform facility. It is a single cell encoded as with encode-int.
593 *
594 * Example:
595 * # od -x ibm,max-sg-len
596 * 0000000 0000 0ff0
597 *
598 * In this example, the maximum byte length of a scatter list is
599 * 0x0ff0 (4,080).
600 *
601 * @devdata: struct nx842_devdata to update
602 * @prop: struct property point containing the maxsyncop for the update
603 *
604 * Returns:
605 * 0 on success
606 * -EINVAL on failure
607 */
nx842_OF_upd_maxsglen(struct nx842_devdata * devdata,struct property * prop)608 static int nx842_OF_upd_maxsglen(struct nx842_devdata *devdata,
609 struct property *prop) {
610 int ret = 0;
611 const unsigned int maxsglen = of_read_number(prop->value, 1);
612
613 if (prop->length != sizeof(maxsglen)) {
614 dev_err(devdata->dev, "%s: unexpected format for ibm,max-sg-len property\n", __func__);
615 dev_dbg(devdata->dev, "%s: ibm,max-sg-len is %d bytes long, expected %lu bytes\n", __func__,
616 prop->length, sizeof(maxsglen));
617 ret = -EINVAL;
618 } else {
619 devdata->max_sg_len = min_t(unsigned int,
620 maxsglen, NX842_HW_PAGE_SIZE);
621 }
622
623 return ret;
624 }
625
626 /**
627 * nx842_OF_upd_maxsyncop -- Update the device info from OF maxsyncop prop
628 *
629 * Definition of the 'ibm,max-sync-cop' OF property:
630 * Two series of cells. The first series of cells represents the maximums
631 * that can be synchronously compressed. The second series of cells
632 * represents the maximums that can be synchronously decompressed.
633 * 1. The first cell in each series contains the count of the number of
634 * data length, scatter list elements pairs that follow – each being
635 * of the form
636 * a. One cell data byte length
637 * b. One cell total number of scatter list elements
638 *
639 * Example:
640 * # od -x ibm,max-sync-cop
641 * 0000000 0000 0001 0000 1000 0000 01fe 0000 0001
642 * 0000020 0000 1000 0000 01fe
643 *
644 * In this example, compression supports 0x1000 (4,096) data byte length
645 * and 0x1fe (510) total scatter list elements. Decompression supports
646 * 0x1000 (4,096) data byte length and 0x1f3 (510) total scatter list
647 * elements.
648 *
649 * @devdata: struct nx842_devdata to update
650 * @prop: struct property point containing the maxsyncop for the update
651 *
652 * Returns:
653 * 0 on success
654 * -EINVAL on failure
655 */
nx842_OF_upd_maxsyncop(struct nx842_devdata * devdata,struct property * prop)656 static int nx842_OF_upd_maxsyncop(struct nx842_devdata *devdata,
657 struct property *prop) {
658 int ret = 0;
659 unsigned int comp_data_limit, decomp_data_limit;
660 unsigned int comp_sg_limit, decomp_sg_limit;
661 const struct maxsynccop_t {
662 __be32 comp_elements;
663 __be32 comp_data_limit;
664 __be32 comp_sg_limit;
665 __be32 decomp_elements;
666 __be32 decomp_data_limit;
667 __be32 decomp_sg_limit;
668 } *maxsynccop;
669
670 if (prop->length != sizeof(*maxsynccop)) {
671 dev_err(devdata->dev, "%s: unexpected format for ibm,max-sync-cop property\n", __func__);
672 dev_dbg(devdata->dev, "%s: ibm,max-sync-cop is %d bytes long, expected %lu bytes\n", __func__, prop->length,
673 sizeof(*maxsynccop));
674 ret = -EINVAL;
675 goto out;
676 }
677
678 maxsynccop = (const struct maxsynccop_t *)prop->value;
679 comp_data_limit = be32_to_cpu(maxsynccop->comp_data_limit);
680 comp_sg_limit = be32_to_cpu(maxsynccop->comp_sg_limit);
681 decomp_data_limit = be32_to_cpu(maxsynccop->decomp_data_limit);
682 decomp_sg_limit = be32_to_cpu(maxsynccop->decomp_sg_limit);
683
684 /* Use one limit rather than separate limits for compression and
685 * decompression. Set a maximum for this so as not to exceed the
686 * size that the header can support and round the value down to
687 * the hardware page size (4K) */
688 devdata->max_sync_size = min(comp_data_limit, decomp_data_limit);
689
690 devdata->max_sync_size = min_t(unsigned int, devdata->max_sync_size,
691 65536);
692
693 if (devdata->max_sync_size < 4096) {
694 dev_err(devdata->dev, "%s: hardware max data size (%u) is "
695 "less than the driver minimum, unable to use "
696 "the hardware device\n",
697 __func__, devdata->max_sync_size);
698 ret = -EINVAL;
699 goto out;
700 }
701
702 nx842_pseries_constraints.maximum = devdata->max_sync_size;
703
704 devdata->max_sync_sg = min(comp_sg_limit, decomp_sg_limit);
705 if (devdata->max_sync_sg < 1) {
706 dev_err(devdata->dev, "%s: hardware max sg size (%u) is "
707 "less than the driver minimum, unable to use "
708 "the hardware device\n",
709 __func__, devdata->max_sync_sg);
710 ret = -EINVAL;
711 goto out;
712 }
713
714 out:
715 return ret;
716 }
717
718 /**
719 * nx842_OF_upd -- Handle OF properties updates for the device.
720 *
721 * Set all properties from the OF tree. Optionally, a new property
722 * can be provided by the @new_prop pointer to overwrite an existing value.
723 * The device will remain disabled until all values are valid, this function
724 * will return an error for updates unless all values are valid.
725 *
726 * @new_prop: If not NULL, this property is being updated. If NULL, update
727 * all properties from the current values in the OF tree.
728 *
729 * Returns:
730 * 0 - Success
731 * -ENOMEM - Could not allocate memory for new devdata structure
732 * -EINVAL - property value not found, new_prop is not a recognized
733 * property for the device or property value is not valid.
734 * -ENODEV - Device is not available
735 */
nx842_OF_upd(struct property * new_prop)736 static int nx842_OF_upd(struct property *new_prop)
737 {
738 struct nx842_devdata *old_devdata = NULL;
739 struct nx842_devdata *new_devdata = NULL;
740 struct device_node *of_node = NULL;
741 struct property *status = NULL;
742 struct property *maxsglen = NULL;
743 struct property *maxsyncop = NULL;
744 int ret = 0;
745 unsigned long flags;
746
747 new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS);
748 if (!new_devdata)
749 return -ENOMEM;
750
751 spin_lock_irqsave(&devdata_mutex, flags);
752 old_devdata = rcu_dereference_check(devdata,
753 lockdep_is_held(&devdata_mutex));
754 if (old_devdata)
755 of_node = old_devdata->dev->of_node;
756
757 if (!old_devdata || !of_node) {
758 pr_err("%s: device is not available\n", __func__);
759 spin_unlock_irqrestore(&devdata_mutex, flags);
760 kfree(new_devdata);
761 return -ENODEV;
762 }
763
764 memcpy(new_devdata, old_devdata, sizeof(*old_devdata));
765 new_devdata->counters = old_devdata->counters;
766
767 /* Set ptrs for existing properties */
768 status = of_find_property(of_node, "status", NULL);
769 maxsglen = of_find_property(of_node, "ibm,max-sg-len", NULL);
770 maxsyncop = of_find_property(of_node, "ibm,max-sync-cop", NULL);
771 if (!status || !maxsglen || !maxsyncop) {
772 dev_err(old_devdata->dev, "%s: Could not locate device properties\n", __func__);
773 ret = -EINVAL;
774 goto error_out;
775 }
776
777 /*
778 * If this is a property update, there are only certain properties that
779 * we care about. Bail if it isn't in the below list
780 */
781 if (new_prop && (strncmp(new_prop->name, "status", new_prop->length) ||
782 strncmp(new_prop->name, "ibm,max-sg-len", new_prop->length) ||
783 strncmp(new_prop->name, "ibm,max-sync-cop", new_prop->length)))
784 goto out;
785
786 /* Perform property updates */
787 ret = nx842_OF_upd_status(new_devdata, status);
788 if (ret)
789 goto error_out;
790
791 ret = nx842_OF_upd_maxsglen(new_devdata, maxsglen);
792 if (ret)
793 goto error_out;
794
795 ret = nx842_OF_upd_maxsyncop(new_devdata, maxsyncop);
796 if (ret)
797 goto error_out;
798
799 out:
800 dev_info(old_devdata->dev, "%s: max_sync_size new:%u old:%u\n",
801 __func__, new_devdata->max_sync_size,
802 old_devdata->max_sync_size);
803 dev_info(old_devdata->dev, "%s: max_sync_sg new:%u old:%u\n",
804 __func__, new_devdata->max_sync_sg,
805 old_devdata->max_sync_sg);
806 dev_info(old_devdata->dev, "%s: max_sg_len new:%u old:%u\n",
807 __func__, new_devdata->max_sg_len,
808 old_devdata->max_sg_len);
809
810 rcu_assign_pointer(devdata, new_devdata);
811 spin_unlock_irqrestore(&devdata_mutex, flags);
812 synchronize_rcu();
813 dev_set_drvdata(new_devdata->dev, new_devdata);
814 kfree(old_devdata);
815 return 0;
816
817 error_out:
818 if (new_devdata) {
819 dev_info(old_devdata->dev, "%s: device disabled\n", __func__);
820 nx842_OF_set_defaults(new_devdata);
821 rcu_assign_pointer(devdata, new_devdata);
822 spin_unlock_irqrestore(&devdata_mutex, flags);
823 synchronize_rcu();
824 dev_set_drvdata(new_devdata->dev, new_devdata);
825 kfree(old_devdata);
826 } else {
827 dev_err(old_devdata->dev, "%s: could not update driver from hardware\n", __func__);
828 spin_unlock_irqrestore(&devdata_mutex, flags);
829 }
830
831 if (!ret)
832 ret = -EINVAL;
833 return ret;
834 }
835
836 /**
837 * nx842_OF_notifier - Process updates to OF properties for the device
838 *
839 * @np: notifier block
840 * @action: notifier action
841 * @data: struct of_reconfig_data pointer
842 *
843 * Returns:
844 * NOTIFY_OK on success
845 * NOTIFY_BAD encoded with error number on failure, use
846 * notifier_to_errno() to decode this value
847 */
nx842_OF_notifier(struct notifier_block * np,unsigned long action,void * data)848 static int nx842_OF_notifier(struct notifier_block *np, unsigned long action,
849 void *data)
850 {
851 struct of_reconfig_data *upd = data;
852 struct nx842_devdata *local_devdata;
853 struct device_node *node = NULL;
854
855 rcu_read_lock();
856 local_devdata = rcu_dereference(devdata);
857 if (local_devdata)
858 node = local_devdata->dev->of_node;
859
860 if (local_devdata &&
861 action == OF_RECONFIG_UPDATE_PROPERTY &&
862 !strcmp(upd->dn->name, node->name)) {
863 rcu_read_unlock();
864 nx842_OF_upd(upd->prop);
865 } else
866 rcu_read_unlock();
867
868 return NOTIFY_OK;
869 }
870
871 static struct notifier_block nx842_of_nb = {
872 .notifier_call = nx842_OF_notifier,
873 };
874
875 #define nx842_counter_read(_name) \
876 static ssize_t nx842_##_name##_show(struct device *dev, \
877 struct device_attribute *attr, \
878 char *buf) { \
879 struct nx842_devdata *local_devdata; \
880 int p = 0; \
881 rcu_read_lock(); \
882 local_devdata = rcu_dereference(devdata); \
883 if (local_devdata) \
884 p = snprintf(buf, PAGE_SIZE, "%lld\n", \
885 atomic64_read(&local_devdata->counters->_name)); \
886 rcu_read_unlock(); \
887 return p; \
888 }
889
890 #define NX842DEV_COUNTER_ATTR_RO(_name) \
891 nx842_counter_read(_name); \
892 static struct device_attribute dev_attr_##_name = __ATTR(_name, \
893 0444, \
894 nx842_##_name##_show,\
895 NULL);
896
897 NX842DEV_COUNTER_ATTR_RO(comp_complete);
898 NX842DEV_COUNTER_ATTR_RO(comp_failed);
899 NX842DEV_COUNTER_ATTR_RO(decomp_complete);
900 NX842DEV_COUNTER_ATTR_RO(decomp_failed);
901 NX842DEV_COUNTER_ATTR_RO(swdecomp);
902
903 static ssize_t nx842_timehist_show(struct device *,
904 struct device_attribute *, char *);
905
906 static struct device_attribute dev_attr_comp_times = __ATTR(comp_times, 0444,
907 nx842_timehist_show, NULL);
908 static struct device_attribute dev_attr_decomp_times = __ATTR(decomp_times,
909 0444, nx842_timehist_show, NULL);
910
nx842_timehist_show(struct device * dev,struct device_attribute * attr,char * buf)911 static ssize_t nx842_timehist_show(struct device *dev,
912 struct device_attribute *attr, char *buf) {
913 char *p = buf;
914 struct nx842_devdata *local_devdata;
915 atomic64_t *times;
916 int bytes_remain = PAGE_SIZE;
917 int bytes;
918 int i;
919
920 rcu_read_lock();
921 local_devdata = rcu_dereference(devdata);
922 if (!local_devdata) {
923 rcu_read_unlock();
924 return 0;
925 }
926
927 if (attr == &dev_attr_comp_times)
928 times = local_devdata->counters->comp_times;
929 else if (attr == &dev_attr_decomp_times)
930 times = local_devdata->counters->decomp_times;
931 else {
932 rcu_read_unlock();
933 return 0;
934 }
935
936 for (i = 0; i < (NX842_HIST_SLOTS - 2); i++) {
937 bytes = snprintf(p, bytes_remain, "%u-%uus:\t%lld\n",
938 i ? (2<<(i-1)) : 0, (2<<i)-1,
939 atomic64_read(×[i]));
940 bytes_remain -= bytes;
941 p += bytes;
942 }
943 /* The last bucket holds everything over
944 * 2<<(NX842_HIST_SLOTS - 2) us */
945 bytes = snprintf(p, bytes_remain, "%uus - :\t%lld\n",
946 2<<(NX842_HIST_SLOTS - 2),
947 atomic64_read(×[(NX842_HIST_SLOTS - 1)]));
948 p += bytes;
949
950 rcu_read_unlock();
951 return p - buf;
952 }
953
954 static struct attribute *nx842_sysfs_entries[] = {
955 &dev_attr_comp_complete.attr,
956 &dev_attr_comp_failed.attr,
957 &dev_attr_decomp_complete.attr,
958 &dev_attr_decomp_failed.attr,
959 &dev_attr_swdecomp.attr,
960 &dev_attr_comp_times.attr,
961 &dev_attr_decomp_times.attr,
962 NULL,
963 };
964
965 static const struct attribute_group nx842_attribute_group = {
966 .name = NULL, /* put in device directory */
967 .attrs = nx842_sysfs_entries,
968 };
969
970 #define nxcop_caps_read(_name) \
971 static ssize_t nxcop_##_name##_show(struct device *dev, \
972 struct device_attribute *attr, char *buf) \
973 { \
974 return sprintf(buf, "%lld\n", nx_cop_caps._name); \
975 }
976
977 #define NXCT_ATTR_RO(_name) \
978 nxcop_caps_read(_name); \
979 static struct device_attribute dev_attr_##_name = __ATTR(_name, \
980 0444, \
981 nxcop_##_name##_show, \
982 NULL);
983
984 NXCT_ATTR_RO(req_max_processed_len);
985 NXCT_ATTR_RO(min_compress_len);
986 NXCT_ATTR_RO(min_decompress_len);
987
988 static struct attribute *nxcop_caps_sysfs_entries[] = {
989 &dev_attr_req_max_processed_len.attr,
990 &dev_attr_min_compress_len.attr,
991 &dev_attr_min_decompress_len.attr,
992 NULL,
993 };
994
995 static const struct attribute_group nxcop_caps_attr_group = {
996 .name = "nx_gzip_caps",
997 .attrs = nxcop_caps_sysfs_entries,
998 };
999
1000 static struct nx842_driver nx842_pseries_driver = {
1001 .name = KBUILD_MODNAME,
1002 .owner = THIS_MODULE,
1003 .workmem_size = sizeof(struct nx842_workmem),
1004 .constraints = &nx842_pseries_constraints,
1005 .compress = nx842_pseries_compress,
1006 .decompress = nx842_pseries_decompress,
1007 };
1008
nx842_pseries_crypto_init(struct crypto_tfm * tfm)1009 static int nx842_pseries_crypto_init(struct crypto_tfm *tfm)
1010 {
1011 return nx842_crypto_init(tfm, &nx842_pseries_driver);
1012 }
1013
1014 static struct crypto_alg nx842_pseries_alg = {
1015 .cra_name = "842",
1016 .cra_driver_name = "842-nx",
1017 .cra_priority = 300,
1018 .cra_flags = CRYPTO_ALG_TYPE_COMPRESS,
1019 .cra_ctxsize = sizeof(struct nx842_crypto_ctx),
1020 .cra_module = THIS_MODULE,
1021 .cra_init = nx842_pseries_crypto_init,
1022 .cra_exit = nx842_crypto_exit,
1023 .cra_u = { .compress = {
1024 .coa_compress = nx842_crypto_compress,
1025 .coa_decompress = nx842_crypto_decompress } }
1026 };
1027
nx842_probe(struct vio_dev * viodev,const struct vio_device_id * id)1028 static int nx842_probe(struct vio_dev *viodev,
1029 const struct vio_device_id *id)
1030 {
1031 struct nx842_devdata *old_devdata, *new_devdata = NULL;
1032 unsigned long flags;
1033 int ret = 0;
1034
1035 new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS);
1036 if (!new_devdata)
1037 return -ENOMEM;
1038
1039 new_devdata->counters = kzalloc(sizeof(*new_devdata->counters),
1040 GFP_NOFS);
1041 if (!new_devdata->counters) {
1042 kfree(new_devdata);
1043 return -ENOMEM;
1044 }
1045
1046 spin_lock_irqsave(&devdata_mutex, flags);
1047 old_devdata = rcu_dereference_check(devdata,
1048 lockdep_is_held(&devdata_mutex));
1049
1050 if (old_devdata && old_devdata->vdev != NULL) {
1051 dev_err(&viodev->dev, "%s: Attempt to register more than one instance of the hardware\n", __func__);
1052 ret = -1;
1053 goto error_unlock;
1054 }
1055
1056 dev_set_drvdata(&viodev->dev, NULL);
1057
1058 new_devdata->vdev = viodev;
1059 new_devdata->dev = &viodev->dev;
1060 nx842_OF_set_defaults(new_devdata);
1061
1062 rcu_assign_pointer(devdata, new_devdata);
1063 spin_unlock_irqrestore(&devdata_mutex, flags);
1064 synchronize_rcu();
1065 kfree(old_devdata);
1066
1067 of_reconfig_notifier_register(&nx842_of_nb);
1068
1069 ret = nx842_OF_upd(NULL);
1070 if (ret)
1071 goto error;
1072
1073 ret = crypto_register_alg(&nx842_pseries_alg);
1074 if (ret) {
1075 dev_err(&viodev->dev, "could not register comp alg: %d\n", ret);
1076 goto error;
1077 }
1078
1079 rcu_read_lock();
1080 dev_set_drvdata(&viodev->dev, rcu_dereference(devdata));
1081 rcu_read_unlock();
1082
1083 if (sysfs_create_group(&viodev->dev.kobj, &nx842_attribute_group)) {
1084 dev_err(&viodev->dev, "could not create sysfs device attributes\n");
1085 ret = -1;
1086 goto error;
1087 }
1088
1089 if (caps_feat) {
1090 if (sysfs_create_group(&viodev->dev.kobj,
1091 &nxcop_caps_attr_group)) {
1092 dev_err(&viodev->dev,
1093 "Could not create sysfs NX capability entries\n");
1094 ret = -1;
1095 goto error;
1096 }
1097 }
1098
1099 return 0;
1100
1101 error_unlock:
1102 spin_unlock_irqrestore(&devdata_mutex, flags);
1103 if (new_devdata)
1104 kfree(new_devdata->counters);
1105 kfree(new_devdata);
1106 error:
1107 return ret;
1108 }
1109
nx842_remove(struct vio_dev * viodev)1110 static void nx842_remove(struct vio_dev *viodev)
1111 {
1112 struct nx842_devdata *old_devdata;
1113 unsigned long flags;
1114
1115 pr_info("Removing IBM Power 842 compression device\n");
1116 sysfs_remove_group(&viodev->dev.kobj, &nx842_attribute_group);
1117
1118 if (caps_feat)
1119 sysfs_remove_group(&viodev->dev.kobj, &nxcop_caps_attr_group);
1120
1121 crypto_unregister_alg(&nx842_pseries_alg);
1122
1123 spin_lock_irqsave(&devdata_mutex, flags);
1124 old_devdata = rcu_dereference_check(devdata,
1125 lockdep_is_held(&devdata_mutex));
1126 of_reconfig_notifier_unregister(&nx842_of_nb);
1127 RCU_INIT_POINTER(devdata, NULL);
1128 spin_unlock_irqrestore(&devdata_mutex, flags);
1129 synchronize_rcu();
1130 dev_set_drvdata(&viodev->dev, NULL);
1131 if (old_devdata)
1132 kfree(old_devdata->counters);
1133 kfree(old_devdata);
1134 }
1135
1136 /*
1137 * Get NX capabilities from the hypervisor.
1138 * Only NXGZIP capabilities are provided by the hypersvisor right
1139 * now and these values are available to user space with sysfs.
1140 */
nxcop_get_capabilities(void)1141 static void __init nxcop_get_capabilities(void)
1142 {
1143 struct hv_vas_all_caps *hv_caps;
1144 struct hv_nx_cop_caps *hv_nxc;
1145 int rc;
1146
1147 hv_caps = kmalloc(sizeof(*hv_caps), GFP_KERNEL);
1148 if (!hv_caps)
1149 return;
1150 /*
1151 * Get NX overall capabilities with feature type=0
1152 */
1153 rc = h_query_vas_capabilities(H_QUERY_NX_CAPABILITIES, 0,
1154 (u64)virt_to_phys(hv_caps));
1155 if (rc)
1156 goto out;
1157
1158 caps_feat = be64_to_cpu(hv_caps->feat_type);
1159 /*
1160 * NX-GZIP feature available
1161 */
1162 if (caps_feat & VAS_NX_GZIP_FEAT_BIT) {
1163 hv_nxc = kmalloc(sizeof(*hv_nxc), GFP_KERNEL);
1164 if (!hv_nxc)
1165 goto out;
1166 /*
1167 * Get capabilities for NX-GZIP feature
1168 */
1169 rc = h_query_vas_capabilities(H_QUERY_NX_CAPABILITIES,
1170 VAS_NX_GZIP_FEAT,
1171 (u64)virt_to_phys(hv_nxc));
1172 } else {
1173 pr_err("NX-GZIP feature is not available\n");
1174 rc = -EINVAL;
1175 }
1176
1177 if (!rc) {
1178 nx_cop_caps.descriptor = be64_to_cpu(hv_nxc->descriptor);
1179 nx_cop_caps.req_max_processed_len =
1180 be64_to_cpu(hv_nxc->req_max_processed_len);
1181 nx_cop_caps.min_compress_len =
1182 be64_to_cpu(hv_nxc->min_compress_len);
1183 nx_cop_caps.min_decompress_len =
1184 be64_to_cpu(hv_nxc->min_decompress_len);
1185 } else {
1186 caps_feat = 0;
1187 }
1188
1189 kfree(hv_nxc);
1190 out:
1191 kfree(hv_caps);
1192 }
1193
1194 static const struct vio_device_id nx842_vio_driver_ids[] = {
1195 {"ibm,compression-v1", "ibm,compression"},
1196 {"", ""},
1197 };
1198 MODULE_DEVICE_TABLE(vio, nx842_vio_driver_ids);
1199
1200 static struct vio_driver nx842_vio_driver = {
1201 .name = KBUILD_MODNAME,
1202 .probe = nx842_probe,
1203 .remove = nx842_remove,
1204 .get_desired_dma = nx842_get_desired_dma,
1205 .id_table = nx842_vio_driver_ids,
1206 };
1207
nx842_pseries_init(void)1208 static int __init nx842_pseries_init(void)
1209 {
1210 struct nx842_devdata *new_devdata;
1211 struct device_node *np;
1212 int ret;
1213
1214 np = of_find_compatible_node(NULL, NULL, "ibm,compression");
1215 if (!np)
1216 return -ENODEV;
1217 of_node_put(np);
1218
1219 RCU_INIT_POINTER(devdata, NULL);
1220 new_devdata = kzalloc(sizeof(*new_devdata), GFP_KERNEL);
1221 if (!new_devdata)
1222 return -ENOMEM;
1223
1224 RCU_INIT_POINTER(devdata, new_devdata);
1225 /*
1226 * Get NX capabilities from the hypervisor.
1227 */
1228 nxcop_get_capabilities();
1229
1230 ret = vio_register_driver(&nx842_vio_driver);
1231 if (ret) {
1232 pr_err("Could not register VIO driver %d\n", ret);
1233
1234 kfree(new_devdata);
1235 return ret;
1236 }
1237
1238 ret = vas_register_api_pseries(THIS_MODULE, VAS_COP_TYPE_GZIP,
1239 "nx-gzip");
1240
1241 if (ret)
1242 pr_err("NX-GZIP is not supported. Returned=%d\n", ret);
1243
1244 return 0;
1245 }
1246
1247 module_init(nx842_pseries_init);
1248
nx842_pseries_exit(void)1249 static void __exit nx842_pseries_exit(void)
1250 {
1251 struct nx842_devdata *old_devdata;
1252 unsigned long flags;
1253
1254 vas_unregister_api_pseries();
1255
1256 crypto_unregister_alg(&nx842_pseries_alg);
1257
1258 spin_lock_irqsave(&devdata_mutex, flags);
1259 old_devdata = rcu_dereference_check(devdata,
1260 lockdep_is_held(&devdata_mutex));
1261 RCU_INIT_POINTER(devdata, NULL);
1262 spin_unlock_irqrestore(&devdata_mutex, flags);
1263 synchronize_rcu();
1264 if (old_devdata && old_devdata->dev)
1265 dev_set_drvdata(old_devdata->dev, NULL);
1266 kfree(old_devdata);
1267 vio_unregister_driver(&nx842_vio_driver);
1268 }
1269
1270 module_exit(nx842_pseries_exit);
1271
1272