1// SPDX-License-Identifier: GPL-2.0
2/*
3 * raid6_vx$#.c
4 *
5 * $#-way unrolled RAID6 gen/xor functions for s390
6 * based on the vector facility
7 *
8 * Copyright IBM Corp. 2016
9 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
10 *
11 * This file is postprocessed using unroll.awk.
12 */
13
14#include <linux/raid/pq.h>
15#include <asm/fpu/api.h>
16
17asm(".include \"asm/vx-insn.h\"\n");
18
19#define NSIZE 16
20
21static inline void LOAD_CONST(void)
22{
23	asm volatile("VREPIB %v24,7");
24	asm volatile("VREPIB %v25,0x1d");
25}
26
27/*
28 * The SHLBYTE() operation shifts each of the 16 bytes in
29 * vector register y left by 1 bit and stores the result in
30 * vector register x.
31 */
32static inline void SHLBYTE(int x, int y)
33{
34	asm volatile ("VAB %0,%1,%1" : : "i" (x), "i" (y));
35}
36
37/*
38 * For each of the 16 bytes in the vector register y the MASK()
39 * operation returns 0xFF if the high bit of the byte is 1,
40 * or 0x00 if the high bit is 0. The result is stored in vector
41 * register x.
42 */
43static inline void MASK(int x, int y)
44{
45	asm volatile ("VESRAVB	%0,%1,24" : : "i" (x), "i" (y));
46}
47
48static inline void AND(int x, int y, int z)
49{
50	asm volatile ("VN %0,%1,%2" : : "i" (x), "i" (y), "i" (z));
51}
52
53static inline void XOR(int x, int y, int z)
54{
55	asm volatile ("VX %0,%1,%2" : : "i" (x), "i" (y), "i" (z));
56}
57
58static inline void LOAD_DATA(int x, u8 *ptr)
59{
60	typedef struct { u8 _[16 * $#]; } addrtype;
61	register addrtype *__ptr asm("1") = (addrtype *) ptr;
62
63	asm volatile ("VLM %2,%3,0,%1"
64		      : : "m" (*__ptr), "a" (__ptr), "i" (x),
65			  "i" (x + $# - 1));
66}
67
68static inline void STORE_DATA(int x, u8 *ptr)
69{
70	typedef struct { u8 _[16 * $#]; } addrtype;
71	register addrtype *__ptr asm("1") = (addrtype *) ptr;
72
73	asm volatile ("VSTM %2,%3,0,1"
74		      : "=m" (*__ptr) : "a" (__ptr), "i" (x),
75			"i" (x + $# - 1));
76}
77
78static inline void COPY_VEC(int x, int y)
79{
80	asm volatile ("VLR %0,%1" : : "i" (x), "i" (y));
81}
82
83static void raid6_s390vx$#_gen_syndrome(int disks, size_t bytes, void **ptrs)
84{
85	struct kernel_fpu vxstate;
86	u8 **dptr, *p, *q;
87	int d, z, z0;
88
89	kernel_fpu_begin(&vxstate, KERNEL_VXR);
90	LOAD_CONST();
91
92	dptr = (u8 **) ptrs;
93	z0 = disks - 3;		/* Highest data disk */
94	p = dptr[z0 + 1];	/* XOR parity */
95	q = dptr[z0 + 2];	/* RS syndrome */
96
97	for (d = 0; d < bytes; d += $#*NSIZE) {
98		LOAD_DATA(0,&dptr[z0][d]);
99		COPY_VEC(8+$$,0+$$);
100		for (z = z0 - 1; z >= 0; z--) {
101			MASK(16+$$,8+$$);
102			AND(16+$$,16+$$,25);
103			SHLBYTE(8+$$,8+$$);
104			XOR(8+$$,8+$$,16+$$);
105			LOAD_DATA(16,&dptr[z][d]);
106			XOR(0+$$,0+$$,16+$$);
107			XOR(8+$$,8+$$,16+$$);
108		}
109		STORE_DATA(0,&p[d]);
110		STORE_DATA(8,&q[d]);
111	}
112	kernel_fpu_end(&vxstate, KERNEL_VXR);
113}
114
115static void raid6_s390vx$#_xor_syndrome(int disks, int start, int stop,
116					size_t bytes, void **ptrs)
117{
118	struct kernel_fpu vxstate;
119	u8 **dptr, *p, *q;
120	int d, z, z0;
121
122	dptr = (u8 **) ptrs;
123	z0 = stop;		/* P/Q right side optimization */
124	p = dptr[disks - 2];	/* XOR parity */
125	q = dptr[disks - 1];	/* RS syndrome */
126
127	kernel_fpu_begin(&vxstate, KERNEL_VXR);
128	LOAD_CONST();
129
130	for (d = 0; d < bytes; d += $#*NSIZE) {
131		/* P/Q data pages */
132		LOAD_DATA(0,&dptr[z0][d]);
133		COPY_VEC(8+$$,0+$$);
134		for (z = z0 - 1; z >= start; z--) {
135			MASK(16+$$,8+$$);
136			AND(16+$$,16+$$,25);
137			SHLBYTE(8+$$,8+$$);
138			XOR(8+$$,8+$$,16+$$);
139			LOAD_DATA(16,&dptr[z][d]);
140			XOR(0+$$,0+$$,16+$$);
141			XOR(8+$$,8+$$,16+$$);
142		}
143		/* P/Q left side optimization */
144		for (z = start - 1; z >= 0; z--) {
145			MASK(16+$$,8+$$);
146			AND(16+$$,16+$$,25);
147			SHLBYTE(8+$$,8+$$);
148			XOR(8+$$,8+$$,16+$$);
149		}
150		LOAD_DATA(16,&p[d]);
151		XOR(16+$$,16+$$,0+$$);
152		STORE_DATA(16,&p[d]);
153		LOAD_DATA(16,&q[d]);
154		XOR(16+$$,16+$$,8+$$);
155		STORE_DATA(16,&q[d]);
156	}
157	kernel_fpu_end(&vxstate, KERNEL_VXR);
158}
159
160static int raid6_s390vx$#_valid(void)
161{
162	return MACHINE_HAS_VX;
163}
164
165const struct raid6_calls raid6_s390vx$# = {
166	raid6_s390vx$#_gen_syndrome,
167	raid6_s390vx$#_xor_syndrome,
168	raid6_s390vx$#_valid,
169	"vx128x$#",
170	1
171};
172