1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3  * Copyright (C) 2018 HUAWEI, Inc.
4  *             https://www.huawei.com/
5  */
6 #ifndef __EROFS_FS_ZDATA_H
7 #define __EROFS_FS_ZDATA_H
8 
9 #include "internal.h"
10 #include "tagptr.h"
11 
12 #define Z_EROFS_PCLUSTER_MAX_PAGES	(Z_EROFS_PCLUSTER_MAX_SIZE / PAGE_SIZE)
13 #define Z_EROFS_INLINE_BVECS		2
14 
15 /*
16  * let's leave a type here in case of introducing
17  * another tagged pointer later.
18  */
19 typedef void *z_erofs_next_pcluster_t;
20 
21 struct z_erofs_bvec {
22 	struct page *page;
23 	int offset;
24 	unsigned int end;
25 };
26 
27 #define __Z_EROFS_BVSET(name, total) \
28 struct name { \
29 	/* point to the next page which contains the following bvecs */ \
30 	struct page *nextpage; \
31 	struct z_erofs_bvec bvec[total]; \
32 }
33 __Z_EROFS_BVSET(z_erofs_bvset,);
34 __Z_EROFS_BVSET(z_erofs_bvset_inline, Z_EROFS_INLINE_BVECS);
35 
36 /*
37  * Structure fields follow one of the following exclusion rules.
38  *
39  * I: Modifiable by initialization/destruction paths and read-only
40  *    for everyone else;
41  *
42  * L: Field should be protected by the pcluster lock;
43  *
44  * A: Field should be accessed / updated in atomic for parallelized code.
45  */
46 struct z_erofs_pcluster {
47 	struct erofs_workgroup obj;
48 	struct mutex lock;
49 
50 	/* A: point to next chained pcluster or TAILs */
51 	z_erofs_next_pcluster_t next;
52 
53 	/* L: the maximum decompression size of this round */
54 	unsigned int length;
55 
56 	/* L: total number of bvecs */
57 	unsigned int vcnt;
58 
59 	/* I: page offset of start position of decompression */
60 	unsigned short pageofs_out;
61 
62 	/* I: page offset of inline compressed data */
63 	unsigned short pageofs_in;
64 
65 	union {
66 		/* L: inline a certain number of bvec for bootstrap */
67 		struct z_erofs_bvset_inline bvset;
68 
69 		/* I: can be used to free the pcluster by RCU. */
70 		struct rcu_head rcu;
71 	};
72 
73 	union {
74 		/* I: physical cluster size in pages */
75 		unsigned short pclusterpages;
76 
77 		/* I: tailpacking inline compressed size */
78 		unsigned short tailpacking_size;
79 	};
80 
81 	/* I: compression algorithm format */
82 	unsigned char algorithmformat;
83 
84 	/* L: whether partial decompression or not */
85 	bool partial;
86 
87 	/* L: indicate several pageofs_outs or not */
88 	bool multibases;
89 
90 	/* A: compressed bvecs (can be cached or inplaced pages) */
91 	struct z_erofs_bvec compressed_bvecs[];
92 };
93 
94 /* let's avoid the valid 32-bit kernel addresses */
95 
96 /* the chained workgroup has't submitted io (still open) */
97 #define Z_EROFS_PCLUSTER_TAIL           ((void *)0x5F0ECAFE)
98 /* the chained workgroup has already submitted io */
99 #define Z_EROFS_PCLUSTER_TAIL_CLOSED    ((void *)0x5F0EDEAD)
100 
101 #define Z_EROFS_PCLUSTER_NIL            (NULL)
102 
103 struct z_erofs_decompressqueue {
104 	struct super_block *sb;
105 	atomic_t pending_bios;
106 	z_erofs_next_pcluster_t head;
107 
108 	union {
109 		struct completion done;
110 		struct work_struct work;
111 	} u;
112 
113 	bool eio;
114 };
115 
z_erofs_is_inline_pcluster(struct z_erofs_pcluster * pcl)116 static inline bool z_erofs_is_inline_pcluster(struct z_erofs_pcluster *pcl)
117 {
118 	return !pcl->obj.index;
119 }
120 
z_erofs_pclusterpages(struct z_erofs_pcluster * pcl)121 static inline unsigned int z_erofs_pclusterpages(struct z_erofs_pcluster *pcl)
122 {
123 	if (z_erofs_is_inline_pcluster(pcl))
124 		return 1;
125 	return pcl->pclusterpages;
126 }
127 
128 /*
129  * bit 30: I/O error occurred on this page
130  * bit 0 - 29: remaining parts to complete this page
131  */
132 #define Z_EROFS_PAGE_EIO			(1 << 30)
133 
z_erofs_onlinepage_init(struct page * page)134 static inline void z_erofs_onlinepage_init(struct page *page)
135 {
136 	union {
137 		atomic_t o;
138 		unsigned long v;
139 	} u = { .o = ATOMIC_INIT(1) };
140 
141 	set_page_private(page, u.v);
142 	smp_wmb();
143 	SetPagePrivate(page);
144 }
145 
z_erofs_onlinepage_split(struct page * page)146 static inline void z_erofs_onlinepage_split(struct page *page)
147 {
148 	atomic_inc((atomic_t *)&page->private);
149 }
150 
z_erofs_page_mark_eio(struct page * page)151 static inline void z_erofs_page_mark_eio(struct page *page)
152 {
153 	int orig;
154 
155 	do {
156 		orig = atomic_read((atomic_t *)&page->private);
157 	} while (atomic_cmpxchg((atomic_t *)&page->private, orig,
158 				orig | Z_EROFS_PAGE_EIO) != orig);
159 }
160 
z_erofs_onlinepage_endio(struct page * page)161 static inline void z_erofs_onlinepage_endio(struct page *page)
162 {
163 	unsigned int v;
164 
165 	DBG_BUGON(!PagePrivate(page));
166 	v = atomic_dec_return((atomic_t *)&page->private);
167 	if (!(v & ~Z_EROFS_PAGE_EIO)) {
168 		set_page_private(page, 0);
169 		ClearPagePrivate(page);
170 		if (!(v & Z_EROFS_PAGE_EIO))
171 			SetPageUptodate(page);
172 		unlock_page(page);
173 	}
174 }
175 
176 #define Z_EROFS_ONSTACK_PAGES		32
177 
178 #endif
179