1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _FS_CEPH_OSD_CLIENT_H
3 #define _FS_CEPH_OSD_CLIENT_H
4 
5 #include <linux/bitrev.h>
6 #include <linux/completion.h>
7 #include <linux/kref.h>
8 #include <linux/mempool.h>
9 #include <linux/rbtree.h>
10 #include <linux/refcount.h>
11 #include <linux/ktime.h>
12 
13 #include <linux/ceph/types.h>
14 #include <linux/ceph/osdmap.h>
15 #include <linux/ceph/messenger.h>
16 #include <linux/ceph/msgpool.h>
17 #include <linux/ceph/auth.h>
18 #include <linux/ceph/pagelist.h>
19 
20 struct ceph_msg;
21 struct ceph_snap_context;
22 struct ceph_osd_request;
23 struct ceph_osd_client;
24 
25 /*
26  * completion callback for async writepages
27  */
28 typedef void (*ceph_osdc_callback_t)(struct ceph_osd_request *);
29 
30 #define CEPH_HOMELESS_OSD	-1
31 
32 /* a given osd we're communicating with */
33 struct ceph_osd {
34 	refcount_t o_ref;
35 	struct ceph_osd_client *o_osdc;
36 	int o_osd;
37 	int o_incarnation;
38 	struct rb_node o_node;
39 	struct ceph_connection o_con;
40 	struct rb_root o_requests;
41 	struct rb_root o_linger_requests;
42 	struct rb_root o_backoff_mappings;
43 	struct rb_root o_backoffs_by_id;
44 	struct list_head o_osd_lru;
45 	struct ceph_auth_handshake o_auth;
46 	unsigned long lru_ttl;
47 	struct list_head o_keepalive_item;
48 	struct mutex lock;
49 };
50 
51 #define CEPH_OSD_SLAB_OPS	2
52 #define CEPH_OSD_MAX_OPS	16
53 
54 enum ceph_osd_data_type {
55 	CEPH_OSD_DATA_TYPE_NONE = 0,
56 	CEPH_OSD_DATA_TYPE_PAGES,
57 	CEPH_OSD_DATA_TYPE_PAGELIST,
58 #ifdef CONFIG_BLOCK
59 	CEPH_OSD_DATA_TYPE_BIO,
60 #endif /* CONFIG_BLOCK */
61 	CEPH_OSD_DATA_TYPE_BVECS,
62 };
63 
64 struct ceph_osd_data {
65 	enum ceph_osd_data_type	type;
66 	union {
67 		struct {
68 			struct page	**pages;
69 			u64		length;
70 			u32		alignment;
71 			bool		pages_from_pool;
72 			bool		own_pages;
73 		};
74 		struct ceph_pagelist	*pagelist;
75 #ifdef CONFIG_BLOCK
76 		struct {
77 			struct ceph_bio_iter	bio_pos;
78 			u32			bio_length;
79 		};
80 #endif /* CONFIG_BLOCK */
81 		struct {
82 			struct ceph_bvec_iter	bvec_pos;
83 			u32			num_bvecs;
84 		};
85 	};
86 };
87 
88 struct ceph_osd_req_op {
89 	u16 op;           /* CEPH_OSD_OP_* */
90 	u32 flags;        /* CEPH_OSD_OP_FLAG_* */
91 	u32 indata_len;   /* request */
92 	u32 outdata_len;  /* reply */
93 	s32 rval;
94 
95 	union {
96 		struct ceph_osd_data raw_data_in;
97 		struct {
98 			u64 offset, length;
99 			u64 truncate_size;
100 			u32 truncate_seq;
101 			struct ceph_osd_data osd_data;
102 		} extent;
103 		struct {
104 			u32 name_len;
105 			u32 value_len;
106 			__u8 cmp_op;       /* CEPH_OSD_CMPXATTR_OP_* */
107 			__u8 cmp_mode;     /* CEPH_OSD_CMPXATTR_MODE_* */
108 			struct ceph_osd_data osd_data;
109 		} xattr;
110 		struct {
111 			const char *class_name;
112 			const char *method_name;
113 			struct ceph_osd_data request_info;
114 			struct ceph_osd_data request_data;
115 			struct ceph_osd_data response_data;
116 			__u8 class_len;
117 			__u8 method_len;
118 			u32 indata_len;
119 		} cls;
120 		struct {
121 			u64 cookie;
122 			__u8 op;           /* CEPH_OSD_WATCH_OP_ */
123 			u32 gen;
124 		} watch;
125 		struct {
126 			struct ceph_osd_data request_data;
127 		} notify_ack;
128 		struct {
129 			u64 cookie;
130 			struct ceph_osd_data request_data;
131 			struct ceph_osd_data response_data;
132 		} notify;
133 		struct {
134 			struct ceph_osd_data response_data;
135 		} list_watchers;
136 		struct {
137 			u64 expected_object_size;
138 			u64 expected_write_size;
139 			u32 flags;  /* CEPH_OSD_OP_ALLOC_HINT_FLAG_* */
140 		} alloc_hint;
141 		struct {
142 			u64 snapid;
143 			u64 src_version;
144 			u8 flags;
145 			u32 src_fadvise_flags;
146 			struct ceph_osd_data osd_data;
147 		} copy_from;
148 	};
149 };
150 
151 struct ceph_osd_request_target {
152 	struct ceph_object_id base_oid;
153 	struct ceph_object_locator base_oloc;
154 	struct ceph_object_id target_oid;
155 	struct ceph_object_locator target_oloc;
156 
157 	struct ceph_pg pgid;               /* last raw pg we mapped to */
158 	struct ceph_spg spgid;             /* last actual spg we mapped to */
159 	u32 pg_num;
160 	u32 pg_num_mask;
161 	struct ceph_osds acting;
162 	struct ceph_osds up;
163 	int size;
164 	int min_size;
165 	bool sort_bitwise;
166 	bool recovery_deletes;
167 
168 	unsigned int flags;                /* CEPH_OSD_FLAG_* */
169 	bool used_replica;
170 	bool paused;
171 
172 	u32 epoch;
173 	u32 last_force_resend;
174 
175 	int osd;
176 };
177 
178 /* an in-flight request */
179 struct ceph_osd_request {
180 	u64             r_tid;              /* unique for this client */
181 	struct rb_node  r_node;
182 	struct rb_node  r_mc_node;          /* map check */
183 	struct work_struct r_complete_work;
184 	struct ceph_osd *r_osd;
185 
186 	struct ceph_osd_request_target r_t;
187 #define r_base_oid	r_t.base_oid
188 #define r_base_oloc	r_t.base_oloc
189 #define r_flags		r_t.flags
190 
191 	struct ceph_msg  *r_request, *r_reply;
192 	u32               r_sent;      /* >0 if r_request is sending/sent */
193 
194 	/* request osd ops array  */
195 	unsigned int		r_num_ops;
196 
197 	int               r_result;
198 
199 	struct ceph_osd_client *r_osdc;
200 	struct kref       r_kref;
201 	bool              r_mempool;
202 	struct completion r_completion;       /* private to osd_client.c */
203 	ceph_osdc_callback_t r_callback;
204 
205 	struct inode *r_inode;         	      /* for use by callbacks */
206 	struct list_head r_private_item;      /* ditto */
207 	void *r_priv;			      /* ditto */
208 
209 	/* set by submitter */
210 	u64 r_snapid;                         /* for reads, CEPH_NOSNAP o/w */
211 	struct ceph_snap_context *r_snapc;    /* for writes */
212 	struct timespec64 r_mtime;            /* ditto */
213 	u64 r_data_offset;                    /* ditto */
214 	bool r_linger;                        /* don't resend on failure */
215 
216 	/* internal */
217 	unsigned long r_stamp;                /* jiffies, send or check time */
218 	unsigned long r_start_stamp;          /* jiffies */
219 	ktime_t r_start_latency;              /* ktime_t */
220 	ktime_t r_end_latency;                /* ktime_t */
221 	int r_attempts;
222 	u32 r_map_dne_bound;
223 
224 	struct ceph_osd_req_op r_ops[];
225 };
226 
227 struct ceph_request_redirect {
228 	struct ceph_object_locator oloc;
229 };
230 
231 /*
232  * osd request identifier
233  *
234  * caller name + incarnation# + tid to unique identify this request
235  */
236 struct ceph_osd_reqid {
237 	struct ceph_entity_name name;
238 	__le64 tid;
239 	__le32 inc;
240 } __packed;
241 
242 struct ceph_blkin_trace_info {
243 	__le64 trace_id;
244 	__le64 span_id;
245 	__le64 parent_span_id;
246 } __packed;
247 
248 typedef void (*rados_watchcb2_t)(void *arg, u64 notify_id, u64 cookie,
249 				 u64 notifier_id, void *data, size_t data_len);
250 typedef void (*rados_watcherrcb_t)(void *arg, u64 cookie, int err);
251 
252 struct ceph_osd_linger_request {
253 	struct ceph_osd_client *osdc;
254 	u64 linger_id;
255 	bool committed;
256 	bool is_watch;                  /* watch or notify */
257 
258 	struct ceph_osd *osd;
259 	struct ceph_osd_request *reg_req;
260 	struct ceph_osd_request *ping_req;
261 	unsigned long ping_sent;
262 	unsigned long watch_valid_thru;
263 	struct list_head pending_lworks;
264 
265 	struct ceph_osd_request_target t;
266 	u32 map_dne_bound;
267 
268 	struct timespec64 mtime;
269 
270 	struct kref kref;
271 	struct mutex lock;
272 	struct rb_node node;            /* osd */
273 	struct rb_node osdc_node;       /* osdc */
274 	struct rb_node mc_node;         /* map check */
275 	struct list_head scan_item;
276 
277 	struct completion reg_commit_wait;
278 	struct completion notify_finish_wait;
279 	int reg_commit_error;
280 	int notify_finish_error;
281 	int last_error;
282 
283 	u32 register_gen;
284 	u64 notify_id;
285 
286 	rados_watchcb2_t wcb;
287 	rados_watcherrcb_t errcb;
288 	void *data;
289 
290 	struct ceph_pagelist *request_pl;
291 	struct page **notify_id_pages;
292 
293 	struct page ***preply_pages;
294 	size_t *preply_len;
295 };
296 
297 struct ceph_watch_item {
298 	struct ceph_entity_name name;
299 	u64 cookie;
300 	struct ceph_entity_addr addr;
301 };
302 
303 struct ceph_spg_mapping {
304 	struct rb_node node;
305 	struct ceph_spg spgid;
306 
307 	struct rb_root backoffs;
308 };
309 
310 struct ceph_hobject_id {
311 	void *key;
312 	size_t key_len;
313 	void *oid;
314 	size_t oid_len;
315 	u64 snapid;
316 	u32 hash;
317 	u8 is_max;
318 	void *nspace;
319 	size_t nspace_len;
320 	s64 pool;
321 
322 	/* cache */
323 	u32 hash_reverse_bits;
324 };
325 
ceph_hoid_build_hash_cache(struct ceph_hobject_id * hoid)326 static inline void ceph_hoid_build_hash_cache(struct ceph_hobject_id *hoid)
327 {
328 	hoid->hash_reverse_bits = bitrev32(hoid->hash);
329 }
330 
331 /*
332  * PG-wide backoff: [begin, end)
333  * per-object backoff: begin == end
334  */
335 struct ceph_osd_backoff {
336 	struct rb_node spg_node;
337 	struct rb_node id_node;
338 
339 	struct ceph_spg spgid;
340 	u64 id;
341 	struct ceph_hobject_id *begin;
342 	struct ceph_hobject_id *end;
343 };
344 
345 #define CEPH_LINGER_ID_START	0xffff000000000000ULL
346 
347 struct ceph_osd_client {
348 	struct ceph_client     *client;
349 
350 	struct ceph_osdmap     *osdmap;       /* current map */
351 	struct rw_semaphore    lock;
352 
353 	struct rb_root         osds;          /* osds */
354 	struct list_head       osd_lru;       /* idle osds */
355 	spinlock_t             osd_lru_lock;
356 	u32		       epoch_barrier;
357 	struct ceph_osd        homeless_osd;
358 	atomic64_t             last_tid;      /* tid of last request */
359 	u64                    last_linger_id;
360 	struct rb_root         linger_requests; /* lingering requests */
361 	struct rb_root         map_checks;
362 	struct rb_root         linger_map_checks;
363 	atomic_t               num_requests;
364 	atomic_t               num_homeless;
365 	int                    abort_err;
366 	struct delayed_work    timeout_work;
367 	struct delayed_work    osds_timeout_work;
368 #ifdef CONFIG_DEBUG_FS
369 	struct dentry 	       *debugfs_file;
370 #endif
371 
372 	mempool_t              *req_mempool;
373 
374 	struct ceph_msgpool	msgpool_op;
375 	struct ceph_msgpool	msgpool_op_reply;
376 
377 	struct workqueue_struct	*notify_wq;
378 	struct workqueue_struct	*completion_wq;
379 };
380 
ceph_osdmap_flag(struct ceph_osd_client * osdc,int flag)381 static inline bool ceph_osdmap_flag(struct ceph_osd_client *osdc, int flag)
382 {
383 	return osdc->osdmap->flags & flag;
384 }
385 
386 extern int ceph_osdc_setup(void);
387 extern void ceph_osdc_cleanup(void);
388 
389 extern int ceph_osdc_init(struct ceph_osd_client *osdc,
390 			  struct ceph_client *client);
391 extern void ceph_osdc_stop(struct ceph_osd_client *osdc);
392 extern void ceph_osdc_reopen_osds(struct ceph_osd_client *osdc);
393 
394 extern void ceph_osdc_handle_reply(struct ceph_osd_client *osdc,
395 				   struct ceph_msg *msg);
396 extern void ceph_osdc_handle_map(struct ceph_osd_client *osdc,
397 				 struct ceph_msg *msg);
398 void ceph_osdc_update_epoch_barrier(struct ceph_osd_client *osdc, u32 eb);
399 void ceph_osdc_abort_requests(struct ceph_osd_client *osdc, int err);
400 void ceph_osdc_clear_abort_err(struct ceph_osd_client *osdc);
401 
402 #define osd_req_op_data(oreq, whch, typ, fld)				\
403 ({									\
404 	struct ceph_osd_request *__oreq = (oreq);			\
405 	unsigned int __whch = (whch);					\
406 	BUG_ON(__whch >= __oreq->r_num_ops);				\
407 	&__oreq->r_ops[__whch].typ.fld;					\
408 })
409 
410 struct ceph_osd_req_op *osd_req_op_init(struct ceph_osd_request *osd_req,
411 			    unsigned int which, u16 opcode, u32 flags);
412 
413 extern void osd_req_op_raw_data_in_pages(struct ceph_osd_request *,
414 					unsigned int which,
415 					struct page **pages, u64 length,
416 					u32 alignment, bool pages_from_pool,
417 					bool own_pages);
418 
419 extern void osd_req_op_extent_init(struct ceph_osd_request *osd_req,
420 					unsigned int which, u16 opcode,
421 					u64 offset, u64 length,
422 					u64 truncate_size, u32 truncate_seq);
423 extern void osd_req_op_extent_update(struct ceph_osd_request *osd_req,
424 					unsigned int which, u64 length);
425 extern void osd_req_op_extent_dup_last(struct ceph_osd_request *osd_req,
426 				       unsigned int which, u64 offset_inc);
427 
428 extern struct ceph_osd_data *osd_req_op_extent_osd_data(
429 					struct ceph_osd_request *osd_req,
430 					unsigned int which);
431 
432 extern void osd_req_op_extent_osd_data_pages(struct ceph_osd_request *,
433 					unsigned int which,
434 					struct page **pages, u64 length,
435 					u32 alignment, bool pages_from_pool,
436 					bool own_pages);
437 extern void osd_req_op_extent_osd_data_pagelist(struct ceph_osd_request *,
438 					unsigned int which,
439 					struct ceph_pagelist *pagelist);
440 #ifdef CONFIG_BLOCK
441 void osd_req_op_extent_osd_data_bio(struct ceph_osd_request *osd_req,
442 				    unsigned int which,
443 				    struct ceph_bio_iter *bio_pos,
444 				    u32 bio_length);
445 #endif /* CONFIG_BLOCK */
446 void osd_req_op_extent_osd_data_bvecs(struct ceph_osd_request *osd_req,
447 				      unsigned int which,
448 				      struct bio_vec *bvecs, u32 num_bvecs,
449 				      u32 bytes);
450 void osd_req_op_extent_osd_data_bvec_pos(struct ceph_osd_request *osd_req,
451 					 unsigned int which,
452 					 struct ceph_bvec_iter *bvec_pos);
453 
454 extern void osd_req_op_cls_request_data_pagelist(struct ceph_osd_request *,
455 					unsigned int which,
456 					struct ceph_pagelist *pagelist);
457 extern void osd_req_op_cls_request_data_pages(struct ceph_osd_request *,
458 					unsigned int which,
459 					struct page **pages, u64 length,
460 					u32 alignment, bool pages_from_pool,
461 					bool own_pages);
462 void osd_req_op_cls_request_data_bvecs(struct ceph_osd_request *osd_req,
463 				       unsigned int which,
464 				       struct bio_vec *bvecs, u32 num_bvecs,
465 				       u32 bytes);
466 extern void osd_req_op_cls_response_data_pages(struct ceph_osd_request *,
467 					unsigned int which,
468 					struct page **pages, u64 length,
469 					u32 alignment, bool pages_from_pool,
470 					bool own_pages);
471 int osd_req_op_cls_init(struct ceph_osd_request *osd_req, unsigned int which,
472 			const char *class, const char *method);
473 extern int osd_req_op_xattr_init(struct ceph_osd_request *osd_req, unsigned int which,
474 				 u16 opcode, const char *name, const void *value,
475 				 size_t size, u8 cmp_op, u8 cmp_mode);
476 extern void osd_req_op_alloc_hint_init(struct ceph_osd_request *osd_req,
477 				       unsigned int which,
478 				       u64 expected_object_size,
479 				       u64 expected_write_size,
480 				       u32 flags);
481 extern int osd_req_op_copy_from_init(struct ceph_osd_request *req,
482 				     u64 src_snapid, u64 src_version,
483 				     struct ceph_object_id *src_oid,
484 				     struct ceph_object_locator *src_oloc,
485 				     u32 src_fadvise_flags,
486 				     u32 dst_fadvise_flags,
487 				     u32 truncate_seq, u64 truncate_size,
488 				     u8 copy_from_flags);
489 
490 extern struct ceph_osd_request *ceph_osdc_alloc_request(struct ceph_osd_client *osdc,
491 					       struct ceph_snap_context *snapc,
492 					       unsigned int num_ops,
493 					       bool use_mempool,
494 					       gfp_t gfp_flags);
495 int ceph_osdc_alloc_messages(struct ceph_osd_request *req, gfp_t gfp);
496 
497 extern struct ceph_osd_request *ceph_osdc_new_request(struct ceph_osd_client *,
498 				      struct ceph_file_layout *layout,
499 				      struct ceph_vino vino,
500 				      u64 offset, u64 *len,
501 				      unsigned int which, int num_ops,
502 				      int opcode, int flags,
503 				      struct ceph_snap_context *snapc,
504 				      u32 truncate_seq, u64 truncate_size,
505 				      bool use_mempool);
506 
507 extern void ceph_osdc_get_request(struct ceph_osd_request *req);
508 extern void ceph_osdc_put_request(struct ceph_osd_request *req);
509 
510 extern int ceph_osdc_start_request(struct ceph_osd_client *osdc,
511 				   struct ceph_osd_request *req,
512 				   bool nofail);
513 extern void ceph_osdc_cancel_request(struct ceph_osd_request *req);
514 extern int ceph_osdc_wait_request(struct ceph_osd_client *osdc,
515 				  struct ceph_osd_request *req);
516 extern void ceph_osdc_sync(struct ceph_osd_client *osdc);
517 
518 extern void ceph_osdc_flush_notifies(struct ceph_osd_client *osdc);
519 void ceph_osdc_maybe_request_map(struct ceph_osd_client *osdc);
520 
521 int ceph_osdc_call(struct ceph_osd_client *osdc,
522 		   struct ceph_object_id *oid,
523 		   struct ceph_object_locator *oloc,
524 		   const char *class, const char *method,
525 		   unsigned int flags,
526 		   struct page *req_page, size_t req_len,
527 		   struct page **resp_pages, size_t *resp_len);
528 
529 /* watch/notify */
530 struct ceph_osd_linger_request *
531 ceph_osdc_watch(struct ceph_osd_client *osdc,
532 		struct ceph_object_id *oid,
533 		struct ceph_object_locator *oloc,
534 		rados_watchcb2_t wcb,
535 		rados_watcherrcb_t errcb,
536 		void *data);
537 int ceph_osdc_unwatch(struct ceph_osd_client *osdc,
538 		      struct ceph_osd_linger_request *lreq);
539 
540 int ceph_osdc_notify_ack(struct ceph_osd_client *osdc,
541 			 struct ceph_object_id *oid,
542 			 struct ceph_object_locator *oloc,
543 			 u64 notify_id,
544 			 u64 cookie,
545 			 void *payload,
546 			 u32 payload_len);
547 int ceph_osdc_notify(struct ceph_osd_client *osdc,
548 		     struct ceph_object_id *oid,
549 		     struct ceph_object_locator *oloc,
550 		     void *payload,
551 		     u32 payload_len,
552 		     u32 timeout,
553 		     struct page ***preply_pages,
554 		     size_t *preply_len);
555 int ceph_osdc_watch_check(struct ceph_osd_client *osdc,
556 			  struct ceph_osd_linger_request *lreq);
557 int ceph_osdc_list_watchers(struct ceph_osd_client *osdc,
558 			    struct ceph_object_id *oid,
559 			    struct ceph_object_locator *oloc,
560 			    struct ceph_watch_item **watchers,
561 			    u32 *num_watchers);
562 #endif
563 
564