1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  *  linux/kernel/printk.c
4  *
5  *  Copyright (C) 1991, 1992  Linus Torvalds
6  *
7  * Modified to make sys_syslog() more flexible: added commands to
8  * return the last 4k of kernel messages, regardless of whether
9  * they've been read or not.  Added option to suppress kernel printk's
10  * to the console.  Added hook for sending the console messages
11  * elsewhere, in preparation for a serial line console (someday).
12  * Ted Ts'o, 2/11/93.
13  * Modified for sysctl support, 1/8/97, Chris Horn.
14  * Fixed SMP synchronization, 08/08/99, Manfred Spraul
15  *     manfred@colorfullife.com
16  * Rewrote bits to get rid of console_lock
17  *	01Mar01 Andrew Morton
18  */
19 
20 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
21 
22 #include <linux/kernel.h>
23 #include <linux/mm.h>
24 #include <linux/tty.h>
25 #include <linux/tty_driver.h>
26 #include <linux/console.h>
27 #include <linux/init.h>
28 #include <linux/jiffies.h>
29 #include <linux/nmi.h>
30 #include <linux/module.h>
31 #include <linux/moduleparam.h>
32 #include <linux/delay.h>
33 #include <linux/smp.h>
34 #include <linux/security.h>
35 #include <linux/memblock.h>
36 #include <linux/syscalls.h>
37 #include <linux/crash_core.h>
38 #include <linux/ratelimit.h>
39 #include <linux/kmsg_dump.h>
40 #include <linux/syslog.h>
41 #include <linux/cpu.h>
42 #include <linux/rculist.h>
43 #include <linux/poll.h>
44 #include <linux/irq_work.h>
45 #include <linux/ctype.h>
46 #include <linux/uio.h>
47 #include <linux/sched/clock.h>
48 #include <linux/sched/debug.h>
49 #include <linux/sched/task_stack.h>
50 
51 #include <linux/uaccess.h>
52 #include <asm/sections.h>
53 
54 #include <trace/events/initcall.h>
55 #define CREATE_TRACE_POINTS
56 #include <trace/events/printk.h>
57 
58 #include "printk_ringbuffer.h"
59 #include "console_cmdline.h"
60 #include "braille.h"
61 #include "internal.h"
62 
63 int console_printk[4] = {
64 	CONSOLE_LOGLEVEL_DEFAULT,	/* console_loglevel */
65 	MESSAGE_LOGLEVEL_DEFAULT,	/* default_message_loglevel */
66 	CONSOLE_LOGLEVEL_MIN,		/* minimum_console_loglevel */
67 	CONSOLE_LOGLEVEL_DEFAULT,	/* default_console_loglevel */
68 };
69 EXPORT_SYMBOL_GPL(console_printk);
70 
71 atomic_t ignore_console_lock_warning __read_mostly = ATOMIC_INIT(0);
72 EXPORT_SYMBOL(ignore_console_lock_warning);
73 
74 EXPORT_TRACEPOINT_SYMBOL_GPL(console);
75 
76 /*
77  * Low level drivers may need that to know if they can schedule in
78  * their unblank() callback or not. So let's export it.
79  */
80 int oops_in_progress;
81 EXPORT_SYMBOL(oops_in_progress);
82 
83 /*
84  * console_mutex protects console_list updates and console->flags updates.
85  * The flags are synchronized only for consoles that are registered, i.e.
86  * accessible via the console list.
87  */
88 static DEFINE_MUTEX(console_mutex);
89 
90 /*
91  * console_sem protects updates to console->seq
92  * and also provides serialization for console printing.
93  */
94 static DEFINE_SEMAPHORE(console_sem, 1);
95 HLIST_HEAD(console_list);
96 EXPORT_SYMBOL_GPL(console_list);
97 DEFINE_STATIC_SRCU(console_srcu);
98 
99 /*
100  * System may need to suppress printk message under certain
101  * circumstances, like after kernel panic happens.
102  */
103 int __read_mostly suppress_printk;
104 
105 /*
106  * During panic, heavy printk by other CPUs can delay the
107  * panic and risk deadlock on console resources.
108  */
109 static int __read_mostly suppress_panic_printk;
110 
111 #ifdef CONFIG_LOCKDEP
112 static struct lockdep_map console_lock_dep_map = {
113 	.name = "console_lock"
114 };
115 
lockdep_assert_console_list_lock_held(void)116 void lockdep_assert_console_list_lock_held(void)
117 {
118 	lockdep_assert_held(&console_mutex);
119 }
120 EXPORT_SYMBOL(lockdep_assert_console_list_lock_held);
121 #endif
122 
123 #ifdef CONFIG_DEBUG_LOCK_ALLOC
console_srcu_read_lock_is_held(void)124 bool console_srcu_read_lock_is_held(void)
125 {
126 	return srcu_read_lock_held(&console_srcu);
127 }
128 EXPORT_SYMBOL(console_srcu_read_lock_is_held);
129 #endif
130 
131 enum devkmsg_log_bits {
132 	__DEVKMSG_LOG_BIT_ON = 0,
133 	__DEVKMSG_LOG_BIT_OFF,
134 	__DEVKMSG_LOG_BIT_LOCK,
135 };
136 
137 enum devkmsg_log_masks {
138 	DEVKMSG_LOG_MASK_ON             = BIT(__DEVKMSG_LOG_BIT_ON),
139 	DEVKMSG_LOG_MASK_OFF            = BIT(__DEVKMSG_LOG_BIT_OFF),
140 	DEVKMSG_LOG_MASK_LOCK           = BIT(__DEVKMSG_LOG_BIT_LOCK),
141 };
142 
143 /* Keep both the 'on' and 'off' bits clear, i.e. ratelimit by default: */
144 #define DEVKMSG_LOG_MASK_DEFAULT	0
145 
146 static unsigned int __read_mostly devkmsg_log = DEVKMSG_LOG_MASK_DEFAULT;
147 
__control_devkmsg(char * str)148 static int __control_devkmsg(char *str)
149 {
150 	size_t len;
151 
152 	if (!str)
153 		return -EINVAL;
154 
155 	len = str_has_prefix(str, "on");
156 	if (len) {
157 		devkmsg_log = DEVKMSG_LOG_MASK_ON;
158 		return len;
159 	}
160 
161 	len = str_has_prefix(str, "off");
162 	if (len) {
163 		devkmsg_log = DEVKMSG_LOG_MASK_OFF;
164 		return len;
165 	}
166 
167 	len = str_has_prefix(str, "ratelimit");
168 	if (len) {
169 		devkmsg_log = DEVKMSG_LOG_MASK_DEFAULT;
170 		return len;
171 	}
172 
173 	return -EINVAL;
174 }
175 
control_devkmsg(char * str)176 static int __init control_devkmsg(char *str)
177 {
178 	if (__control_devkmsg(str) < 0) {
179 		pr_warn("printk.devkmsg: bad option string '%s'\n", str);
180 		return 1;
181 	}
182 
183 	/*
184 	 * Set sysctl string accordingly:
185 	 */
186 	if (devkmsg_log == DEVKMSG_LOG_MASK_ON)
187 		strcpy(devkmsg_log_str, "on");
188 	else if (devkmsg_log == DEVKMSG_LOG_MASK_OFF)
189 		strcpy(devkmsg_log_str, "off");
190 	/* else "ratelimit" which is set by default. */
191 
192 	/*
193 	 * Sysctl cannot change it anymore. The kernel command line setting of
194 	 * this parameter is to force the setting to be permanent throughout the
195 	 * runtime of the system. This is a precation measure against userspace
196 	 * trying to be a smarta** and attempting to change it up on us.
197 	 */
198 	devkmsg_log |= DEVKMSG_LOG_MASK_LOCK;
199 
200 	return 1;
201 }
202 __setup("printk.devkmsg=", control_devkmsg);
203 
204 char devkmsg_log_str[DEVKMSG_STR_MAX_SIZE] = "ratelimit";
205 #if defined(CONFIG_PRINTK) && defined(CONFIG_SYSCTL)
devkmsg_sysctl_set_loglvl(struct ctl_table * table,int write,void * buffer,size_t * lenp,loff_t * ppos)206 int devkmsg_sysctl_set_loglvl(struct ctl_table *table, int write,
207 			      void *buffer, size_t *lenp, loff_t *ppos)
208 {
209 	char old_str[DEVKMSG_STR_MAX_SIZE];
210 	unsigned int old;
211 	int err;
212 
213 	if (write) {
214 		if (devkmsg_log & DEVKMSG_LOG_MASK_LOCK)
215 			return -EINVAL;
216 
217 		old = devkmsg_log;
218 		strncpy(old_str, devkmsg_log_str, DEVKMSG_STR_MAX_SIZE);
219 	}
220 
221 	err = proc_dostring(table, write, buffer, lenp, ppos);
222 	if (err)
223 		return err;
224 
225 	if (write) {
226 		err = __control_devkmsg(devkmsg_log_str);
227 
228 		/*
229 		 * Do not accept an unknown string OR a known string with
230 		 * trailing crap...
231 		 */
232 		if (err < 0 || (err + 1 != *lenp)) {
233 
234 			/* ... and restore old setting. */
235 			devkmsg_log = old;
236 			strncpy(devkmsg_log_str, old_str, DEVKMSG_STR_MAX_SIZE);
237 
238 			return -EINVAL;
239 		}
240 	}
241 
242 	return 0;
243 }
244 #endif /* CONFIG_PRINTK && CONFIG_SYSCTL */
245 
246 /**
247  * console_list_lock - Lock the console list
248  *
249  * For console list or console->flags updates
250  */
console_list_lock(void)251 void console_list_lock(void)
252 {
253 	/*
254 	 * In unregister_console() and console_force_preferred_locked(),
255 	 * synchronize_srcu() is called with the console_list_lock held.
256 	 * Therefore it is not allowed that the console_list_lock is taken
257 	 * with the srcu_lock held.
258 	 *
259 	 * Detecting if this context is really in the read-side critical
260 	 * section is only possible if the appropriate debug options are
261 	 * enabled.
262 	 */
263 	WARN_ON_ONCE(debug_lockdep_rcu_enabled() &&
264 		     srcu_read_lock_held(&console_srcu));
265 
266 	mutex_lock(&console_mutex);
267 }
268 EXPORT_SYMBOL(console_list_lock);
269 
270 /**
271  * console_list_unlock - Unlock the console list
272  *
273  * Counterpart to console_list_lock()
274  */
console_list_unlock(void)275 void console_list_unlock(void)
276 {
277 	mutex_unlock(&console_mutex);
278 }
279 EXPORT_SYMBOL(console_list_unlock);
280 
281 /**
282  * console_srcu_read_lock - Register a new reader for the
283  *	SRCU-protected console list
284  *
285  * Use for_each_console_srcu() to iterate the console list
286  *
287  * Context: Any context.
288  * Return: A cookie to pass to console_srcu_read_unlock().
289  */
console_srcu_read_lock(void)290 int console_srcu_read_lock(void)
291 {
292 	return srcu_read_lock_nmisafe(&console_srcu);
293 }
294 EXPORT_SYMBOL(console_srcu_read_lock);
295 
296 /**
297  * console_srcu_read_unlock - Unregister an old reader from
298  *	the SRCU-protected console list
299  * @cookie: cookie returned from console_srcu_read_lock()
300  *
301  * Counterpart to console_srcu_read_lock()
302  */
console_srcu_read_unlock(int cookie)303 void console_srcu_read_unlock(int cookie)
304 {
305 	srcu_read_unlock_nmisafe(&console_srcu, cookie);
306 }
307 EXPORT_SYMBOL(console_srcu_read_unlock);
308 
309 /*
310  * Helper macros to handle lockdep when locking/unlocking console_sem. We use
311  * macros instead of functions so that _RET_IP_ contains useful information.
312  */
313 #define down_console_sem() do { \
314 	down(&console_sem);\
315 	mutex_acquire(&console_lock_dep_map, 0, 0, _RET_IP_);\
316 } while (0)
317 
__down_trylock_console_sem(unsigned long ip)318 static int __down_trylock_console_sem(unsigned long ip)
319 {
320 	int lock_failed;
321 	unsigned long flags;
322 
323 	/*
324 	 * Here and in __up_console_sem() we need to be in safe mode,
325 	 * because spindump/WARN/etc from under console ->lock will
326 	 * deadlock in printk()->down_trylock_console_sem() otherwise.
327 	 */
328 	printk_safe_enter_irqsave(flags);
329 	lock_failed = down_trylock(&console_sem);
330 	printk_safe_exit_irqrestore(flags);
331 
332 	if (lock_failed)
333 		return 1;
334 	mutex_acquire(&console_lock_dep_map, 0, 1, ip);
335 	return 0;
336 }
337 #define down_trylock_console_sem() __down_trylock_console_sem(_RET_IP_)
338 
__up_console_sem(unsigned long ip)339 static void __up_console_sem(unsigned long ip)
340 {
341 	unsigned long flags;
342 
343 	mutex_release(&console_lock_dep_map, ip);
344 
345 	printk_safe_enter_irqsave(flags);
346 	up(&console_sem);
347 	printk_safe_exit_irqrestore(flags);
348 }
349 #define up_console_sem() __up_console_sem(_RET_IP_)
350 
panic_in_progress(void)351 static bool panic_in_progress(void)
352 {
353 	return unlikely(atomic_read(&panic_cpu) != PANIC_CPU_INVALID);
354 }
355 
356 /*
357  * This is used for debugging the mess that is the VT code by
358  * keeping track if we have the console semaphore held. It's
359  * definitely not the perfect debug tool (we don't know if _WE_
360  * hold it and are racing, but it helps tracking those weird code
361  * paths in the console code where we end up in places I want
362  * locked without the console semaphore held).
363  */
364 static int console_locked;
365 
366 /*
367  *	Array of consoles built from command line options (console=)
368  */
369 
370 #define MAX_CMDLINECONSOLES 8
371 
372 static struct console_cmdline console_cmdline[MAX_CMDLINECONSOLES];
373 
374 static int preferred_console = -1;
375 int console_set_on_cmdline;
376 EXPORT_SYMBOL(console_set_on_cmdline);
377 
378 /* Flag: console code may call schedule() */
379 static int console_may_schedule;
380 
381 enum con_msg_format_flags {
382 	MSG_FORMAT_DEFAULT	= 0,
383 	MSG_FORMAT_SYSLOG	= (1 << 0),
384 };
385 
386 static int console_msg_format = MSG_FORMAT_DEFAULT;
387 
388 /*
389  * The printk log buffer consists of a sequenced collection of records, each
390  * containing variable length message text. Every record also contains its
391  * own meta-data (@info).
392  *
393  * Every record meta-data carries the timestamp in microseconds, as well as
394  * the standard userspace syslog level and syslog facility. The usual kernel
395  * messages use LOG_KERN; userspace-injected messages always carry a matching
396  * syslog facility, by default LOG_USER. The origin of every message can be
397  * reliably determined that way.
398  *
399  * The human readable log message of a record is available in @text, the
400  * length of the message text in @text_len. The stored message is not
401  * terminated.
402  *
403  * Optionally, a record can carry a dictionary of properties (key/value
404  * pairs), to provide userspace with a machine-readable message context.
405  *
406  * Examples for well-defined, commonly used property names are:
407  *   DEVICE=b12:8               device identifier
408  *                                b12:8         block dev_t
409  *                                c127:3        char dev_t
410  *                                n8            netdev ifindex
411  *                                +sound:card0  subsystem:devname
412  *   SUBSYSTEM=pci              driver-core subsystem name
413  *
414  * Valid characters in property names are [a-zA-Z0-9.-_]. Property names
415  * and values are terminated by a '\0' character.
416  *
417  * Example of record values:
418  *   record.text_buf                = "it's a line" (unterminated)
419  *   record.info.seq                = 56
420  *   record.info.ts_nsec            = 36863
421  *   record.info.text_len           = 11
422  *   record.info.facility           = 0 (LOG_KERN)
423  *   record.info.flags              = 0
424  *   record.info.level              = 3 (LOG_ERR)
425  *   record.info.caller_id          = 299 (task 299)
426  *   record.info.dev_info.subsystem = "pci" (terminated)
427  *   record.info.dev_info.device    = "+pci:0000:00:01.0" (terminated)
428  *
429  * The 'struct printk_info' buffer must never be directly exported to
430  * userspace, it is a kernel-private implementation detail that might
431  * need to be changed in the future, when the requirements change.
432  *
433  * /dev/kmsg exports the structured data in the following line format:
434  *   "<level>,<sequnum>,<timestamp>,<contflag>[,additional_values, ... ];<message text>\n"
435  *
436  * Users of the export format should ignore possible additional values
437  * separated by ',', and find the message after the ';' character.
438  *
439  * The optional key/value pairs are attached as continuation lines starting
440  * with a space character and terminated by a newline. All possible
441  * non-prinatable characters are escaped in the "\xff" notation.
442  */
443 
444 /* syslog_lock protects syslog_* variables and write access to clear_seq. */
445 static DEFINE_MUTEX(syslog_lock);
446 
447 #ifdef CONFIG_PRINTK
448 DECLARE_WAIT_QUEUE_HEAD(log_wait);
449 /* All 3 protected by @syslog_lock. */
450 /* the next printk record to read by syslog(READ) or /proc/kmsg */
451 static u64 syslog_seq;
452 static size_t syslog_partial;
453 static bool syslog_time;
454 
455 struct latched_seq {
456 	seqcount_latch_t	latch;
457 	u64			val[2];
458 };
459 
460 /*
461  * The next printk record to read after the last 'clear' command. There are
462  * two copies (updated with seqcount_latch) so that reads can locklessly
463  * access a valid value. Writers are synchronized by @syslog_lock.
464  */
465 static struct latched_seq clear_seq = {
466 	.latch		= SEQCNT_LATCH_ZERO(clear_seq.latch),
467 	.val[0]		= 0,
468 	.val[1]		= 0,
469 };
470 
471 #define LOG_LEVEL(v)		((v) & 0x07)
472 #define LOG_FACILITY(v)		((v) >> 3 & 0xff)
473 
474 /* record buffer */
475 #define LOG_ALIGN __alignof__(unsigned long)
476 #define __LOG_BUF_LEN (1 << CONFIG_LOG_BUF_SHIFT)
477 #define LOG_BUF_LEN_MAX (u32)(1 << 31)
478 static char __log_buf[__LOG_BUF_LEN] __aligned(LOG_ALIGN);
479 static char *log_buf = __log_buf;
480 static u32 log_buf_len = __LOG_BUF_LEN;
481 
482 /*
483  * Define the average message size. This only affects the number of
484  * descriptors that will be available. Underestimating is better than
485  * overestimating (too many available descriptors is better than not enough).
486  */
487 #define PRB_AVGBITS 5	/* 32 character average length */
488 
489 #if CONFIG_LOG_BUF_SHIFT <= PRB_AVGBITS
490 #error CONFIG_LOG_BUF_SHIFT value too small.
491 #endif
492 _DEFINE_PRINTKRB(printk_rb_static, CONFIG_LOG_BUF_SHIFT - PRB_AVGBITS,
493 		 PRB_AVGBITS, &__log_buf[0]);
494 
495 static struct printk_ringbuffer printk_rb_dynamic;
496 
497 static struct printk_ringbuffer *prb = &printk_rb_static;
498 
499 /*
500  * We cannot access per-CPU data (e.g. per-CPU flush irq_work) before
501  * per_cpu_areas are initialised. This variable is set to true when
502  * it's safe to access per-CPU data.
503  */
504 static bool __printk_percpu_data_ready __ro_after_init;
505 
printk_percpu_data_ready(void)506 bool printk_percpu_data_ready(void)
507 {
508 	return __printk_percpu_data_ready;
509 }
510 
511 /* Must be called under syslog_lock. */
latched_seq_write(struct latched_seq * ls,u64 val)512 static void latched_seq_write(struct latched_seq *ls, u64 val)
513 {
514 	raw_write_seqcount_latch(&ls->latch);
515 	ls->val[0] = val;
516 	raw_write_seqcount_latch(&ls->latch);
517 	ls->val[1] = val;
518 }
519 
520 /* Can be called from any context. */
latched_seq_read_nolock(struct latched_seq * ls)521 static u64 latched_seq_read_nolock(struct latched_seq *ls)
522 {
523 	unsigned int seq;
524 	unsigned int idx;
525 	u64 val;
526 
527 	do {
528 		seq = raw_read_seqcount_latch(&ls->latch);
529 		idx = seq & 0x1;
530 		val = ls->val[idx];
531 	} while (raw_read_seqcount_latch_retry(&ls->latch, seq));
532 
533 	return val;
534 }
535 
536 /* Return log buffer address */
log_buf_addr_get(void)537 char *log_buf_addr_get(void)
538 {
539 	return log_buf;
540 }
541 
542 /* Return log buffer size */
log_buf_len_get(void)543 u32 log_buf_len_get(void)
544 {
545 	return log_buf_len;
546 }
547 
548 /*
549  * Define how much of the log buffer we could take at maximum. The value
550  * must be greater than two. Note that only half of the buffer is available
551  * when the index points to the middle.
552  */
553 #define MAX_LOG_TAKE_PART 4
554 static const char trunc_msg[] = "<truncated>";
555 
truncate_msg(u16 * text_len,u16 * trunc_msg_len)556 static void truncate_msg(u16 *text_len, u16 *trunc_msg_len)
557 {
558 	/*
559 	 * The message should not take the whole buffer. Otherwise, it might
560 	 * get removed too soon.
561 	 */
562 	u32 max_text_len = log_buf_len / MAX_LOG_TAKE_PART;
563 
564 	if (*text_len > max_text_len)
565 		*text_len = max_text_len;
566 
567 	/* enable the warning message (if there is room) */
568 	*trunc_msg_len = strlen(trunc_msg);
569 	if (*text_len >= *trunc_msg_len)
570 		*text_len -= *trunc_msg_len;
571 	else
572 		*trunc_msg_len = 0;
573 }
574 
575 int dmesg_restrict = IS_ENABLED(CONFIG_SECURITY_DMESG_RESTRICT);
576 
syslog_action_restricted(int type)577 static int syslog_action_restricted(int type)
578 {
579 	if (dmesg_restrict)
580 		return 1;
581 	/*
582 	 * Unless restricted, we allow "read all" and "get buffer size"
583 	 * for everybody.
584 	 */
585 	return type != SYSLOG_ACTION_READ_ALL &&
586 	       type != SYSLOG_ACTION_SIZE_BUFFER;
587 }
588 
check_syslog_permissions(int type,int source)589 static int check_syslog_permissions(int type, int source)
590 {
591 	/*
592 	 * If this is from /proc/kmsg and we've already opened it, then we've
593 	 * already done the capabilities checks at open time.
594 	 */
595 	if (source == SYSLOG_FROM_PROC && type != SYSLOG_ACTION_OPEN)
596 		goto ok;
597 
598 	if (syslog_action_restricted(type)) {
599 		if (capable(CAP_SYSLOG))
600 			goto ok;
601 		/*
602 		 * For historical reasons, accept CAP_SYS_ADMIN too, with
603 		 * a warning.
604 		 */
605 		if (capable(CAP_SYS_ADMIN)) {
606 			pr_warn_once("%s (%d): Attempt to access syslog with "
607 				     "CAP_SYS_ADMIN but no CAP_SYSLOG "
608 				     "(deprecated).\n",
609 				 current->comm, task_pid_nr(current));
610 			goto ok;
611 		}
612 		return -EPERM;
613 	}
614 ok:
615 	return security_syslog(type);
616 }
617 
append_char(char ** pp,char * e,char c)618 static void append_char(char **pp, char *e, char c)
619 {
620 	if (*pp < e)
621 		*(*pp)++ = c;
622 }
623 
info_print_ext_header(char * buf,size_t size,struct printk_info * info)624 static ssize_t info_print_ext_header(char *buf, size_t size,
625 				     struct printk_info *info)
626 {
627 	u64 ts_usec = info->ts_nsec;
628 	char caller[20];
629 #ifdef CONFIG_PRINTK_CALLER
630 	u32 id = info->caller_id;
631 
632 	snprintf(caller, sizeof(caller), ",caller=%c%u",
633 		 id & 0x80000000 ? 'C' : 'T', id & ~0x80000000);
634 #else
635 	caller[0] = '\0';
636 #endif
637 
638 	do_div(ts_usec, 1000);
639 
640 	return scnprintf(buf, size, "%u,%llu,%llu,%c%s;",
641 			 (info->facility << 3) | info->level, info->seq,
642 			 ts_usec, info->flags & LOG_CONT ? 'c' : '-', caller);
643 }
644 
msg_add_ext_text(char * buf,size_t size,const char * text,size_t text_len,unsigned char endc)645 static ssize_t msg_add_ext_text(char *buf, size_t size,
646 				const char *text, size_t text_len,
647 				unsigned char endc)
648 {
649 	char *p = buf, *e = buf + size;
650 	size_t i;
651 
652 	/* escape non-printable characters */
653 	for (i = 0; i < text_len; i++) {
654 		unsigned char c = text[i];
655 
656 		if (c < ' ' || c >= 127 || c == '\\')
657 			p += scnprintf(p, e - p, "\\x%02x", c);
658 		else
659 			append_char(&p, e, c);
660 	}
661 	append_char(&p, e, endc);
662 
663 	return p - buf;
664 }
665 
msg_add_dict_text(char * buf,size_t size,const char * key,const char * val)666 static ssize_t msg_add_dict_text(char *buf, size_t size,
667 				 const char *key, const char *val)
668 {
669 	size_t val_len = strlen(val);
670 	ssize_t len;
671 
672 	if (!val_len)
673 		return 0;
674 
675 	len = msg_add_ext_text(buf, size, "", 0, ' ');	/* dict prefix */
676 	len += msg_add_ext_text(buf + len, size - len, key, strlen(key), '=');
677 	len += msg_add_ext_text(buf + len, size - len, val, val_len, '\n');
678 
679 	return len;
680 }
681 
msg_print_ext_body(char * buf,size_t size,char * text,size_t text_len,struct dev_printk_info * dev_info)682 static ssize_t msg_print_ext_body(char *buf, size_t size,
683 				  char *text, size_t text_len,
684 				  struct dev_printk_info *dev_info)
685 {
686 	ssize_t len;
687 
688 	len = msg_add_ext_text(buf, size, text, text_len, '\n');
689 
690 	if (!dev_info)
691 		goto out;
692 
693 	len += msg_add_dict_text(buf + len, size - len, "SUBSYSTEM",
694 				 dev_info->subsystem);
695 	len += msg_add_dict_text(buf + len, size - len, "DEVICE",
696 				 dev_info->device);
697 out:
698 	return len;
699 }
700 
701 static bool printk_get_next_message(struct printk_message *pmsg, u64 seq,
702 				    bool is_extended, bool may_supress);
703 
704 /* /dev/kmsg - userspace message inject/listen interface */
705 struct devkmsg_user {
706 	atomic64_t seq;
707 	struct ratelimit_state rs;
708 	struct mutex lock;
709 	struct printk_buffers pbufs;
710 };
711 
712 static __printf(3, 4) __cold
devkmsg_emit(int facility,int level,const char * fmt,...)713 int devkmsg_emit(int facility, int level, const char *fmt, ...)
714 {
715 	va_list args;
716 	int r;
717 
718 	va_start(args, fmt);
719 	r = vprintk_emit(facility, level, NULL, fmt, args);
720 	va_end(args);
721 
722 	return r;
723 }
724 
devkmsg_write(struct kiocb * iocb,struct iov_iter * from)725 static ssize_t devkmsg_write(struct kiocb *iocb, struct iov_iter *from)
726 {
727 	char *buf, *line;
728 	int level = default_message_loglevel;
729 	int facility = 1;	/* LOG_USER */
730 	struct file *file = iocb->ki_filp;
731 	struct devkmsg_user *user = file->private_data;
732 	size_t len = iov_iter_count(from);
733 	ssize_t ret = len;
734 
735 	if (len > PRINTKRB_RECORD_MAX)
736 		return -EINVAL;
737 
738 	/* Ignore when user logging is disabled. */
739 	if (devkmsg_log & DEVKMSG_LOG_MASK_OFF)
740 		return len;
741 
742 	/* Ratelimit when not explicitly enabled. */
743 	if (!(devkmsg_log & DEVKMSG_LOG_MASK_ON)) {
744 		if (!___ratelimit(&user->rs, current->comm))
745 			return ret;
746 	}
747 
748 	buf = kmalloc(len+1, GFP_KERNEL);
749 	if (buf == NULL)
750 		return -ENOMEM;
751 
752 	buf[len] = '\0';
753 	if (!copy_from_iter_full(buf, len, from)) {
754 		kfree(buf);
755 		return -EFAULT;
756 	}
757 
758 	/*
759 	 * Extract and skip the syslog prefix <[0-9]*>. Coming from userspace
760 	 * the decimal value represents 32bit, the lower 3 bit are the log
761 	 * level, the rest are the log facility.
762 	 *
763 	 * If no prefix or no userspace facility is specified, we
764 	 * enforce LOG_USER, to be able to reliably distinguish
765 	 * kernel-generated messages from userspace-injected ones.
766 	 */
767 	line = buf;
768 	if (line[0] == '<') {
769 		char *endp = NULL;
770 		unsigned int u;
771 
772 		u = simple_strtoul(line + 1, &endp, 10);
773 		if (endp && endp[0] == '>') {
774 			level = LOG_LEVEL(u);
775 			if (LOG_FACILITY(u) != 0)
776 				facility = LOG_FACILITY(u);
777 			endp++;
778 			line = endp;
779 		}
780 	}
781 
782 	devkmsg_emit(facility, level, "%s", line);
783 	kfree(buf);
784 	return ret;
785 }
786 
devkmsg_read(struct file * file,char __user * buf,size_t count,loff_t * ppos)787 static ssize_t devkmsg_read(struct file *file, char __user *buf,
788 			    size_t count, loff_t *ppos)
789 {
790 	struct devkmsg_user *user = file->private_data;
791 	char *outbuf = &user->pbufs.outbuf[0];
792 	struct printk_message pmsg = {
793 		.pbufs = &user->pbufs,
794 	};
795 	ssize_t ret;
796 
797 	ret = mutex_lock_interruptible(&user->lock);
798 	if (ret)
799 		return ret;
800 
801 	if (!printk_get_next_message(&pmsg, atomic64_read(&user->seq), true, false)) {
802 		if (file->f_flags & O_NONBLOCK) {
803 			ret = -EAGAIN;
804 			goto out;
805 		}
806 
807 		/*
808 		 * Guarantee this task is visible on the waitqueue before
809 		 * checking the wake condition.
810 		 *
811 		 * The full memory barrier within set_current_state() of
812 		 * prepare_to_wait_event() pairs with the full memory barrier
813 		 * within wq_has_sleeper().
814 		 *
815 		 * This pairs with __wake_up_klogd:A.
816 		 */
817 		ret = wait_event_interruptible(log_wait,
818 				printk_get_next_message(&pmsg, atomic64_read(&user->seq), true,
819 							false)); /* LMM(devkmsg_read:A) */
820 		if (ret)
821 			goto out;
822 	}
823 
824 	if (pmsg.dropped) {
825 		/* our last seen message is gone, return error and reset */
826 		atomic64_set(&user->seq, pmsg.seq);
827 		ret = -EPIPE;
828 		goto out;
829 	}
830 
831 	atomic64_set(&user->seq, pmsg.seq + 1);
832 
833 	if (pmsg.outbuf_len > count) {
834 		ret = -EINVAL;
835 		goto out;
836 	}
837 
838 	if (copy_to_user(buf, outbuf, pmsg.outbuf_len)) {
839 		ret = -EFAULT;
840 		goto out;
841 	}
842 	ret = pmsg.outbuf_len;
843 out:
844 	mutex_unlock(&user->lock);
845 	return ret;
846 }
847 
848 /*
849  * Be careful when modifying this function!!!
850  *
851  * Only few operations are supported because the device works only with the
852  * entire variable length messages (records). Non-standard values are
853  * returned in the other cases and has been this way for quite some time.
854  * User space applications might depend on this behavior.
855  */
devkmsg_llseek(struct file * file,loff_t offset,int whence)856 static loff_t devkmsg_llseek(struct file *file, loff_t offset, int whence)
857 {
858 	struct devkmsg_user *user = file->private_data;
859 	loff_t ret = 0;
860 
861 	if (offset)
862 		return -ESPIPE;
863 
864 	switch (whence) {
865 	case SEEK_SET:
866 		/* the first record */
867 		atomic64_set(&user->seq, prb_first_valid_seq(prb));
868 		break;
869 	case SEEK_DATA:
870 		/*
871 		 * The first record after the last SYSLOG_ACTION_CLEAR,
872 		 * like issued by 'dmesg -c'. Reading /dev/kmsg itself
873 		 * changes no global state, and does not clear anything.
874 		 */
875 		atomic64_set(&user->seq, latched_seq_read_nolock(&clear_seq));
876 		break;
877 	case SEEK_END:
878 		/* after the last record */
879 		atomic64_set(&user->seq, prb_next_seq(prb));
880 		break;
881 	default:
882 		ret = -EINVAL;
883 	}
884 	return ret;
885 }
886 
devkmsg_poll(struct file * file,poll_table * wait)887 static __poll_t devkmsg_poll(struct file *file, poll_table *wait)
888 {
889 	struct devkmsg_user *user = file->private_data;
890 	struct printk_info info;
891 	__poll_t ret = 0;
892 
893 	poll_wait(file, &log_wait, wait);
894 
895 	if (prb_read_valid_info(prb, atomic64_read(&user->seq), &info, NULL)) {
896 		/* return error when data has vanished underneath us */
897 		if (info.seq != atomic64_read(&user->seq))
898 			ret = EPOLLIN|EPOLLRDNORM|EPOLLERR|EPOLLPRI;
899 		else
900 			ret = EPOLLIN|EPOLLRDNORM;
901 	}
902 
903 	return ret;
904 }
905 
devkmsg_open(struct inode * inode,struct file * file)906 static int devkmsg_open(struct inode *inode, struct file *file)
907 {
908 	struct devkmsg_user *user;
909 	int err;
910 
911 	if (devkmsg_log & DEVKMSG_LOG_MASK_OFF)
912 		return -EPERM;
913 
914 	/* write-only does not need any file context */
915 	if ((file->f_flags & O_ACCMODE) != O_WRONLY) {
916 		err = check_syslog_permissions(SYSLOG_ACTION_READ_ALL,
917 					       SYSLOG_FROM_READER);
918 		if (err)
919 			return err;
920 	}
921 
922 	user = kvmalloc(sizeof(struct devkmsg_user), GFP_KERNEL);
923 	if (!user)
924 		return -ENOMEM;
925 
926 	ratelimit_default_init(&user->rs);
927 	ratelimit_set_flags(&user->rs, RATELIMIT_MSG_ON_RELEASE);
928 
929 	mutex_init(&user->lock);
930 
931 	atomic64_set(&user->seq, prb_first_valid_seq(prb));
932 
933 	file->private_data = user;
934 	return 0;
935 }
936 
devkmsg_release(struct inode * inode,struct file * file)937 static int devkmsg_release(struct inode *inode, struct file *file)
938 {
939 	struct devkmsg_user *user = file->private_data;
940 
941 	ratelimit_state_exit(&user->rs);
942 
943 	mutex_destroy(&user->lock);
944 	kvfree(user);
945 	return 0;
946 }
947 
948 const struct file_operations kmsg_fops = {
949 	.open = devkmsg_open,
950 	.read = devkmsg_read,
951 	.write_iter = devkmsg_write,
952 	.llseek = devkmsg_llseek,
953 	.poll = devkmsg_poll,
954 	.release = devkmsg_release,
955 };
956 
957 #ifdef CONFIG_CRASH_CORE
958 /*
959  * This appends the listed symbols to /proc/vmcore
960  *
961  * /proc/vmcore is used by various utilities, like crash and makedumpfile to
962  * obtain access to symbols that are otherwise very difficult to locate.  These
963  * symbols are specifically used so that utilities can access and extract the
964  * dmesg log from a vmcore file after a crash.
965  */
log_buf_vmcoreinfo_setup(void)966 void log_buf_vmcoreinfo_setup(void)
967 {
968 	struct dev_printk_info *dev_info = NULL;
969 
970 	VMCOREINFO_SYMBOL(prb);
971 	VMCOREINFO_SYMBOL(printk_rb_static);
972 	VMCOREINFO_SYMBOL(clear_seq);
973 
974 	/*
975 	 * Export struct size and field offsets. User space tools can
976 	 * parse it and detect any changes to structure down the line.
977 	 */
978 
979 	VMCOREINFO_STRUCT_SIZE(printk_ringbuffer);
980 	VMCOREINFO_OFFSET(printk_ringbuffer, desc_ring);
981 	VMCOREINFO_OFFSET(printk_ringbuffer, text_data_ring);
982 	VMCOREINFO_OFFSET(printk_ringbuffer, fail);
983 
984 	VMCOREINFO_STRUCT_SIZE(prb_desc_ring);
985 	VMCOREINFO_OFFSET(prb_desc_ring, count_bits);
986 	VMCOREINFO_OFFSET(prb_desc_ring, descs);
987 	VMCOREINFO_OFFSET(prb_desc_ring, infos);
988 	VMCOREINFO_OFFSET(prb_desc_ring, head_id);
989 	VMCOREINFO_OFFSET(prb_desc_ring, tail_id);
990 
991 	VMCOREINFO_STRUCT_SIZE(prb_desc);
992 	VMCOREINFO_OFFSET(prb_desc, state_var);
993 	VMCOREINFO_OFFSET(prb_desc, text_blk_lpos);
994 
995 	VMCOREINFO_STRUCT_SIZE(prb_data_blk_lpos);
996 	VMCOREINFO_OFFSET(prb_data_blk_lpos, begin);
997 	VMCOREINFO_OFFSET(prb_data_blk_lpos, next);
998 
999 	VMCOREINFO_STRUCT_SIZE(printk_info);
1000 	VMCOREINFO_OFFSET(printk_info, seq);
1001 	VMCOREINFO_OFFSET(printk_info, ts_nsec);
1002 	VMCOREINFO_OFFSET(printk_info, text_len);
1003 	VMCOREINFO_OFFSET(printk_info, caller_id);
1004 	VMCOREINFO_OFFSET(printk_info, dev_info);
1005 
1006 	VMCOREINFO_STRUCT_SIZE(dev_printk_info);
1007 	VMCOREINFO_OFFSET(dev_printk_info, subsystem);
1008 	VMCOREINFO_LENGTH(printk_info_subsystem, sizeof(dev_info->subsystem));
1009 	VMCOREINFO_OFFSET(dev_printk_info, device);
1010 	VMCOREINFO_LENGTH(printk_info_device, sizeof(dev_info->device));
1011 
1012 	VMCOREINFO_STRUCT_SIZE(prb_data_ring);
1013 	VMCOREINFO_OFFSET(prb_data_ring, size_bits);
1014 	VMCOREINFO_OFFSET(prb_data_ring, data);
1015 	VMCOREINFO_OFFSET(prb_data_ring, head_lpos);
1016 	VMCOREINFO_OFFSET(prb_data_ring, tail_lpos);
1017 
1018 	VMCOREINFO_SIZE(atomic_long_t);
1019 	VMCOREINFO_TYPE_OFFSET(atomic_long_t, counter);
1020 
1021 	VMCOREINFO_STRUCT_SIZE(latched_seq);
1022 	VMCOREINFO_OFFSET(latched_seq, val);
1023 }
1024 #endif
1025 
1026 /* requested log_buf_len from kernel cmdline */
1027 static unsigned long __initdata new_log_buf_len;
1028 
1029 /* we practice scaling the ring buffer by powers of 2 */
log_buf_len_update(u64 size)1030 static void __init log_buf_len_update(u64 size)
1031 {
1032 	if (size > (u64)LOG_BUF_LEN_MAX) {
1033 		size = (u64)LOG_BUF_LEN_MAX;
1034 		pr_err("log_buf over 2G is not supported.\n");
1035 	}
1036 
1037 	if (size)
1038 		size = roundup_pow_of_two(size);
1039 	if (size > log_buf_len)
1040 		new_log_buf_len = (unsigned long)size;
1041 }
1042 
1043 /* save requested log_buf_len since it's too early to process it */
log_buf_len_setup(char * str)1044 static int __init log_buf_len_setup(char *str)
1045 {
1046 	u64 size;
1047 
1048 	if (!str)
1049 		return -EINVAL;
1050 
1051 	size = memparse(str, &str);
1052 
1053 	log_buf_len_update(size);
1054 
1055 	return 0;
1056 }
1057 early_param("log_buf_len", log_buf_len_setup);
1058 
1059 #ifdef CONFIG_SMP
1060 #define __LOG_CPU_MAX_BUF_LEN (1 << CONFIG_LOG_CPU_MAX_BUF_SHIFT)
1061 
log_buf_add_cpu(void)1062 static void __init log_buf_add_cpu(void)
1063 {
1064 	unsigned int cpu_extra;
1065 
1066 	/*
1067 	 * archs should set up cpu_possible_bits properly with
1068 	 * set_cpu_possible() after setup_arch() but just in
1069 	 * case lets ensure this is valid.
1070 	 */
1071 	if (num_possible_cpus() == 1)
1072 		return;
1073 
1074 	cpu_extra = (num_possible_cpus() - 1) * __LOG_CPU_MAX_BUF_LEN;
1075 
1076 	/* by default this will only continue through for large > 64 CPUs */
1077 	if (cpu_extra <= __LOG_BUF_LEN / 2)
1078 		return;
1079 
1080 	pr_info("log_buf_len individual max cpu contribution: %d bytes\n",
1081 		__LOG_CPU_MAX_BUF_LEN);
1082 	pr_info("log_buf_len total cpu_extra contributions: %d bytes\n",
1083 		cpu_extra);
1084 	pr_info("log_buf_len min size: %d bytes\n", __LOG_BUF_LEN);
1085 
1086 	log_buf_len_update(cpu_extra + __LOG_BUF_LEN);
1087 }
1088 #else /* !CONFIG_SMP */
log_buf_add_cpu(void)1089 static inline void log_buf_add_cpu(void) {}
1090 #endif /* CONFIG_SMP */
1091 
set_percpu_data_ready(void)1092 static void __init set_percpu_data_ready(void)
1093 {
1094 	__printk_percpu_data_ready = true;
1095 }
1096 
add_to_rb(struct printk_ringbuffer * rb,struct printk_record * r)1097 static unsigned int __init add_to_rb(struct printk_ringbuffer *rb,
1098 				     struct printk_record *r)
1099 {
1100 	struct prb_reserved_entry e;
1101 	struct printk_record dest_r;
1102 
1103 	prb_rec_init_wr(&dest_r, r->info->text_len);
1104 
1105 	if (!prb_reserve(&e, rb, &dest_r))
1106 		return 0;
1107 
1108 	memcpy(&dest_r.text_buf[0], &r->text_buf[0], r->info->text_len);
1109 	dest_r.info->text_len = r->info->text_len;
1110 	dest_r.info->facility = r->info->facility;
1111 	dest_r.info->level = r->info->level;
1112 	dest_r.info->flags = r->info->flags;
1113 	dest_r.info->ts_nsec = r->info->ts_nsec;
1114 	dest_r.info->caller_id = r->info->caller_id;
1115 	memcpy(&dest_r.info->dev_info, &r->info->dev_info, sizeof(dest_r.info->dev_info));
1116 
1117 	prb_final_commit(&e);
1118 
1119 	return prb_record_text_space(&e);
1120 }
1121 
1122 static char setup_text_buf[PRINTKRB_RECORD_MAX] __initdata;
1123 
setup_log_buf(int early)1124 void __init setup_log_buf(int early)
1125 {
1126 	struct printk_info *new_infos;
1127 	unsigned int new_descs_count;
1128 	struct prb_desc *new_descs;
1129 	struct printk_info info;
1130 	struct printk_record r;
1131 	unsigned int text_size;
1132 	size_t new_descs_size;
1133 	size_t new_infos_size;
1134 	unsigned long flags;
1135 	char *new_log_buf;
1136 	unsigned int free;
1137 	u64 seq;
1138 
1139 	/*
1140 	 * Some archs call setup_log_buf() multiple times - first is very
1141 	 * early, e.g. from setup_arch(), and second - when percpu_areas
1142 	 * are initialised.
1143 	 */
1144 	if (!early)
1145 		set_percpu_data_ready();
1146 
1147 	if (log_buf != __log_buf)
1148 		return;
1149 
1150 	if (!early && !new_log_buf_len)
1151 		log_buf_add_cpu();
1152 
1153 	if (!new_log_buf_len)
1154 		return;
1155 
1156 	new_descs_count = new_log_buf_len >> PRB_AVGBITS;
1157 	if (new_descs_count == 0) {
1158 		pr_err("new_log_buf_len: %lu too small\n", new_log_buf_len);
1159 		return;
1160 	}
1161 
1162 	new_log_buf = memblock_alloc(new_log_buf_len, LOG_ALIGN);
1163 	if (unlikely(!new_log_buf)) {
1164 		pr_err("log_buf_len: %lu text bytes not available\n",
1165 		       new_log_buf_len);
1166 		return;
1167 	}
1168 
1169 	new_descs_size = new_descs_count * sizeof(struct prb_desc);
1170 	new_descs = memblock_alloc(new_descs_size, LOG_ALIGN);
1171 	if (unlikely(!new_descs)) {
1172 		pr_err("log_buf_len: %zu desc bytes not available\n",
1173 		       new_descs_size);
1174 		goto err_free_log_buf;
1175 	}
1176 
1177 	new_infos_size = new_descs_count * sizeof(struct printk_info);
1178 	new_infos = memblock_alloc(new_infos_size, LOG_ALIGN);
1179 	if (unlikely(!new_infos)) {
1180 		pr_err("log_buf_len: %zu info bytes not available\n",
1181 		       new_infos_size);
1182 		goto err_free_descs;
1183 	}
1184 
1185 	prb_rec_init_rd(&r, &info, &setup_text_buf[0], sizeof(setup_text_buf));
1186 
1187 	prb_init(&printk_rb_dynamic,
1188 		 new_log_buf, ilog2(new_log_buf_len),
1189 		 new_descs, ilog2(new_descs_count),
1190 		 new_infos);
1191 
1192 	local_irq_save(flags);
1193 
1194 	log_buf_len = new_log_buf_len;
1195 	log_buf = new_log_buf;
1196 	new_log_buf_len = 0;
1197 
1198 	free = __LOG_BUF_LEN;
1199 	prb_for_each_record(0, &printk_rb_static, seq, &r) {
1200 		text_size = add_to_rb(&printk_rb_dynamic, &r);
1201 		if (text_size > free)
1202 			free = 0;
1203 		else
1204 			free -= text_size;
1205 	}
1206 
1207 	prb = &printk_rb_dynamic;
1208 
1209 	local_irq_restore(flags);
1210 
1211 	/*
1212 	 * Copy any remaining messages that might have appeared from
1213 	 * NMI context after copying but before switching to the
1214 	 * dynamic buffer.
1215 	 */
1216 	prb_for_each_record(seq, &printk_rb_static, seq, &r) {
1217 		text_size = add_to_rb(&printk_rb_dynamic, &r);
1218 		if (text_size > free)
1219 			free = 0;
1220 		else
1221 			free -= text_size;
1222 	}
1223 
1224 	if (seq != prb_next_seq(&printk_rb_static)) {
1225 		pr_err("dropped %llu messages\n",
1226 		       prb_next_seq(&printk_rb_static) - seq);
1227 	}
1228 
1229 	pr_info("log_buf_len: %u bytes\n", log_buf_len);
1230 	pr_info("early log buf free: %u(%u%%)\n",
1231 		free, (free * 100) / __LOG_BUF_LEN);
1232 	return;
1233 
1234 err_free_descs:
1235 	memblock_free(new_descs, new_descs_size);
1236 err_free_log_buf:
1237 	memblock_free(new_log_buf, new_log_buf_len);
1238 }
1239 
1240 static bool __read_mostly ignore_loglevel;
1241 
ignore_loglevel_setup(char * str)1242 static int __init ignore_loglevel_setup(char *str)
1243 {
1244 	ignore_loglevel = true;
1245 	pr_info("debug: ignoring loglevel setting.\n");
1246 
1247 	return 0;
1248 }
1249 
1250 early_param("ignore_loglevel", ignore_loglevel_setup);
1251 module_param(ignore_loglevel, bool, S_IRUGO | S_IWUSR);
1252 MODULE_PARM_DESC(ignore_loglevel,
1253 		 "ignore loglevel setting (prints all kernel messages to the console)");
1254 
suppress_message_printing(int level)1255 static bool suppress_message_printing(int level)
1256 {
1257 	return (level >= console_loglevel && !ignore_loglevel);
1258 }
1259 
1260 #ifdef CONFIG_BOOT_PRINTK_DELAY
1261 
1262 static int boot_delay; /* msecs delay after each printk during bootup */
1263 static unsigned long long loops_per_msec;	/* based on boot_delay */
1264 
boot_delay_setup(char * str)1265 static int __init boot_delay_setup(char *str)
1266 {
1267 	unsigned long lpj;
1268 
1269 	lpj = preset_lpj ? preset_lpj : 1000000;	/* some guess */
1270 	loops_per_msec = (unsigned long long)lpj / 1000 * HZ;
1271 
1272 	get_option(&str, &boot_delay);
1273 	if (boot_delay > 10 * 1000)
1274 		boot_delay = 0;
1275 
1276 	pr_debug("boot_delay: %u, preset_lpj: %ld, lpj: %lu, "
1277 		"HZ: %d, loops_per_msec: %llu\n",
1278 		boot_delay, preset_lpj, lpj, HZ, loops_per_msec);
1279 	return 0;
1280 }
1281 early_param("boot_delay", boot_delay_setup);
1282 
boot_delay_msec(int level)1283 static void boot_delay_msec(int level)
1284 {
1285 	unsigned long long k;
1286 	unsigned long timeout;
1287 
1288 	if ((boot_delay == 0 || system_state >= SYSTEM_RUNNING)
1289 		|| suppress_message_printing(level)) {
1290 		return;
1291 	}
1292 
1293 	k = (unsigned long long)loops_per_msec * boot_delay;
1294 
1295 	timeout = jiffies + msecs_to_jiffies(boot_delay);
1296 	while (k) {
1297 		k--;
1298 		cpu_relax();
1299 		/*
1300 		 * use (volatile) jiffies to prevent
1301 		 * compiler reduction; loop termination via jiffies
1302 		 * is secondary and may or may not happen.
1303 		 */
1304 		if (time_after(jiffies, timeout))
1305 			break;
1306 		touch_nmi_watchdog();
1307 	}
1308 }
1309 #else
boot_delay_msec(int level)1310 static inline void boot_delay_msec(int level)
1311 {
1312 }
1313 #endif
1314 
1315 static bool printk_time = IS_ENABLED(CONFIG_PRINTK_TIME);
1316 module_param_named(time, printk_time, bool, S_IRUGO | S_IWUSR);
1317 
print_syslog(unsigned int level,char * buf)1318 static size_t print_syslog(unsigned int level, char *buf)
1319 {
1320 	return sprintf(buf, "<%u>", level);
1321 }
1322 
print_time(u64 ts,char * buf)1323 static size_t print_time(u64 ts, char *buf)
1324 {
1325 	unsigned long rem_nsec = do_div(ts, 1000000000);
1326 
1327 	return sprintf(buf, "[%5lu.%06lu]",
1328 		       (unsigned long)ts, rem_nsec / 1000);
1329 }
1330 
1331 #ifdef CONFIG_PRINTK_CALLER
print_caller(u32 id,char * buf)1332 static size_t print_caller(u32 id, char *buf)
1333 {
1334 	char caller[12];
1335 
1336 	snprintf(caller, sizeof(caller), "%c%u",
1337 		 id & 0x80000000 ? 'C' : 'T', id & ~0x80000000);
1338 	return sprintf(buf, "[%6s]", caller);
1339 }
1340 #else
1341 #define print_caller(id, buf) 0
1342 #endif
1343 
info_print_prefix(const struct printk_info * info,bool syslog,bool time,char * buf)1344 static size_t info_print_prefix(const struct printk_info  *info, bool syslog,
1345 				bool time, char *buf)
1346 {
1347 	size_t len = 0;
1348 
1349 	if (syslog)
1350 		len = print_syslog((info->facility << 3) | info->level, buf);
1351 
1352 	if (time)
1353 		len += print_time(info->ts_nsec, buf + len);
1354 
1355 	len += print_caller(info->caller_id, buf + len);
1356 
1357 	if (IS_ENABLED(CONFIG_PRINTK_CALLER) || time) {
1358 		buf[len++] = ' ';
1359 		buf[len] = '\0';
1360 	}
1361 
1362 	return len;
1363 }
1364 
1365 /*
1366  * Prepare the record for printing. The text is shifted within the given
1367  * buffer to avoid a need for another one. The following operations are
1368  * done:
1369  *
1370  *   - Add prefix for each line.
1371  *   - Drop truncated lines that no longer fit into the buffer.
1372  *   - Add the trailing newline that has been removed in vprintk_store().
1373  *   - Add a string terminator.
1374  *
1375  * Since the produced string is always terminated, the maximum possible
1376  * return value is @r->text_buf_size - 1;
1377  *
1378  * Return: The length of the updated/prepared text, including the added
1379  * prefixes and the newline. The terminator is not counted. The dropped
1380  * line(s) are not counted.
1381  */
record_print_text(struct printk_record * r,bool syslog,bool time)1382 static size_t record_print_text(struct printk_record *r, bool syslog,
1383 				bool time)
1384 {
1385 	size_t text_len = r->info->text_len;
1386 	size_t buf_size = r->text_buf_size;
1387 	char *text = r->text_buf;
1388 	char prefix[PRINTK_PREFIX_MAX];
1389 	bool truncated = false;
1390 	size_t prefix_len;
1391 	size_t line_len;
1392 	size_t len = 0;
1393 	char *next;
1394 
1395 	/*
1396 	 * If the message was truncated because the buffer was not large
1397 	 * enough, treat the available text as if it were the full text.
1398 	 */
1399 	if (text_len > buf_size)
1400 		text_len = buf_size;
1401 
1402 	prefix_len = info_print_prefix(r->info, syslog, time, prefix);
1403 
1404 	/*
1405 	 * @text_len: bytes of unprocessed text
1406 	 * @line_len: bytes of current line _without_ newline
1407 	 * @text:     pointer to beginning of current line
1408 	 * @len:      number of bytes prepared in r->text_buf
1409 	 */
1410 	for (;;) {
1411 		next = memchr(text, '\n', text_len);
1412 		if (next) {
1413 			line_len = next - text;
1414 		} else {
1415 			/* Drop truncated line(s). */
1416 			if (truncated)
1417 				break;
1418 			line_len = text_len;
1419 		}
1420 
1421 		/*
1422 		 * Truncate the text if there is not enough space to add the
1423 		 * prefix and a trailing newline and a terminator.
1424 		 */
1425 		if (len + prefix_len + text_len + 1 + 1 > buf_size) {
1426 			/* Drop even the current line if no space. */
1427 			if (len + prefix_len + line_len + 1 + 1 > buf_size)
1428 				break;
1429 
1430 			text_len = buf_size - len - prefix_len - 1 - 1;
1431 			truncated = true;
1432 		}
1433 
1434 		memmove(text + prefix_len, text, text_len);
1435 		memcpy(text, prefix, prefix_len);
1436 
1437 		/*
1438 		 * Increment the prepared length to include the text and
1439 		 * prefix that were just moved+copied. Also increment for the
1440 		 * newline at the end of this line. If this is the last line,
1441 		 * there is no newline, but it will be added immediately below.
1442 		 */
1443 		len += prefix_len + line_len + 1;
1444 		if (text_len == line_len) {
1445 			/*
1446 			 * This is the last line. Add the trailing newline
1447 			 * removed in vprintk_store().
1448 			 */
1449 			text[prefix_len + line_len] = '\n';
1450 			break;
1451 		}
1452 
1453 		/*
1454 		 * Advance beyond the added prefix and the related line with
1455 		 * its newline.
1456 		 */
1457 		text += prefix_len + line_len + 1;
1458 
1459 		/*
1460 		 * The remaining text has only decreased by the line with its
1461 		 * newline.
1462 		 *
1463 		 * Note that @text_len can become zero. It happens when @text
1464 		 * ended with a newline (either due to truncation or the
1465 		 * original string ending with "\n\n"). The loop is correctly
1466 		 * repeated and (if not truncated) an empty line with a prefix
1467 		 * will be prepared.
1468 		 */
1469 		text_len -= line_len + 1;
1470 	}
1471 
1472 	/*
1473 	 * If a buffer was provided, it will be terminated. Space for the
1474 	 * string terminator is guaranteed to be available. The terminator is
1475 	 * not counted in the return value.
1476 	 */
1477 	if (buf_size > 0)
1478 		r->text_buf[len] = 0;
1479 
1480 	return len;
1481 }
1482 
get_record_print_text_size(struct printk_info * info,unsigned int line_count,bool syslog,bool time)1483 static size_t get_record_print_text_size(struct printk_info *info,
1484 					 unsigned int line_count,
1485 					 bool syslog, bool time)
1486 {
1487 	char prefix[PRINTK_PREFIX_MAX];
1488 	size_t prefix_len;
1489 
1490 	prefix_len = info_print_prefix(info, syslog, time, prefix);
1491 
1492 	/*
1493 	 * Each line will be preceded with a prefix. The intermediate
1494 	 * newlines are already within the text, but a final trailing
1495 	 * newline will be added.
1496 	 */
1497 	return ((prefix_len * line_count) + info->text_len + 1);
1498 }
1499 
1500 /*
1501  * Beginning with @start_seq, find the first record where it and all following
1502  * records up to (but not including) @max_seq fit into @size.
1503  *
1504  * @max_seq is simply an upper bound and does not need to exist. If the caller
1505  * does not require an upper bound, -1 can be used for @max_seq.
1506  */
find_first_fitting_seq(u64 start_seq,u64 max_seq,size_t size,bool syslog,bool time)1507 static u64 find_first_fitting_seq(u64 start_seq, u64 max_seq, size_t size,
1508 				  bool syslog, bool time)
1509 {
1510 	struct printk_info info;
1511 	unsigned int line_count;
1512 	size_t len = 0;
1513 	u64 seq;
1514 
1515 	/* Determine the size of the records up to @max_seq. */
1516 	prb_for_each_info(start_seq, prb, seq, &info, &line_count) {
1517 		if (info.seq >= max_seq)
1518 			break;
1519 		len += get_record_print_text_size(&info, line_count, syslog, time);
1520 	}
1521 
1522 	/*
1523 	 * Adjust the upper bound for the next loop to avoid subtracting
1524 	 * lengths that were never added.
1525 	 */
1526 	if (seq < max_seq)
1527 		max_seq = seq;
1528 
1529 	/*
1530 	 * Move first record forward until length fits into the buffer. Ignore
1531 	 * newest messages that were not counted in the above cycle. Messages
1532 	 * might appear and get lost in the meantime. This is a best effort
1533 	 * that prevents an infinite loop that could occur with a retry.
1534 	 */
1535 	prb_for_each_info(start_seq, prb, seq, &info, &line_count) {
1536 		if (len <= size || info.seq >= max_seq)
1537 			break;
1538 		len -= get_record_print_text_size(&info, line_count, syslog, time);
1539 	}
1540 
1541 	return seq;
1542 }
1543 
1544 /* The caller is responsible for making sure @size is greater than 0. */
syslog_print(char __user * buf,int size)1545 static int syslog_print(char __user *buf, int size)
1546 {
1547 	struct printk_info info;
1548 	struct printk_record r;
1549 	char *text;
1550 	int len = 0;
1551 	u64 seq;
1552 
1553 	text = kmalloc(PRINTK_MESSAGE_MAX, GFP_KERNEL);
1554 	if (!text)
1555 		return -ENOMEM;
1556 
1557 	prb_rec_init_rd(&r, &info, text, PRINTK_MESSAGE_MAX);
1558 
1559 	mutex_lock(&syslog_lock);
1560 
1561 	/*
1562 	 * Wait for the @syslog_seq record to be available. @syslog_seq may
1563 	 * change while waiting.
1564 	 */
1565 	do {
1566 		seq = syslog_seq;
1567 
1568 		mutex_unlock(&syslog_lock);
1569 		/*
1570 		 * Guarantee this task is visible on the waitqueue before
1571 		 * checking the wake condition.
1572 		 *
1573 		 * The full memory barrier within set_current_state() of
1574 		 * prepare_to_wait_event() pairs with the full memory barrier
1575 		 * within wq_has_sleeper().
1576 		 *
1577 		 * This pairs with __wake_up_klogd:A.
1578 		 */
1579 		len = wait_event_interruptible(log_wait,
1580 				prb_read_valid(prb, seq, NULL)); /* LMM(syslog_print:A) */
1581 		mutex_lock(&syslog_lock);
1582 
1583 		if (len)
1584 			goto out;
1585 	} while (syslog_seq != seq);
1586 
1587 	/*
1588 	 * Copy records that fit into the buffer. The above cycle makes sure
1589 	 * that the first record is always available.
1590 	 */
1591 	do {
1592 		size_t n;
1593 		size_t skip;
1594 		int err;
1595 
1596 		if (!prb_read_valid(prb, syslog_seq, &r))
1597 			break;
1598 
1599 		if (r.info->seq != syslog_seq) {
1600 			/* message is gone, move to next valid one */
1601 			syslog_seq = r.info->seq;
1602 			syslog_partial = 0;
1603 		}
1604 
1605 		/*
1606 		 * To keep reading/counting partial line consistent,
1607 		 * use printk_time value as of the beginning of a line.
1608 		 */
1609 		if (!syslog_partial)
1610 			syslog_time = printk_time;
1611 
1612 		skip = syslog_partial;
1613 		n = record_print_text(&r, true, syslog_time);
1614 		if (n - syslog_partial <= size) {
1615 			/* message fits into buffer, move forward */
1616 			syslog_seq = r.info->seq + 1;
1617 			n -= syslog_partial;
1618 			syslog_partial = 0;
1619 		} else if (!len){
1620 			/* partial read(), remember position */
1621 			n = size;
1622 			syslog_partial += n;
1623 		} else
1624 			n = 0;
1625 
1626 		if (!n)
1627 			break;
1628 
1629 		mutex_unlock(&syslog_lock);
1630 		err = copy_to_user(buf, text + skip, n);
1631 		mutex_lock(&syslog_lock);
1632 
1633 		if (err) {
1634 			if (!len)
1635 				len = -EFAULT;
1636 			break;
1637 		}
1638 
1639 		len += n;
1640 		size -= n;
1641 		buf += n;
1642 	} while (size);
1643 out:
1644 	mutex_unlock(&syslog_lock);
1645 	kfree(text);
1646 	return len;
1647 }
1648 
syslog_print_all(char __user * buf,int size,bool clear)1649 static int syslog_print_all(char __user *buf, int size, bool clear)
1650 {
1651 	struct printk_info info;
1652 	struct printk_record r;
1653 	char *text;
1654 	int len = 0;
1655 	u64 seq;
1656 	bool time;
1657 
1658 	text = kmalloc(PRINTK_MESSAGE_MAX, GFP_KERNEL);
1659 	if (!text)
1660 		return -ENOMEM;
1661 
1662 	time = printk_time;
1663 	/*
1664 	 * Find first record that fits, including all following records,
1665 	 * into the user-provided buffer for this dump.
1666 	 */
1667 	seq = find_first_fitting_seq(latched_seq_read_nolock(&clear_seq), -1,
1668 				     size, true, time);
1669 
1670 	prb_rec_init_rd(&r, &info, text, PRINTK_MESSAGE_MAX);
1671 
1672 	len = 0;
1673 	prb_for_each_record(seq, prb, seq, &r) {
1674 		int textlen;
1675 
1676 		textlen = record_print_text(&r, true, time);
1677 
1678 		if (len + textlen > size) {
1679 			seq--;
1680 			break;
1681 		}
1682 
1683 		if (copy_to_user(buf + len, text, textlen))
1684 			len = -EFAULT;
1685 		else
1686 			len += textlen;
1687 
1688 		if (len < 0)
1689 			break;
1690 	}
1691 
1692 	if (clear) {
1693 		mutex_lock(&syslog_lock);
1694 		latched_seq_write(&clear_seq, seq);
1695 		mutex_unlock(&syslog_lock);
1696 	}
1697 
1698 	kfree(text);
1699 	return len;
1700 }
1701 
syslog_clear(void)1702 static void syslog_clear(void)
1703 {
1704 	mutex_lock(&syslog_lock);
1705 	latched_seq_write(&clear_seq, prb_next_seq(prb));
1706 	mutex_unlock(&syslog_lock);
1707 }
1708 
do_syslog(int type,char __user * buf,int len,int source)1709 int do_syslog(int type, char __user *buf, int len, int source)
1710 {
1711 	struct printk_info info;
1712 	bool clear = false;
1713 	static int saved_console_loglevel = LOGLEVEL_DEFAULT;
1714 	int error;
1715 
1716 	error = check_syslog_permissions(type, source);
1717 	if (error)
1718 		return error;
1719 
1720 	switch (type) {
1721 	case SYSLOG_ACTION_CLOSE:	/* Close log */
1722 		break;
1723 	case SYSLOG_ACTION_OPEN:	/* Open log */
1724 		break;
1725 	case SYSLOG_ACTION_READ:	/* Read from log */
1726 		if (!buf || len < 0)
1727 			return -EINVAL;
1728 		if (!len)
1729 			return 0;
1730 		if (!access_ok(buf, len))
1731 			return -EFAULT;
1732 		error = syslog_print(buf, len);
1733 		break;
1734 	/* Read/clear last kernel messages */
1735 	case SYSLOG_ACTION_READ_CLEAR:
1736 		clear = true;
1737 		fallthrough;
1738 	/* Read last kernel messages */
1739 	case SYSLOG_ACTION_READ_ALL:
1740 		if (!buf || len < 0)
1741 			return -EINVAL;
1742 		if (!len)
1743 			return 0;
1744 		if (!access_ok(buf, len))
1745 			return -EFAULT;
1746 		error = syslog_print_all(buf, len, clear);
1747 		break;
1748 	/* Clear ring buffer */
1749 	case SYSLOG_ACTION_CLEAR:
1750 		syslog_clear();
1751 		break;
1752 	/* Disable logging to console */
1753 	case SYSLOG_ACTION_CONSOLE_OFF:
1754 		if (saved_console_loglevel == LOGLEVEL_DEFAULT)
1755 			saved_console_loglevel = console_loglevel;
1756 		console_loglevel = minimum_console_loglevel;
1757 		break;
1758 	/* Enable logging to console */
1759 	case SYSLOG_ACTION_CONSOLE_ON:
1760 		if (saved_console_loglevel != LOGLEVEL_DEFAULT) {
1761 			console_loglevel = saved_console_loglevel;
1762 			saved_console_loglevel = LOGLEVEL_DEFAULT;
1763 		}
1764 		break;
1765 	/* Set level of messages printed to console */
1766 	case SYSLOG_ACTION_CONSOLE_LEVEL:
1767 		if (len < 1 || len > 8)
1768 			return -EINVAL;
1769 		if (len < minimum_console_loglevel)
1770 			len = minimum_console_loglevel;
1771 		console_loglevel = len;
1772 		/* Implicitly re-enable logging to console */
1773 		saved_console_loglevel = LOGLEVEL_DEFAULT;
1774 		break;
1775 	/* Number of chars in the log buffer */
1776 	case SYSLOG_ACTION_SIZE_UNREAD:
1777 		mutex_lock(&syslog_lock);
1778 		if (!prb_read_valid_info(prb, syslog_seq, &info, NULL)) {
1779 			/* No unread messages. */
1780 			mutex_unlock(&syslog_lock);
1781 			return 0;
1782 		}
1783 		if (info.seq != syslog_seq) {
1784 			/* messages are gone, move to first one */
1785 			syslog_seq = info.seq;
1786 			syslog_partial = 0;
1787 		}
1788 		if (source == SYSLOG_FROM_PROC) {
1789 			/*
1790 			 * Short-cut for poll(/"proc/kmsg") which simply checks
1791 			 * for pending data, not the size; return the count of
1792 			 * records, not the length.
1793 			 */
1794 			error = prb_next_seq(prb) - syslog_seq;
1795 		} else {
1796 			bool time = syslog_partial ? syslog_time : printk_time;
1797 			unsigned int line_count;
1798 			u64 seq;
1799 
1800 			prb_for_each_info(syslog_seq, prb, seq, &info,
1801 					  &line_count) {
1802 				error += get_record_print_text_size(&info, line_count,
1803 								    true, time);
1804 				time = printk_time;
1805 			}
1806 			error -= syslog_partial;
1807 		}
1808 		mutex_unlock(&syslog_lock);
1809 		break;
1810 	/* Size of the log buffer */
1811 	case SYSLOG_ACTION_SIZE_BUFFER:
1812 		error = log_buf_len;
1813 		break;
1814 	default:
1815 		error = -EINVAL;
1816 		break;
1817 	}
1818 
1819 	return error;
1820 }
1821 
SYSCALL_DEFINE3(syslog,int,type,char __user *,buf,int,len)1822 SYSCALL_DEFINE3(syslog, int, type, char __user *, buf, int, len)
1823 {
1824 	return do_syslog(type, buf, len, SYSLOG_FROM_READER);
1825 }
1826 
1827 /*
1828  * Special console_lock variants that help to reduce the risk of soft-lockups.
1829  * They allow to pass console_lock to another printk() call using a busy wait.
1830  */
1831 
1832 #ifdef CONFIG_LOCKDEP
1833 static struct lockdep_map console_owner_dep_map = {
1834 	.name = "console_owner"
1835 };
1836 #endif
1837 
1838 static DEFINE_RAW_SPINLOCK(console_owner_lock);
1839 static struct task_struct *console_owner;
1840 static bool console_waiter;
1841 
1842 /**
1843  * console_lock_spinning_enable - mark beginning of code where another
1844  *	thread might safely busy wait
1845  *
1846  * This basically converts console_lock into a spinlock. This marks
1847  * the section where the console_lock owner can not sleep, because
1848  * there may be a waiter spinning (like a spinlock). Also it must be
1849  * ready to hand over the lock at the end of the section.
1850  */
console_lock_spinning_enable(void)1851 static void console_lock_spinning_enable(void)
1852 {
1853 	raw_spin_lock(&console_owner_lock);
1854 	console_owner = current;
1855 	raw_spin_unlock(&console_owner_lock);
1856 
1857 	/* The waiter may spin on us after setting console_owner */
1858 	spin_acquire(&console_owner_dep_map, 0, 0, _THIS_IP_);
1859 }
1860 
1861 /**
1862  * console_lock_spinning_disable_and_check - mark end of code where another
1863  *	thread was able to busy wait and check if there is a waiter
1864  * @cookie: cookie returned from console_srcu_read_lock()
1865  *
1866  * This is called at the end of the section where spinning is allowed.
1867  * It has two functions. First, it is a signal that it is no longer
1868  * safe to start busy waiting for the lock. Second, it checks if
1869  * there is a busy waiter and passes the lock rights to her.
1870  *
1871  * Important: Callers lose both the console_lock and the SRCU read lock if
1872  *	there was a busy waiter. They must not touch items synchronized by
1873  *	console_lock or SRCU read lock in this case.
1874  *
1875  * Return: 1 if the lock rights were passed, 0 otherwise.
1876  */
console_lock_spinning_disable_and_check(int cookie)1877 static int console_lock_spinning_disable_and_check(int cookie)
1878 {
1879 	int waiter;
1880 
1881 	raw_spin_lock(&console_owner_lock);
1882 	waiter = READ_ONCE(console_waiter);
1883 	console_owner = NULL;
1884 	raw_spin_unlock(&console_owner_lock);
1885 
1886 	if (!waiter) {
1887 		spin_release(&console_owner_dep_map, _THIS_IP_);
1888 		return 0;
1889 	}
1890 
1891 	/* The waiter is now free to continue */
1892 	WRITE_ONCE(console_waiter, false);
1893 
1894 	spin_release(&console_owner_dep_map, _THIS_IP_);
1895 
1896 	/*
1897 	 * Preserve lockdep lock ordering. Release the SRCU read lock before
1898 	 * releasing the console_lock.
1899 	 */
1900 	console_srcu_read_unlock(cookie);
1901 
1902 	/*
1903 	 * Hand off console_lock to waiter. The waiter will perform
1904 	 * the up(). After this, the waiter is the console_lock owner.
1905 	 */
1906 	mutex_release(&console_lock_dep_map, _THIS_IP_);
1907 	return 1;
1908 }
1909 
1910 /**
1911  * console_trylock_spinning - try to get console_lock by busy waiting
1912  *
1913  * This allows to busy wait for the console_lock when the current
1914  * owner is running in specially marked sections. It means that
1915  * the current owner is running and cannot reschedule until it
1916  * is ready to lose the lock.
1917  *
1918  * Return: 1 if we got the lock, 0 othrewise
1919  */
console_trylock_spinning(void)1920 static int console_trylock_spinning(void)
1921 {
1922 	struct task_struct *owner = NULL;
1923 	bool waiter;
1924 	bool spin = false;
1925 	unsigned long flags;
1926 
1927 	if (console_trylock())
1928 		return 1;
1929 
1930 	/*
1931 	 * It's unsafe to spin once a panic has begun. If we are the
1932 	 * panic CPU, we may have already halted the owner of the
1933 	 * console_sem. If we are not the panic CPU, then we should
1934 	 * avoid taking console_sem, so the panic CPU has a better
1935 	 * chance of cleanly acquiring it later.
1936 	 */
1937 	if (panic_in_progress())
1938 		return 0;
1939 
1940 	printk_safe_enter_irqsave(flags);
1941 
1942 	raw_spin_lock(&console_owner_lock);
1943 	owner = READ_ONCE(console_owner);
1944 	waiter = READ_ONCE(console_waiter);
1945 	if (!waiter && owner && owner != current) {
1946 		WRITE_ONCE(console_waiter, true);
1947 		spin = true;
1948 	}
1949 	raw_spin_unlock(&console_owner_lock);
1950 
1951 	/*
1952 	 * If there is an active printk() writing to the
1953 	 * consoles, instead of having it write our data too,
1954 	 * see if we can offload that load from the active
1955 	 * printer, and do some printing ourselves.
1956 	 * Go into a spin only if there isn't already a waiter
1957 	 * spinning, and there is an active printer, and
1958 	 * that active printer isn't us (recursive printk?).
1959 	 */
1960 	if (!spin) {
1961 		printk_safe_exit_irqrestore(flags);
1962 		return 0;
1963 	}
1964 
1965 	/* We spin waiting for the owner to release us */
1966 	spin_acquire(&console_owner_dep_map, 0, 0, _THIS_IP_);
1967 	/* Owner will clear console_waiter on hand off */
1968 	while (READ_ONCE(console_waiter))
1969 		cpu_relax();
1970 	spin_release(&console_owner_dep_map, _THIS_IP_);
1971 
1972 	printk_safe_exit_irqrestore(flags);
1973 	/*
1974 	 * The owner passed the console lock to us.
1975 	 * Since we did not spin on console lock, annotate
1976 	 * this as a trylock. Otherwise lockdep will
1977 	 * complain.
1978 	 */
1979 	mutex_acquire(&console_lock_dep_map, 0, 1, _THIS_IP_);
1980 
1981 	return 1;
1982 }
1983 
1984 /*
1985  * Recursion is tracked separately on each CPU. If NMIs are supported, an
1986  * additional NMI context per CPU is also separately tracked. Until per-CPU
1987  * is available, a separate "early tracking" is performed.
1988  */
1989 static DEFINE_PER_CPU(u8, printk_count);
1990 static u8 printk_count_early;
1991 #ifdef CONFIG_HAVE_NMI
1992 static DEFINE_PER_CPU(u8, printk_count_nmi);
1993 static u8 printk_count_nmi_early;
1994 #endif
1995 
1996 /*
1997  * Recursion is limited to keep the output sane. printk() should not require
1998  * more than 1 level of recursion (allowing, for example, printk() to trigger
1999  * a WARN), but a higher value is used in case some printk-internal errors
2000  * exist, such as the ringbuffer validation checks failing.
2001  */
2002 #define PRINTK_MAX_RECURSION 3
2003 
2004 /*
2005  * Return a pointer to the dedicated counter for the CPU+context of the
2006  * caller.
2007  */
__printk_recursion_counter(void)2008 static u8 *__printk_recursion_counter(void)
2009 {
2010 #ifdef CONFIG_HAVE_NMI
2011 	if (in_nmi()) {
2012 		if (printk_percpu_data_ready())
2013 			return this_cpu_ptr(&printk_count_nmi);
2014 		return &printk_count_nmi_early;
2015 	}
2016 #endif
2017 	if (printk_percpu_data_ready())
2018 		return this_cpu_ptr(&printk_count);
2019 	return &printk_count_early;
2020 }
2021 
2022 /*
2023  * Enter recursion tracking. Interrupts are disabled to simplify tracking.
2024  * The caller must check the boolean return value to see if the recursion is
2025  * allowed. On failure, interrupts are not disabled.
2026  *
2027  * @recursion_ptr must be a variable of type (u8 *) and is the same variable
2028  * that is passed to printk_exit_irqrestore().
2029  */
2030 #define printk_enter_irqsave(recursion_ptr, flags)	\
2031 ({							\
2032 	bool success = true;				\
2033 							\
2034 	typecheck(u8 *, recursion_ptr);			\
2035 	local_irq_save(flags);				\
2036 	(recursion_ptr) = __printk_recursion_counter();	\
2037 	if (*(recursion_ptr) > PRINTK_MAX_RECURSION) {	\
2038 		local_irq_restore(flags);		\
2039 		success = false;			\
2040 	} else {					\
2041 		(*(recursion_ptr))++;			\
2042 	}						\
2043 	success;					\
2044 })
2045 
2046 /* Exit recursion tracking, restoring interrupts. */
2047 #define printk_exit_irqrestore(recursion_ptr, flags)	\
2048 	do {						\
2049 		typecheck(u8 *, recursion_ptr);		\
2050 		(*(recursion_ptr))--;			\
2051 		local_irq_restore(flags);		\
2052 	} while (0)
2053 
2054 int printk_delay_msec __read_mostly;
2055 
printk_delay(int level)2056 static inline void printk_delay(int level)
2057 {
2058 	boot_delay_msec(level);
2059 
2060 	if (unlikely(printk_delay_msec)) {
2061 		int m = printk_delay_msec;
2062 
2063 		while (m--) {
2064 			mdelay(1);
2065 			touch_nmi_watchdog();
2066 		}
2067 	}
2068 }
2069 
printk_caller_id(void)2070 static inline u32 printk_caller_id(void)
2071 {
2072 	return in_task() ? task_pid_nr(current) :
2073 		0x80000000 + smp_processor_id();
2074 }
2075 
2076 /**
2077  * printk_parse_prefix - Parse level and control flags.
2078  *
2079  * @text:     The terminated text message.
2080  * @level:    A pointer to the current level value, will be updated.
2081  * @flags:    A pointer to the current printk_info flags, will be updated.
2082  *
2083  * @level may be NULL if the caller is not interested in the parsed value.
2084  * Otherwise the variable pointed to by @level must be set to
2085  * LOGLEVEL_DEFAULT in order to be updated with the parsed value.
2086  *
2087  * @flags may be NULL if the caller is not interested in the parsed value.
2088  * Otherwise the variable pointed to by @flags will be OR'd with the parsed
2089  * value.
2090  *
2091  * Return: The length of the parsed level and control flags.
2092  */
printk_parse_prefix(const char * text,int * level,enum printk_info_flags * flags)2093 u16 printk_parse_prefix(const char *text, int *level,
2094 			enum printk_info_flags *flags)
2095 {
2096 	u16 prefix_len = 0;
2097 	int kern_level;
2098 
2099 	while (*text) {
2100 		kern_level = printk_get_level(text);
2101 		if (!kern_level)
2102 			break;
2103 
2104 		switch (kern_level) {
2105 		case '0' ... '7':
2106 			if (level && *level == LOGLEVEL_DEFAULT)
2107 				*level = kern_level - '0';
2108 			break;
2109 		case 'c':	/* KERN_CONT */
2110 			if (flags)
2111 				*flags |= LOG_CONT;
2112 		}
2113 
2114 		prefix_len += 2;
2115 		text += 2;
2116 	}
2117 
2118 	return prefix_len;
2119 }
2120 
2121 __printf(5, 0)
printk_sprint(char * text,u16 size,int facility,enum printk_info_flags * flags,const char * fmt,va_list args)2122 static u16 printk_sprint(char *text, u16 size, int facility,
2123 			 enum printk_info_flags *flags, const char *fmt,
2124 			 va_list args)
2125 {
2126 	u16 text_len;
2127 
2128 	text_len = vscnprintf(text, size, fmt, args);
2129 
2130 	/* Mark and strip a trailing newline. */
2131 	if (text_len && text[text_len - 1] == '\n') {
2132 		text_len--;
2133 		*flags |= LOG_NEWLINE;
2134 	}
2135 
2136 	/* Strip log level and control flags. */
2137 	if (facility == 0) {
2138 		u16 prefix_len;
2139 
2140 		prefix_len = printk_parse_prefix(text, NULL, NULL);
2141 		if (prefix_len) {
2142 			text_len -= prefix_len;
2143 			memmove(text, text + prefix_len, text_len);
2144 		}
2145 	}
2146 
2147 	trace_console(text, text_len);
2148 
2149 	return text_len;
2150 }
2151 
2152 __printf(4, 0)
vprintk_store(int facility,int level,const struct dev_printk_info * dev_info,const char * fmt,va_list args)2153 int vprintk_store(int facility, int level,
2154 		  const struct dev_printk_info *dev_info,
2155 		  const char *fmt, va_list args)
2156 {
2157 	struct prb_reserved_entry e;
2158 	enum printk_info_flags flags = 0;
2159 	struct printk_record r;
2160 	unsigned long irqflags;
2161 	u16 trunc_msg_len = 0;
2162 	char prefix_buf[8];
2163 	u8 *recursion_ptr;
2164 	u16 reserve_size;
2165 	va_list args2;
2166 	u32 caller_id;
2167 	u16 text_len;
2168 	int ret = 0;
2169 	u64 ts_nsec;
2170 
2171 	if (!printk_enter_irqsave(recursion_ptr, irqflags))
2172 		return 0;
2173 
2174 	/*
2175 	 * Since the duration of printk() can vary depending on the message
2176 	 * and state of the ringbuffer, grab the timestamp now so that it is
2177 	 * close to the call of printk(). This provides a more deterministic
2178 	 * timestamp with respect to the caller.
2179 	 */
2180 	ts_nsec = local_clock();
2181 
2182 	caller_id = printk_caller_id();
2183 
2184 	/*
2185 	 * The sprintf needs to come first since the syslog prefix might be
2186 	 * passed in as a parameter. An extra byte must be reserved so that
2187 	 * later the vscnprintf() into the reserved buffer has room for the
2188 	 * terminating '\0', which is not counted by vsnprintf().
2189 	 */
2190 	va_copy(args2, args);
2191 	reserve_size = vsnprintf(&prefix_buf[0], sizeof(prefix_buf), fmt, args2) + 1;
2192 	va_end(args2);
2193 
2194 	if (reserve_size > PRINTKRB_RECORD_MAX)
2195 		reserve_size = PRINTKRB_RECORD_MAX;
2196 
2197 	/* Extract log level or control flags. */
2198 	if (facility == 0)
2199 		printk_parse_prefix(&prefix_buf[0], &level, &flags);
2200 
2201 	if (level == LOGLEVEL_DEFAULT)
2202 		level = default_message_loglevel;
2203 
2204 	if (dev_info)
2205 		flags |= LOG_NEWLINE;
2206 
2207 	if (flags & LOG_CONT) {
2208 		prb_rec_init_wr(&r, reserve_size);
2209 		if (prb_reserve_in_last(&e, prb, &r, caller_id, PRINTKRB_RECORD_MAX)) {
2210 			text_len = printk_sprint(&r.text_buf[r.info->text_len], reserve_size,
2211 						 facility, &flags, fmt, args);
2212 			r.info->text_len += text_len;
2213 
2214 			if (flags & LOG_NEWLINE) {
2215 				r.info->flags |= LOG_NEWLINE;
2216 				prb_final_commit(&e);
2217 			} else {
2218 				prb_commit(&e);
2219 			}
2220 
2221 			ret = text_len;
2222 			goto out;
2223 		}
2224 	}
2225 
2226 	/*
2227 	 * Explicitly initialize the record before every prb_reserve() call.
2228 	 * prb_reserve_in_last() and prb_reserve() purposely invalidate the
2229 	 * structure when they fail.
2230 	 */
2231 	prb_rec_init_wr(&r, reserve_size);
2232 	if (!prb_reserve(&e, prb, &r)) {
2233 		/* truncate the message if it is too long for empty buffer */
2234 		truncate_msg(&reserve_size, &trunc_msg_len);
2235 
2236 		prb_rec_init_wr(&r, reserve_size + trunc_msg_len);
2237 		if (!prb_reserve(&e, prb, &r))
2238 			goto out;
2239 	}
2240 
2241 	/* fill message */
2242 	text_len = printk_sprint(&r.text_buf[0], reserve_size, facility, &flags, fmt, args);
2243 	if (trunc_msg_len)
2244 		memcpy(&r.text_buf[text_len], trunc_msg, trunc_msg_len);
2245 	r.info->text_len = text_len + trunc_msg_len;
2246 	r.info->facility = facility;
2247 	r.info->level = level & 7;
2248 	r.info->flags = flags & 0x1f;
2249 	r.info->ts_nsec = ts_nsec;
2250 	r.info->caller_id = caller_id;
2251 	if (dev_info)
2252 		memcpy(&r.info->dev_info, dev_info, sizeof(r.info->dev_info));
2253 
2254 	/* A message without a trailing newline can be continued. */
2255 	if (!(flags & LOG_NEWLINE))
2256 		prb_commit(&e);
2257 	else
2258 		prb_final_commit(&e);
2259 
2260 	ret = text_len + trunc_msg_len;
2261 out:
2262 	printk_exit_irqrestore(recursion_ptr, irqflags);
2263 	return ret;
2264 }
2265 
vprintk_emit(int facility,int level,const struct dev_printk_info * dev_info,const char * fmt,va_list args)2266 asmlinkage int vprintk_emit(int facility, int level,
2267 			    const struct dev_printk_info *dev_info,
2268 			    const char *fmt, va_list args)
2269 {
2270 	int printed_len;
2271 	bool in_sched = false;
2272 
2273 	/* Suppress unimportant messages after panic happens */
2274 	if (unlikely(suppress_printk))
2275 		return 0;
2276 
2277 	if (unlikely(suppress_panic_printk) &&
2278 	    atomic_read(&panic_cpu) != raw_smp_processor_id())
2279 		return 0;
2280 
2281 	if (level == LOGLEVEL_SCHED) {
2282 		level = LOGLEVEL_DEFAULT;
2283 		in_sched = true;
2284 	}
2285 
2286 	printk_delay(level);
2287 
2288 	printed_len = vprintk_store(facility, level, dev_info, fmt, args);
2289 
2290 	/* If called from the scheduler, we can not call up(). */
2291 	if (!in_sched) {
2292 		/*
2293 		 * The caller may be holding system-critical or
2294 		 * timing-sensitive locks. Disable preemption during
2295 		 * printing of all remaining records to all consoles so that
2296 		 * this context can return as soon as possible. Hopefully
2297 		 * another printk() caller will take over the printing.
2298 		 */
2299 		preempt_disable();
2300 		/*
2301 		 * Try to acquire and then immediately release the console
2302 		 * semaphore. The release will print out buffers. With the
2303 		 * spinning variant, this context tries to take over the
2304 		 * printing from another printing context.
2305 		 */
2306 		if (console_trylock_spinning())
2307 			console_unlock();
2308 		preempt_enable();
2309 	}
2310 
2311 	if (in_sched)
2312 		defer_console_output();
2313 	else
2314 		wake_up_klogd();
2315 
2316 	return printed_len;
2317 }
2318 EXPORT_SYMBOL(vprintk_emit);
2319 
vprintk_default(const char * fmt,va_list args)2320 int vprintk_default(const char *fmt, va_list args)
2321 {
2322 	return vprintk_emit(0, LOGLEVEL_DEFAULT, NULL, fmt, args);
2323 }
2324 EXPORT_SYMBOL_GPL(vprintk_default);
2325 
_printk(const char * fmt,...)2326 asmlinkage __visible int _printk(const char *fmt, ...)
2327 {
2328 	va_list args;
2329 	int r;
2330 
2331 	va_start(args, fmt);
2332 	r = vprintk(fmt, args);
2333 	va_end(args);
2334 
2335 	return r;
2336 }
2337 EXPORT_SYMBOL(_printk);
2338 
2339 static bool pr_flush(int timeout_ms, bool reset_on_progress);
2340 static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progress);
2341 
2342 #else /* CONFIG_PRINTK */
2343 
2344 #define printk_time		false
2345 
2346 #define prb_read_valid(rb, seq, r)	false
2347 #define prb_first_valid_seq(rb)		0
2348 #define prb_next_seq(rb)		0
2349 
2350 static u64 syslog_seq;
2351 
record_print_text(const struct printk_record * r,bool syslog,bool time)2352 static size_t record_print_text(const struct printk_record *r,
2353 				bool syslog, bool time)
2354 {
2355 	return 0;
2356 }
info_print_ext_header(char * buf,size_t size,struct printk_info * info)2357 static ssize_t info_print_ext_header(char *buf, size_t size,
2358 				     struct printk_info *info)
2359 {
2360 	return 0;
2361 }
msg_print_ext_body(char * buf,size_t size,char * text,size_t text_len,struct dev_printk_info * dev_info)2362 static ssize_t msg_print_ext_body(char *buf, size_t size,
2363 				  char *text, size_t text_len,
2364 				  struct dev_printk_info *dev_info) { return 0; }
console_lock_spinning_enable(void)2365 static void console_lock_spinning_enable(void) { }
console_lock_spinning_disable_and_check(int cookie)2366 static int console_lock_spinning_disable_and_check(int cookie) { return 0; }
suppress_message_printing(int level)2367 static bool suppress_message_printing(int level) { return false; }
pr_flush(int timeout_ms,bool reset_on_progress)2368 static bool pr_flush(int timeout_ms, bool reset_on_progress) { return true; }
__pr_flush(struct console * con,int timeout_ms,bool reset_on_progress)2369 static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progress) { return true; }
2370 
2371 #endif /* CONFIG_PRINTK */
2372 
2373 #ifdef CONFIG_EARLY_PRINTK
2374 struct console *early_console;
2375 
early_printk(const char * fmt,...)2376 asmlinkage __visible void early_printk(const char *fmt, ...)
2377 {
2378 	va_list ap;
2379 	char buf[512];
2380 	int n;
2381 
2382 	if (!early_console)
2383 		return;
2384 
2385 	va_start(ap, fmt);
2386 	n = vscnprintf(buf, sizeof(buf), fmt, ap);
2387 	va_end(ap);
2388 
2389 	early_console->write(early_console, buf, n);
2390 }
2391 #endif
2392 
set_user_specified(struct console_cmdline * c,bool user_specified)2393 static void set_user_specified(struct console_cmdline *c, bool user_specified)
2394 {
2395 	if (!user_specified)
2396 		return;
2397 
2398 	/*
2399 	 * @c console was defined by the user on the command line.
2400 	 * Do not clear when added twice also by SPCR or the device tree.
2401 	 */
2402 	c->user_specified = true;
2403 	/* At least one console defined by the user on the command line. */
2404 	console_set_on_cmdline = 1;
2405 }
2406 
__add_preferred_console(char * name,int idx,char * options,char * brl_options,bool user_specified)2407 static int __add_preferred_console(char *name, int idx, char *options,
2408 				   char *brl_options, bool user_specified)
2409 {
2410 	struct console_cmdline *c;
2411 	int i;
2412 
2413 	/*
2414 	 *	See if this tty is not yet registered, and
2415 	 *	if we have a slot free.
2416 	 */
2417 	for (i = 0, c = console_cmdline;
2418 	     i < MAX_CMDLINECONSOLES && c->name[0];
2419 	     i++, c++) {
2420 		if (strcmp(c->name, name) == 0 && c->index == idx) {
2421 			if (!brl_options)
2422 				preferred_console = i;
2423 			set_user_specified(c, user_specified);
2424 			return 0;
2425 		}
2426 	}
2427 	if (i == MAX_CMDLINECONSOLES)
2428 		return -E2BIG;
2429 	if (!brl_options)
2430 		preferred_console = i;
2431 	strscpy(c->name, name, sizeof(c->name));
2432 	c->options = options;
2433 	set_user_specified(c, user_specified);
2434 	braille_set_options(c, brl_options);
2435 
2436 	c->index = idx;
2437 	return 0;
2438 }
2439 
console_msg_format_setup(char * str)2440 static int __init console_msg_format_setup(char *str)
2441 {
2442 	if (!strcmp(str, "syslog"))
2443 		console_msg_format = MSG_FORMAT_SYSLOG;
2444 	if (!strcmp(str, "default"))
2445 		console_msg_format = MSG_FORMAT_DEFAULT;
2446 	return 1;
2447 }
2448 __setup("console_msg_format=", console_msg_format_setup);
2449 
2450 /*
2451  * Set up a console.  Called via do_early_param() in init/main.c
2452  * for each "console=" parameter in the boot command line.
2453  */
console_setup(char * str)2454 static int __init console_setup(char *str)
2455 {
2456 	char buf[sizeof(console_cmdline[0].name) + 4]; /* 4 for "ttyS" */
2457 	char *s, *options, *brl_options = NULL;
2458 	int idx;
2459 
2460 	/*
2461 	 * console="" or console=null have been suggested as a way to
2462 	 * disable console output. Use ttynull that has been created
2463 	 * for exactly this purpose.
2464 	 */
2465 	if (str[0] == 0 || strcmp(str, "null") == 0) {
2466 		__add_preferred_console("ttynull", 0, NULL, NULL, true);
2467 		return 1;
2468 	}
2469 
2470 	if (_braille_console_setup(&str, &brl_options))
2471 		return 1;
2472 
2473 	/*
2474 	 * Decode str into name, index, options.
2475 	 */
2476 	if (str[0] >= '0' && str[0] <= '9') {
2477 		strcpy(buf, "ttyS");
2478 		strncpy(buf + 4, str, sizeof(buf) - 5);
2479 	} else {
2480 		strncpy(buf, str, sizeof(buf) - 1);
2481 	}
2482 	buf[sizeof(buf) - 1] = 0;
2483 	options = strchr(str, ',');
2484 	if (options)
2485 		*(options++) = 0;
2486 #ifdef __sparc__
2487 	if (!strcmp(str, "ttya"))
2488 		strcpy(buf, "ttyS0");
2489 	if (!strcmp(str, "ttyb"))
2490 		strcpy(buf, "ttyS1");
2491 #endif
2492 	for (s = buf; *s; s++)
2493 		if (isdigit(*s) || *s == ',')
2494 			break;
2495 	idx = simple_strtoul(s, NULL, 10);
2496 	*s = 0;
2497 
2498 	__add_preferred_console(buf, idx, options, brl_options, true);
2499 	return 1;
2500 }
2501 __setup("console=", console_setup);
2502 
2503 /**
2504  * add_preferred_console - add a device to the list of preferred consoles.
2505  * @name: device name
2506  * @idx: device index
2507  * @options: options for this console
2508  *
2509  * The last preferred console added will be used for kernel messages
2510  * and stdin/out/err for init.  Normally this is used by console_setup
2511  * above to handle user-supplied console arguments; however it can also
2512  * be used by arch-specific code either to override the user or more
2513  * commonly to provide a default console (ie from PROM variables) when
2514  * the user has not supplied one.
2515  */
add_preferred_console(char * name,int idx,char * options)2516 int add_preferred_console(char *name, int idx, char *options)
2517 {
2518 	return __add_preferred_console(name, idx, options, NULL, false);
2519 }
2520 
2521 bool console_suspend_enabled = true;
2522 EXPORT_SYMBOL(console_suspend_enabled);
2523 
console_suspend_disable(char * str)2524 static int __init console_suspend_disable(char *str)
2525 {
2526 	console_suspend_enabled = false;
2527 	return 1;
2528 }
2529 __setup("no_console_suspend", console_suspend_disable);
2530 module_param_named(console_suspend, console_suspend_enabled,
2531 		bool, S_IRUGO | S_IWUSR);
2532 MODULE_PARM_DESC(console_suspend, "suspend console during suspend"
2533 	" and hibernate operations");
2534 
2535 static bool printk_console_no_auto_verbose;
2536 
console_verbose(void)2537 void console_verbose(void)
2538 {
2539 	if (console_loglevel && !printk_console_no_auto_verbose)
2540 		console_loglevel = CONSOLE_LOGLEVEL_MOTORMOUTH;
2541 }
2542 EXPORT_SYMBOL_GPL(console_verbose);
2543 
2544 module_param_named(console_no_auto_verbose, printk_console_no_auto_verbose, bool, 0644);
2545 MODULE_PARM_DESC(console_no_auto_verbose, "Disable console loglevel raise to highest on oops/panic/etc");
2546 
2547 /**
2548  * suspend_console - suspend the console subsystem
2549  *
2550  * This disables printk() while we go into suspend states
2551  */
suspend_console(void)2552 void suspend_console(void)
2553 {
2554 	struct console *con;
2555 
2556 	if (!console_suspend_enabled)
2557 		return;
2558 	pr_info("Suspending console(s) (use no_console_suspend to debug)\n");
2559 	pr_flush(1000, true);
2560 
2561 	console_list_lock();
2562 	for_each_console(con)
2563 		console_srcu_write_flags(con, con->flags | CON_SUSPENDED);
2564 	console_list_unlock();
2565 
2566 	/*
2567 	 * Ensure that all SRCU list walks have completed. All printing
2568 	 * contexts must be able to see that they are suspended so that it
2569 	 * is guaranteed that all printing has stopped when this function
2570 	 * completes.
2571 	 */
2572 	synchronize_srcu(&console_srcu);
2573 }
2574 
resume_console(void)2575 void resume_console(void)
2576 {
2577 	struct console *con;
2578 
2579 	if (!console_suspend_enabled)
2580 		return;
2581 
2582 	console_list_lock();
2583 	for_each_console(con)
2584 		console_srcu_write_flags(con, con->flags & ~CON_SUSPENDED);
2585 	console_list_unlock();
2586 
2587 	/*
2588 	 * Ensure that all SRCU list walks have completed. All printing
2589 	 * contexts must be able to see they are no longer suspended so
2590 	 * that they are guaranteed to wake up and resume printing.
2591 	 */
2592 	synchronize_srcu(&console_srcu);
2593 
2594 	pr_flush(1000, true);
2595 }
2596 
2597 /**
2598  * console_cpu_notify - print deferred console messages after CPU hotplug
2599  * @cpu: unused
2600  *
2601  * If printk() is called from a CPU that is not online yet, the messages
2602  * will be printed on the console only if there are CON_ANYTIME consoles.
2603  * This function is called when a new CPU comes online (or fails to come
2604  * up) or goes offline.
2605  */
console_cpu_notify(unsigned int cpu)2606 static int console_cpu_notify(unsigned int cpu)
2607 {
2608 	if (!cpuhp_tasks_frozen) {
2609 		/* If trylock fails, someone else is doing the printing */
2610 		if (console_trylock())
2611 			console_unlock();
2612 	}
2613 	return 0;
2614 }
2615 
2616 /*
2617  * Return true if a panic is in progress on a remote CPU.
2618  *
2619  * On true, the local CPU should immediately release any printing resources
2620  * that may be needed by the panic CPU.
2621  */
other_cpu_in_panic(void)2622 bool other_cpu_in_panic(void)
2623 {
2624 	if (!panic_in_progress())
2625 		return false;
2626 
2627 	/*
2628 	 * We can use raw_smp_processor_id() here because it is impossible for
2629 	 * the task to be migrated to the panic_cpu, or away from it. If
2630 	 * panic_cpu has already been set, and we're not currently executing on
2631 	 * that CPU, then we never will be.
2632 	 */
2633 	return atomic_read(&panic_cpu) != raw_smp_processor_id();
2634 }
2635 
2636 /**
2637  * console_lock - block the console subsystem from printing
2638  *
2639  * Acquires a lock which guarantees that no consoles will
2640  * be in or enter their write() callback.
2641  *
2642  * Can sleep, returns nothing.
2643  */
console_lock(void)2644 void console_lock(void)
2645 {
2646 	might_sleep();
2647 
2648 	/* On panic, the console_lock must be left to the panic cpu. */
2649 	while (other_cpu_in_panic())
2650 		msleep(1000);
2651 
2652 	down_console_sem();
2653 	console_locked = 1;
2654 	console_may_schedule = 1;
2655 }
2656 EXPORT_SYMBOL(console_lock);
2657 
2658 /**
2659  * console_trylock - try to block the console subsystem from printing
2660  *
2661  * Try to acquire a lock which guarantees that no consoles will
2662  * be in or enter their write() callback.
2663  *
2664  * returns 1 on success, and 0 on failure to acquire the lock.
2665  */
console_trylock(void)2666 int console_trylock(void)
2667 {
2668 	/* On panic, the console_lock must be left to the panic cpu. */
2669 	if (other_cpu_in_panic())
2670 		return 0;
2671 	if (down_trylock_console_sem())
2672 		return 0;
2673 	console_locked = 1;
2674 	console_may_schedule = 0;
2675 	return 1;
2676 }
2677 EXPORT_SYMBOL(console_trylock);
2678 
is_console_locked(void)2679 int is_console_locked(void)
2680 {
2681 	return console_locked;
2682 }
2683 EXPORT_SYMBOL(is_console_locked);
2684 
2685 /*
2686  * Check if the given console is currently capable and allowed to print
2687  * records.
2688  *
2689  * Requires the console_srcu_read_lock.
2690  */
console_is_usable(struct console * con)2691 static inline bool console_is_usable(struct console *con)
2692 {
2693 	short flags = console_srcu_read_flags(con);
2694 
2695 	if (!(flags & CON_ENABLED))
2696 		return false;
2697 
2698 	if ((flags & CON_SUSPENDED))
2699 		return false;
2700 
2701 	if (!con->write)
2702 		return false;
2703 
2704 	/*
2705 	 * Console drivers may assume that per-cpu resources have been
2706 	 * allocated. So unless they're explicitly marked as being able to
2707 	 * cope (CON_ANYTIME) don't call them until this CPU is officially up.
2708 	 */
2709 	if (!cpu_online(raw_smp_processor_id()) && !(flags & CON_ANYTIME))
2710 		return false;
2711 
2712 	return true;
2713 }
2714 
__console_unlock(void)2715 static void __console_unlock(void)
2716 {
2717 	console_locked = 0;
2718 	up_console_sem();
2719 }
2720 
2721 /*
2722  * Prepend the message in @pmsg->pbufs->outbuf with a "dropped message". This
2723  * is achieved by shifting the existing message over and inserting the dropped
2724  * message.
2725  *
2726  * @pmsg is the printk message to prepend.
2727  *
2728  * @dropped is the dropped count to report in the dropped message.
2729  *
2730  * If the message text in @pmsg->pbufs->outbuf does not have enough space for
2731  * the dropped message, the message text will be sufficiently truncated.
2732  *
2733  * If @pmsg->pbufs->outbuf is modified, @pmsg->outbuf_len is updated.
2734  */
2735 #ifdef CONFIG_PRINTK
console_prepend_dropped(struct printk_message * pmsg,unsigned long dropped)2736 static void console_prepend_dropped(struct printk_message *pmsg, unsigned long dropped)
2737 {
2738 	struct printk_buffers *pbufs = pmsg->pbufs;
2739 	const size_t scratchbuf_sz = sizeof(pbufs->scratchbuf);
2740 	const size_t outbuf_sz = sizeof(pbufs->outbuf);
2741 	char *scratchbuf = &pbufs->scratchbuf[0];
2742 	char *outbuf = &pbufs->outbuf[0];
2743 	size_t len;
2744 
2745 	len = scnprintf(scratchbuf, scratchbuf_sz,
2746 		       "** %lu printk messages dropped **\n", dropped);
2747 
2748 	/*
2749 	 * Make sure outbuf is sufficiently large before prepending.
2750 	 * Keep at least the prefix when the message must be truncated.
2751 	 * It is a rather theoretical problem when someone tries to
2752 	 * use a minimalist buffer.
2753 	 */
2754 	if (WARN_ON_ONCE(len + PRINTK_PREFIX_MAX >= outbuf_sz))
2755 		return;
2756 
2757 	if (pmsg->outbuf_len + len >= outbuf_sz) {
2758 		/* Truncate the message, but keep it terminated. */
2759 		pmsg->outbuf_len = outbuf_sz - (len + 1);
2760 		outbuf[pmsg->outbuf_len] = 0;
2761 	}
2762 
2763 	memmove(outbuf + len, outbuf, pmsg->outbuf_len + 1);
2764 	memcpy(outbuf, scratchbuf, len);
2765 	pmsg->outbuf_len += len;
2766 }
2767 #else
2768 #define console_prepend_dropped(pmsg, dropped)
2769 #endif /* CONFIG_PRINTK */
2770 
2771 /*
2772  * Read and format the specified record (or a later record if the specified
2773  * record is not available).
2774  *
2775  * @pmsg will contain the formatted result. @pmsg->pbufs must point to a
2776  * struct printk_buffers.
2777  *
2778  * @seq is the record to read and format. If it is not available, the next
2779  * valid record is read.
2780  *
2781  * @is_extended specifies if the message should be formatted for extended
2782  * console output.
2783  *
2784  * @may_supress specifies if records may be skipped based on loglevel.
2785  *
2786  * Returns false if no record is available. Otherwise true and all fields
2787  * of @pmsg are valid. (See the documentation of struct printk_message
2788  * for information about the @pmsg fields.)
2789  */
printk_get_next_message(struct printk_message * pmsg,u64 seq,bool is_extended,bool may_suppress)2790 static bool printk_get_next_message(struct printk_message *pmsg, u64 seq,
2791 				    bool is_extended, bool may_suppress)
2792 {
2793 	static int panic_console_dropped;
2794 
2795 	struct printk_buffers *pbufs = pmsg->pbufs;
2796 	const size_t scratchbuf_sz = sizeof(pbufs->scratchbuf);
2797 	const size_t outbuf_sz = sizeof(pbufs->outbuf);
2798 	char *scratchbuf = &pbufs->scratchbuf[0];
2799 	char *outbuf = &pbufs->outbuf[0];
2800 	struct printk_info info;
2801 	struct printk_record r;
2802 	size_t len = 0;
2803 
2804 	/*
2805 	 * Formatting extended messages requires a separate buffer, so use the
2806 	 * scratch buffer to read in the ringbuffer text.
2807 	 *
2808 	 * Formatting normal messages is done in-place, so read the ringbuffer
2809 	 * text directly into the output buffer.
2810 	 */
2811 	if (is_extended)
2812 		prb_rec_init_rd(&r, &info, scratchbuf, scratchbuf_sz);
2813 	else
2814 		prb_rec_init_rd(&r, &info, outbuf, outbuf_sz);
2815 
2816 	if (!prb_read_valid(prb, seq, &r))
2817 		return false;
2818 
2819 	pmsg->seq = r.info->seq;
2820 	pmsg->dropped = r.info->seq - seq;
2821 
2822 	/*
2823 	 * Check for dropped messages in panic here so that printk
2824 	 * suppression can occur as early as possible if necessary.
2825 	 */
2826 	if (pmsg->dropped &&
2827 	    panic_in_progress() &&
2828 	    panic_console_dropped++ > 10) {
2829 		suppress_panic_printk = 1;
2830 		pr_warn_once("Too many dropped messages. Suppress messages on non-panic CPUs to prevent livelock.\n");
2831 	}
2832 
2833 	/* Skip record that has level above the console loglevel. */
2834 	if (may_suppress && suppress_message_printing(r.info->level))
2835 		goto out;
2836 
2837 	if (is_extended) {
2838 		len = info_print_ext_header(outbuf, outbuf_sz, r.info);
2839 		len += msg_print_ext_body(outbuf + len, outbuf_sz - len,
2840 					  &r.text_buf[0], r.info->text_len, &r.info->dev_info);
2841 	} else {
2842 		len = record_print_text(&r, console_msg_format & MSG_FORMAT_SYSLOG, printk_time);
2843 	}
2844 out:
2845 	pmsg->outbuf_len = len;
2846 	return true;
2847 }
2848 
2849 /*
2850  * Print one record for the given console. The record printed is whatever
2851  * record is the next available record for the given console.
2852  *
2853  * @handover will be set to true if a printk waiter has taken over the
2854  * console_lock, in which case the caller is no longer holding both the
2855  * console_lock and the SRCU read lock. Otherwise it is set to false.
2856  *
2857  * @cookie is the cookie from the SRCU read lock.
2858  *
2859  * Returns false if the given console has no next record to print, otherwise
2860  * true.
2861  *
2862  * Requires the console_lock and the SRCU read lock.
2863  */
console_emit_next_record(struct console * con,bool * handover,int cookie)2864 static bool console_emit_next_record(struct console *con, bool *handover, int cookie)
2865 {
2866 	static struct printk_buffers pbufs;
2867 
2868 	bool is_extended = console_srcu_read_flags(con) & CON_EXTENDED;
2869 	char *outbuf = &pbufs.outbuf[0];
2870 	struct printk_message pmsg = {
2871 		.pbufs = &pbufs,
2872 	};
2873 	unsigned long flags;
2874 
2875 	*handover = false;
2876 
2877 	if (!printk_get_next_message(&pmsg, con->seq, is_extended, true))
2878 		return false;
2879 
2880 	con->dropped += pmsg.dropped;
2881 
2882 	/* Skip messages of formatted length 0. */
2883 	if (pmsg.outbuf_len == 0) {
2884 		con->seq = pmsg.seq + 1;
2885 		goto skip;
2886 	}
2887 
2888 	if (con->dropped && !is_extended) {
2889 		console_prepend_dropped(&pmsg, con->dropped);
2890 		con->dropped = 0;
2891 	}
2892 
2893 	/*
2894 	 * While actively printing out messages, if another printk()
2895 	 * were to occur on another CPU, it may wait for this one to
2896 	 * finish. This task can not be preempted if there is a
2897 	 * waiter waiting to take over.
2898 	 *
2899 	 * Interrupts are disabled because the hand over to a waiter
2900 	 * must not be interrupted until the hand over is completed
2901 	 * (@console_waiter is cleared).
2902 	 */
2903 	printk_safe_enter_irqsave(flags);
2904 	console_lock_spinning_enable();
2905 
2906 	/* Do not trace print latency. */
2907 	stop_critical_timings();
2908 
2909 	/* Write everything out to the hardware. */
2910 	con->write(con, outbuf, pmsg.outbuf_len);
2911 
2912 	start_critical_timings();
2913 
2914 	con->seq = pmsg.seq + 1;
2915 
2916 	*handover = console_lock_spinning_disable_and_check(cookie);
2917 	printk_safe_exit_irqrestore(flags);
2918 skip:
2919 	return true;
2920 }
2921 
2922 /*
2923  * Print out all remaining records to all consoles.
2924  *
2925  * @do_cond_resched is set by the caller. It can be true only in schedulable
2926  * context.
2927  *
2928  * @next_seq is set to the sequence number after the last available record.
2929  * The value is valid only when this function returns true. It means that all
2930  * usable consoles are completely flushed.
2931  *
2932  * @handover will be set to true if a printk waiter has taken over the
2933  * console_lock, in which case the caller is no longer holding the
2934  * console_lock. Otherwise it is set to false.
2935  *
2936  * Returns true when there was at least one usable console and all messages
2937  * were flushed to all usable consoles. A returned false informs the caller
2938  * that everything was not flushed (either there were no usable consoles or
2939  * another context has taken over printing or it is a panic situation and this
2940  * is not the panic CPU). Regardless the reason, the caller should assume it
2941  * is not useful to immediately try again.
2942  *
2943  * Requires the console_lock.
2944  */
console_flush_all(bool do_cond_resched,u64 * next_seq,bool * handover)2945 static bool console_flush_all(bool do_cond_resched, u64 *next_seq, bool *handover)
2946 {
2947 	bool any_usable = false;
2948 	struct console *con;
2949 	bool any_progress;
2950 	int cookie;
2951 
2952 	*next_seq = 0;
2953 	*handover = false;
2954 
2955 	do {
2956 		any_progress = false;
2957 
2958 		cookie = console_srcu_read_lock();
2959 		for_each_console_srcu(con) {
2960 			bool progress;
2961 
2962 			if (!console_is_usable(con))
2963 				continue;
2964 			any_usable = true;
2965 
2966 			progress = console_emit_next_record(con, handover, cookie);
2967 
2968 			/*
2969 			 * If a handover has occurred, the SRCU read lock
2970 			 * is already released.
2971 			 */
2972 			if (*handover)
2973 				return false;
2974 
2975 			/* Track the next of the highest seq flushed. */
2976 			if (con->seq > *next_seq)
2977 				*next_seq = con->seq;
2978 
2979 			if (!progress)
2980 				continue;
2981 			any_progress = true;
2982 
2983 			/* Allow panic_cpu to take over the consoles safely. */
2984 			if (other_cpu_in_panic())
2985 				goto abandon;
2986 
2987 			if (do_cond_resched)
2988 				cond_resched();
2989 		}
2990 		console_srcu_read_unlock(cookie);
2991 	} while (any_progress);
2992 
2993 	return any_usable;
2994 
2995 abandon:
2996 	console_srcu_read_unlock(cookie);
2997 	return false;
2998 }
2999 
3000 /**
3001  * console_unlock - unblock the console subsystem from printing
3002  *
3003  * Releases the console_lock which the caller holds to block printing of
3004  * the console subsystem.
3005  *
3006  * While the console_lock was held, console output may have been buffered
3007  * by printk().  If this is the case, console_unlock(); emits
3008  * the output prior to releasing the lock.
3009  *
3010  * console_unlock(); may be called from any context.
3011  */
console_unlock(void)3012 void console_unlock(void)
3013 {
3014 	bool do_cond_resched;
3015 	bool handover;
3016 	bool flushed;
3017 	u64 next_seq;
3018 
3019 	/*
3020 	 * Console drivers are called with interrupts disabled, so
3021 	 * @console_may_schedule should be cleared before; however, we may
3022 	 * end up dumping a lot of lines, for example, if called from
3023 	 * console registration path, and should invoke cond_resched()
3024 	 * between lines if allowable.  Not doing so can cause a very long
3025 	 * scheduling stall on a slow console leading to RCU stall and
3026 	 * softlockup warnings which exacerbate the issue with more
3027 	 * messages practically incapacitating the system. Therefore, create
3028 	 * a local to use for the printing loop.
3029 	 */
3030 	do_cond_resched = console_may_schedule;
3031 
3032 	do {
3033 		console_may_schedule = 0;
3034 
3035 		flushed = console_flush_all(do_cond_resched, &next_seq, &handover);
3036 		if (!handover)
3037 			__console_unlock();
3038 
3039 		/*
3040 		 * Abort if there was a failure to flush all messages to all
3041 		 * usable consoles. Either it is not possible to flush (in
3042 		 * which case it would be an infinite loop of retrying) or
3043 		 * another context has taken over printing.
3044 		 */
3045 		if (!flushed)
3046 			break;
3047 
3048 		/*
3049 		 * Some context may have added new records after
3050 		 * console_flush_all() but before unlocking the console.
3051 		 * Re-check if there is a new record to flush. If the trylock
3052 		 * fails, another context is already handling the printing.
3053 		 */
3054 	} while (prb_read_valid(prb, next_seq, NULL) && console_trylock());
3055 }
3056 EXPORT_SYMBOL(console_unlock);
3057 
3058 /**
3059  * console_conditional_schedule - yield the CPU if required
3060  *
3061  * If the console code is currently allowed to sleep, and
3062  * if this CPU should yield the CPU to another task, do
3063  * so here.
3064  *
3065  * Must be called within console_lock();.
3066  */
console_conditional_schedule(void)3067 void __sched console_conditional_schedule(void)
3068 {
3069 	if (console_may_schedule)
3070 		cond_resched();
3071 }
3072 EXPORT_SYMBOL(console_conditional_schedule);
3073 
console_unblank(void)3074 void console_unblank(void)
3075 {
3076 	bool found_unblank = false;
3077 	struct console *c;
3078 	int cookie;
3079 
3080 	/*
3081 	 * First check if there are any consoles implementing the unblank()
3082 	 * callback. If not, there is no reason to continue and take the
3083 	 * console lock, which in particular can be dangerous if
3084 	 * @oops_in_progress is set.
3085 	 */
3086 	cookie = console_srcu_read_lock();
3087 	for_each_console_srcu(c) {
3088 		if ((console_srcu_read_flags(c) & CON_ENABLED) && c->unblank) {
3089 			found_unblank = true;
3090 			break;
3091 		}
3092 	}
3093 	console_srcu_read_unlock(cookie);
3094 	if (!found_unblank)
3095 		return;
3096 
3097 	/*
3098 	 * Stop console printing because the unblank() callback may
3099 	 * assume the console is not within its write() callback.
3100 	 *
3101 	 * If @oops_in_progress is set, this may be an atomic context.
3102 	 * In that case, attempt a trylock as best-effort.
3103 	 */
3104 	if (oops_in_progress) {
3105 		/* Semaphores are not NMI-safe. */
3106 		if (in_nmi())
3107 			return;
3108 
3109 		/*
3110 		 * Attempting to trylock the console lock can deadlock
3111 		 * if another CPU was stopped while modifying the
3112 		 * semaphore. "Hope and pray" that this is not the
3113 		 * current situation.
3114 		 */
3115 		if (down_trylock_console_sem() != 0)
3116 			return;
3117 	} else
3118 		console_lock();
3119 
3120 	console_locked = 1;
3121 	console_may_schedule = 0;
3122 
3123 	cookie = console_srcu_read_lock();
3124 	for_each_console_srcu(c) {
3125 		if ((console_srcu_read_flags(c) & CON_ENABLED) && c->unblank)
3126 			c->unblank();
3127 	}
3128 	console_srcu_read_unlock(cookie);
3129 
3130 	console_unlock();
3131 
3132 	if (!oops_in_progress)
3133 		pr_flush(1000, true);
3134 }
3135 
3136 /**
3137  * console_flush_on_panic - flush console content on panic
3138  * @mode: flush all messages in buffer or just the pending ones
3139  *
3140  * Immediately output all pending messages no matter what.
3141  */
console_flush_on_panic(enum con_flush_mode mode)3142 void console_flush_on_panic(enum con_flush_mode mode)
3143 {
3144 	bool handover;
3145 	u64 next_seq;
3146 
3147 	/*
3148 	 * Ignore the console lock and flush out the messages. Attempting a
3149 	 * trylock would not be useful because:
3150 	 *
3151 	 *   - if it is contended, it must be ignored anyway
3152 	 *   - console_lock() and console_trylock() block and fail
3153 	 *     respectively in panic for non-panic CPUs
3154 	 *   - semaphores are not NMI-safe
3155 	 */
3156 
3157 	/*
3158 	 * If another context is holding the console lock,
3159 	 * @console_may_schedule might be set. Clear it so that
3160 	 * this context does not call cond_resched() while flushing.
3161 	 */
3162 	console_may_schedule = 0;
3163 
3164 	if (mode == CONSOLE_REPLAY_ALL) {
3165 		struct console *c;
3166 		int cookie;
3167 		u64 seq;
3168 
3169 		seq = prb_first_valid_seq(prb);
3170 
3171 		cookie = console_srcu_read_lock();
3172 		for_each_console_srcu(c) {
3173 			/*
3174 			 * This is an unsynchronized assignment, but the
3175 			 * kernel is in "hope and pray" mode anyway.
3176 			 */
3177 			c->seq = seq;
3178 		}
3179 		console_srcu_read_unlock(cookie);
3180 	}
3181 
3182 	console_flush_all(false, &next_seq, &handover);
3183 }
3184 
3185 /*
3186  * Return the console tty driver structure and its associated index
3187  */
console_device(int * index)3188 struct tty_driver *console_device(int *index)
3189 {
3190 	struct console *c;
3191 	struct tty_driver *driver = NULL;
3192 	int cookie;
3193 
3194 	/*
3195 	 * Take console_lock to serialize device() callback with
3196 	 * other console operations. For example, fg_console is
3197 	 * modified under console_lock when switching vt.
3198 	 */
3199 	console_lock();
3200 
3201 	cookie = console_srcu_read_lock();
3202 	for_each_console_srcu(c) {
3203 		if (!c->device)
3204 			continue;
3205 		driver = c->device(c, index);
3206 		if (driver)
3207 			break;
3208 	}
3209 	console_srcu_read_unlock(cookie);
3210 
3211 	console_unlock();
3212 	return driver;
3213 }
3214 
3215 /*
3216  * Prevent further output on the passed console device so that (for example)
3217  * serial drivers can disable console output before suspending a port, and can
3218  * re-enable output afterwards.
3219  */
console_stop(struct console * console)3220 void console_stop(struct console *console)
3221 {
3222 	__pr_flush(console, 1000, true);
3223 	console_list_lock();
3224 	console_srcu_write_flags(console, console->flags & ~CON_ENABLED);
3225 	console_list_unlock();
3226 
3227 	/*
3228 	 * Ensure that all SRCU list walks have completed. All contexts must
3229 	 * be able to see that this console is disabled so that (for example)
3230 	 * the caller can suspend the port without risk of another context
3231 	 * using the port.
3232 	 */
3233 	synchronize_srcu(&console_srcu);
3234 }
3235 EXPORT_SYMBOL(console_stop);
3236 
console_start(struct console * console)3237 void console_start(struct console *console)
3238 {
3239 	console_list_lock();
3240 	console_srcu_write_flags(console, console->flags | CON_ENABLED);
3241 	console_list_unlock();
3242 	__pr_flush(console, 1000, true);
3243 }
3244 EXPORT_SYMBOL(console_start);
3245 
3246 static int __read_mostly keep_bootcon;
3247 
keep_bootcon_setup(char * str)3248 static int __init keep_bootcon_setup(char *str)
3249 {
3250 	keep_bootcon = 1;
3251 	pr_info("debug: skip boot console de-registration.\n");
3252 
3253 	return 0;
3254 }
3255 
3256 early_param("keep_bootcon", keep_bootcon_setup);
3257 
3258 /*
3259  * This is called by register_console() to try to match
3260  * the newly registered console with any of the ones selected
3261  * by either the command line or add_preferred_console() and
3262  * setup/enable it.
3263  *
3264  * Care need to be taken with consoles that are statically
3265  * enabled such as netconsole
3266  */
try_enable_preferred_console(struct console * newcon,bool user_specified)3267 static int try_enable_preferred_console(struct console *newcon,
3268 					bool user_specified)
3269 {
3270 	struct console_cmdline *c;
3271 	int i, err;
3272 
3273 	for (i = 0, c = console_cmdline;
3274 	     i < MAX_CMDLINECONSOLES && c->name[0];
3275 	     i++, c++) {
3276 		if (c->user_specified != user_specified)
3277 			continue;
3278 		if (!newcon->match ||
3279 		    newcon->match(newcon, c->name, c->index, c->options) != 0) {
3280 			/* default matching */
3281 			BUILD_BUG_ON(sizeof(c->name) != sizeof(newcon->name));
3282 			if (strcmp(c->name, newcon->name) != 0)
3283 				continue;
3284 			if (newcon->index >= 0 &&
3285 			    newcon->index != c->index)
3286 				continue;
3287 			if (newcon->index < 0)
3288 				newcon->index = c->index;
3289 
3290 			if (_braille_register_console(newcon, c))
3291 				return 0;
3292 
3293 			if (newcon->setup &&
3294 			    (err = newcon->setup(newcon, c->options)) != 0)
3295 				return err;
3296 		}
3297 		newcon->flags |= CON_ENABLED;
3298 		if (i == preferred_console)
3299 			newcon->flags |= CON_CONSDEV;
3300 		return 0;
3301 	}
3302 
3303 	/*
3304 	 * Some consoles, such as pstore and netconsole, can be enabled even
3305 	 * without matching. Accept the pre-enabled consoles only when match()
3306 	 * and setup() had a chance to be called.
3307 	 */
3308 	if (newcon->flags & CON_ENABLED && c->user_specified ==	user_specified)
3309 		return 0;
3310 
3311 	return -ENOENT;
3312 }
3313 
3314 /* Try to enable the console unconditionally */
try_enable_default_console(struct console * newcon)3315 static void try_enable_default_console(struct console *newcon)
3316 {
3317 	if (newcon->index < 0)
3318 		newcon->index = 0;
3319 
3320 	if (newcon->setup && newcon->setup(newcon, NULL) != 0)
3321 		return;
3322 
3323 	newcon->flags |= CON_ENABLED;
3324 
3325 	if (newcon->device)
3326 		newcon->flags |= CON_CONSDEV;
3327 }
3328 
3329 #define con_printk(lvl, con, fmt, ...)			\
3330 	printk(lvl pr_fmt("%sconsole [%s%d] " fmt),	\
3331 	       (con->flags & CON_BOOT) ? "boot" : "",	\
3332 	       con->name, con->index, ##__VA_ARGS__)
3333 
console_init_seq(struct console * newcon,bool bootcon_registered)3334 static void console_init_seq(struct console *newcon, bool bootcon_registered)
3335 {
3336 	struct console *con;
3337 	bool handover;
3338 
3339 	if (newcon->flags & (CON_PRINTBUFFER | CON_BOOT)) {
3340 		/* Get a consistent copy of @syslog_seq. */
3341 		mutex_lock(&syslog_lock);
3342 		newcon->seq = syslog_seq;
3343 		mutex_unlock(&syslog_lock);
3344 	} else {
3345 		/* Begin with next message added to ringbuffer. */
3346 		newcon->seq = prb_next_seq(prb);
3347 
3348 		/*
3349 		 * If any enabled boot consoles are due to be unregistered
3350 		 * shortly, some may not be caught up and may be the same
3351 		 * device as @newcon. Since it is not known which boot console
3352 		 * is the same device, flush all consoles and, if necessary,
3353 		 * start with the message of the enabled boot console that is
3354 		 * the furthest behind.
3355 		 */
3356 		if (bootcon_registered && !keep_bootcon) {
3357 			/*
3358 			 * Hold the console_lock to stop console printing and
3359 			 * guarantee safe access to console->seq.
3360 			 */
3361 			console_lock();
3362 
3363 			/*
3364 			 * Flush all consoles and set the console to start at
3365 			 * the next unprinted sequence number.
3366 			 */
3367 			if (!console_flush_all(true, &newcon->seq, &handover)) {
3368 				/*
3369 				 * Flushing failed. Just choose the lowest
3370 				 * sequence of the enabled boot consoles.
3371 				 */
3372 
3373 				/*
3374 				 * If there was a handover, this context no
3375 				 * longer holds the console_lock.
3376 				 */
3377 				if (handover)
3378 					console_lock();
3379 
3380 				newcon->seq = prb_next_seq(prb);
3381 				for_each_console(con) {
3382 					if ((con->flags & CON_BOOT) &&
3383 					    (con->flags & CON_ENABLED) &&
3384 					    con->seq < newcon->seq) {
3385 						newcon->seq = con->seq;
3386 					}
3387 				}
3388 			}
3389 
3390 			console_unlock();
3391 		}
3392 	}
3393 }
3394 
3395 #define console_first()				\
3396 	hlist_entry(console_list.first, struct console, node)
3397 
3398 static int unregister_console_locked(struct console *console);
3399 
3400 /*
3401  * The console driver calls this routine during kernel initialization
3402  * to register the console printing procedure with printk() and to
3403  * print any messages that were printed by the kernel before the
3404  * console driver was initialized.
3405  *
3406  * This can happen pretty early during the boot process (because of
3407  * early_printk) - sometimes before setup_arch() completes - be careful
3408  * of what kernel features are used - they may not be initialised yet.
3409  *
3410  * There are two types of consoles - bootconsoles (early_printk) and
3411  * "real" consoles (everything which is not a bootconsole) which are
3412  * handled differently.
3413  *  - Any number of bootconsoles can be registered at any time.
3414  *  - As soon as a "real" console is registered, all bootconsoles
3415  *    will be unregistered automatically.
3416  *  - Once a "real" console is registered, any attempt to register a
3417  *    bootconsoles will be rejected
3418  */
register_console(struct console * newcon)3419 void register_console(struct console *newcon)
3420 {
3421 	struct console *con;
3422 	bool bootcon_registered = false;
3423 	bool realcon_registered = false;
3424 	int err;
3425 
3426 	console_list_lock();
3427 
3428 	for_each_console(con) {
3429 		if (WARN(con == newcon, "console '%s%d' already registered\n",
3430 					 con->name, con->index)) {
3431 			goto unlock;
3432 		}
3433 
3434 		if (con->flags & CON_BOOT)
3435 			bootcon_registered = true;
3436 		else
3437 			realcon_registered = true;
3438 	}
3439 
3440 	/* Do not register boot consoles when there already is a real one. */
3441 	if ((newcon->flags & CON_BOOT) && realcon_registered) {
3442 		pr_info("Too late to register bootconsole %s%d\n",
3443 			newcon->name, newcon->index);
3444 		goto unlock;
3445 	}
3446 
3447 	/*
3448 	 * See if we want to enable this console driver by default.
3449 	 *
3450 	 * Nope when a console is preferred by the command line, device
3451 	 * tree, or SPCR.
3452 	 *
3453 	 * The first real console with tty binding (driver) wins. More
3454 	 * consoles might get enabled before the right one is found.
3455 	 *
3456 	 * Note that a console with tty binding will have CON_CONSDEV
3457 	 * flag set and will be first in the list.
3458 	 */
3459 	if (preferred_console < 0) {
3460 		if (hlist_empty(&console_list) || !console_first()->device ||
3461 		    console_first()->flags & CON_BOOT) {
3462 			try_enable_default_console(newcon);
3463 		}
3464 	}
3465 
3466 	/* See if this console matches one we selected on the command line */
3467 	err = try_enable_preferred_console(newcon, true);
3468 
3469 	/* If not, try to match against the platform default(s) */
3470 	if (err == -ENOENT)
3471 		err = try_enable_preferred_console(newcon, false);
3472 
3473 	/* printk() messages are not printed to the Braille console. */
3474 	if (err || newcon->flags & CON_BRL)
3475 		goto unlock;
3476 
3477 	/*
3478 	 * If we have a bootconsole, and are switching to a real console,
3479 	 * don't print everything out again, since when the boot console, and
3480 	 * the real console are the same physical device, it's annoying to
3481 	 * see the beginning boot messages twice
3482 	 */
3483 	if (bootcon_registered &&
3484 	    ((newcon->flags & (CON_CONSDEV | CON_BOOT)) == CON_CONSDEV)) {
3485 		newcon->flags &= ~CON_PRINTBUFFER;
3486 	}
3487 
3488 	newcon->dropped = 0;
3489 	console_init_seq(newcon, bootcon_registered);
3490 
3491 	/*
3492 	 * Put this console in the list - keep the
3493 	 * preferred driver at the head of the list.
3494 	 */
3495 	if (hlist_empty(&console_list)) {
3496 		/* Ensure CON_CONSDEV is always set for the head. */
3497 		newcon->flags |= CON_CONSDEV;
3498 		hlist_add_head_rcu(&newcon->node, &console_list);
3499 
3500 	} else if (newcon->flags & CON_CONSDEV) {
3501 		/* Only the new head can have CON_CONSDEV set. */
3502 		console_srcu_write_flags(console_first(), console_first()->flags & ~CON_CONSDEV);
3503 		hlist_add_head_rcu(&newcon->node, &console_list);
3504 
3505 	} else {
3506 		hlist_add_behind_rcu(&newcon->node, console_list.first);
3507 	}
3508 
3509 	/*
3510 	 * No need to synchronize SRCU here! The caller does not rely
3511 	 * on all contexts being able to see the new console before
3512 	 * register_console() completes.
3513 	 */
3514 
3515 	console_sysfs_notify();
3516 
3517 	/*
3518 	 * By unregistering the bootconsoles after we enable the real console
3519 	 * we get the "console xxx enabled" message on all the consoles -
3520 	 * boot consoles, real consoles, etc - this is to ensure that end
3521 	 * users know there might be something in the kernel's log buffer that
3522 	 * went to the bootconsole (that they do not see on the real console)
3523 	 */
3524 	con_printk(KERN_INFO, newcon, "enabled\n");
3525 	if (bootcon_registered &&
3526 	    ((newcon->flags & (CON_CONSDEV | CON_BOOT)) == CON_CONSDEV) &&
3527 	    !keep_bootcon) {
3528 		struct hlist_node *tmp;
3529 
3530 		hlist_for_each_entry_safe(con, tmp, &console_list, node) {
3531 			if (con->flags & CON_BOOT)
3532 				unregister_console_locked(con);
3533 		}
3534 	}
3535 unlock:
3536 	console_list_unlock();
3537 }
3538 EXPORT_SYMBOL(register_console);
3539 
3540 /* Must be called under console_list_lock(). */
unregister_console_locked(struct console * console)3541 static int unregister_console_locked(struct console *console)
3542 {
3543 	int res;
3544 
3545 	lockdep_assert_console_list_lock_held();
3546 
3547 	con_printk(KERN_INFO, console, "disabled\n");
3548 
3549 	res = _braille_unregister_console(console);
3550 	if (res < 0)
3551 		return res;
3552 	if (res > 0)
3553 		return 0;
3554 
3555 	/* Disable it unconditionally */
3556 	console_srcu_write_flags(console, console->flags & ~CON_ENABLED);
3557 
3558 	if (!console_is_registered_locked(console))
3559 		return -ENODEV;
3560 
3561 	hlist_del_init_rcu(&console->node);
3562 
3563 	/*
3564 	 * <HISTORICAL>
3565 	 * If this isn't the last console and it has CON_CONSDEV set, we
3566 	 * need to set it on the next preferred console.
3567 	 * </HISTORICAL>
3568 	 *
3569 	 * The above makes no sense as there is no guarantee that the next
3570 	 * console has any device attached. Oh well....
3571 	 */
3572 	if (!hlist_empty(&console_list) && console->flags & CON_CONSDEV)
3573 		console_srcu_write_flags(console_first(), console_first()->flags | CON_CONSDEV);
3574 
3575 	/*
3576 	 * Ensure that all SRCU list walks have completed. All contexts
3577 	 * must not be able to see this console in the list so that any
3578 	 * exit/cleanup routines can be performed safely.
3579 	 */
3580 	synchronize_srcu(&console_srcu);
3581 
3582 	console_sysfs_notify();
3583 
3584 	if (console->exit)
3585 		res = console->exit(console);
3586 
3587 	return res;
3588 }
3589 
unregister_console(struct console * console)3590 int unregister_console(struct console *console)
3591 {
3592 	int res;
3593 
3594 	console_list_lock();
3595 	res = unregister_console_locked(console);
3596 	console_list_unlock();
3597 	return res;
3598 }
3599 EXPORT_SYMBOL(unregister_console);
3600 
3601 /**
3602  * console_force_preferred_locked - force a registered console preferred
3603  * @con: The registered console to force preferred.
3604  *
3605  * Must be called under console_list_lock().
3606  */
console_force_preferred_locked(struct console * con)3607 void console_force_preferred_locked(struct console *con)
3608 {
3609 	struct console *cur_pref_con;
3610 
3611 	if (!console_is_registered_locked(con))
3612 		return;
3613 
3614 	cur_pref_con = console_first();
3615 
3616 	/* Already preferred? */
3617 	if (cur_pref_con == con)
3618 		return;
3619 
3620 	/*
3621 	 * Delete, but do not re-initialize the entry. This allows the console
3622 	 * to continue to appear registered (via any hlist_unhashed_lockless()
3623 	 * checks), even though it was briefly removed from the console list.
3624 	 */
3625 	hlist_del_rcu(&con->node);
3626 
3627 	/*
3628 	 * Ensure that all SRCU list walks have completed so that the console
3629 	 * can be added to the beginning of the console list and its forward
3630 	 * list pointer can be re-initialized.
3631 	 */
3632 	synchronize_srcu(&console_srcu);
3633 
3634 	con->flags |= CON_CONSDEV;
3635 	WARN_ON(!con->device);
3636 
3637 	/* Only the new head can have CON_CONSDEV set. */
3638 	console_srcu_write_flags(cur_pref_con, cur_pref_con->flags & ~CON_CONSDEV);
3639 	hlist_add_head_rcu(&con->node, &console_list);
3640 }
3641 EXPORT_SYMBOL(console_force_preferred_locked);
3642 
3643 /*
3644  * Initialize the console device. This is called *early*, so
3645  * we can't necessarily depend on lots of kernel help here.
3646  * Just do some early initializations, and do the complex setup
3647  * later.
3648  */
console_init(void)3649 void __init console_init(void)
3650 {
3651 	int ret;
3652 	initcall_t call;
3653 	initcall_entry_t *ce;
3654 
3655 	/* Setup the default TTY line discipline. */
3656 	n_tty_init();
3657 
3658 	/*
3659 	 * set up the console device so that later boot sequences can
3660 	 * inform about problems etc..
3661 	 */
3662 	ce = __con_initcall_start;
3663 	trace_initcall_level("console");
3664 	while (ce < __con_initcall_end) {
3665 		call = initcall_from_entry(ce);
3666 		trace_initcall_start(call);
3667 		ret = call();
3668 		trace_initcall_finish(call, ret);
3669 		ce++;
3670 	}
3671 }
3672 
3673 /*
3674  * Some boot consoles access data that is in the init section and which will
3675  * be discarded after the initcalls have been run. To make sure that no code
3676  * will access this data, unregister the boot consoles in a late initcall.
3677  *
3678  * If for some reason, such as deferred probe or the driver being a loadable
3679  * module, the real console hasn't registered yet at this point, there will
3680  * be a brief interval in which no messages are logged to the console, which
3681  * makes it difficult to diagnose problems that occur during this time.
3682  *
3683  * To mitigate this problem somewhat, only unregister consoles whose memory
3684  * intersects with the init section. Note that all other boot consoles will
3685  * get unregistered when the real preferred console is registered.
3686  */
printk_late_init(void)3687 static int __init printk_late_init(void)
3688 {
3689 	struct hlist_node *tmp;
3690 	struct console *con;
3691 	int ret;
3692 
3693 	console_list_lock();
3694 	hlist_for_each_entry_safe(con, tmp, &console_list, node) {
3695 		if (!(con->flags & CON_BOOT))
3696 			continue;
3697 
3698 		/* Check addresses that might be used for enabled consoles. */
3699 		if (init_section_intersects(con, sizeof(*con)) ||
3700 		    init_section_contains(con->write, 0) ||
3701 		    init_section_contains(con->read, 0) ||
3702 		    init_section_contains(con->device, 0) ||
3703 		    init_section_contains(con->unblank, 0) ||
3704 		    init_section_contains(con->data, 0)) {
3705 			/*
3706 			 * Please, consider moving the reported consoles out
3707 			 * of the init section.
3708 			 */
3709 			pr_warn("bootconsole [%s%d] uses init memory and must be disabled even before the real one is ready\n",
3710 				con->name, con->index);
3711 			unregister_console_locked(con);
3712 		}
3713 	}
3714 	console_list_unlock();
3715 
3716 	ret = cpuhp_setup_state_nocalls(CPUHP_PRINTK_DEAD, "printk:dead", NULL,
3717 					console_cpu_notify);
3718 	WARN_ON(ret < 0);
3719 	ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "printk:online",
3720 					console_cpu_notify, NULL);
3721 	WARN_ON(ret < 0);
3722 	printk_sysctl_init();
3723 	return 0;
3724 }
3725 late_initcall(printk_late_init);
3726 
3727 #if defined CONFIG_PRINTK
3728 /* If @con is specified, only wait for that console. Otherwise wait for all. */
__pr_flush(struct console * con,int timeout_ms,bool reset_on_progress)3729 static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progress)
3730 {
3731 	int remaining = timeout_ms;
3732 	struct console *c;
3733 	u64 last_diff = 0;
3734 	u64 printk_seq;
3735 	int cookie;
3736 	u64 diff;
3737 	u64 seq;
3738 
3739 	might_sleep();
3740 
3741 	seq = prb_next_seq(prb);
3742 
3743 	/* Flush the consoles so that records up to @seq are printed. */
3744 	console_lock();
3745 	console_unlock();
3746 
3747 	for (;;) {
3748 		diff = 0;
3749 
3750 		/*
3751 		 * Hold the console_lock to guarantee safe access to
3752 		 * console->seq. Releasing console_lock flushes more
3753 		 * records in case @seq is still not printed on all
3754 		 * usable consoles.
3755 		 */
3756 		console_lock();
3757 
3758 		cookie = console_srcu_read_lock();
3759 		for_each_console_srcu(c) {
3760 			if (con && con != c)
3761 				continue;
3762 			/*
3763 			 * If consoles are not usable, it cannot be expected
3764 			 * that they make forward progress, so only increment
3765 			 * @diff for usable consoles.
3766 			 */
3767 			if (!console_is_usable(c))
3768 				continue;
3769 			printk_seq = c->seq;
3770 			if (printk_seq < seq)
3771 				diff += seq - printk_seq;
3772 		}
3773 		console_srcu_read_unlock(cookie);
3774 
3775 		if (diff != last_diff && reset_on_progress)
3776 			remaining = timeout_ms;
3777 
3778 		console_unlock();
3779 
3780 		/* Note: @diff is 0 if there are no usable consoles. */
3781 		if (diff == 0 || remaining == 0)
3782 			break;
3783 
3784 		if (remaining < 0) {
3785 			/* no timeout limit */
3786 			msleep(100);
3787 		} else if (remaining < 100) {
3788 			msleep(remaining);
3789 			remaining = 0;
3790 		} else {
3791 			msleep(100);
3792 			remaining -= 100;
3793 		}
3794 
3795 		last_diff = diff;
3796 	}
3797 
3798 	return (diff == 0);
3799 }
3800 
3801 /**
3802  * pr_flush() - Wait for printing threads to catch up.
3803  *
3804  * @timeout_ms:        The maximum time (in ms) to wait.
3805  * @reset_on_progress: Reset the timeout if forward progress is seen.
3806  *
3807  * A value of 0 for @timeout_ms means no waiting will occur. A value of -1
3808  * represents infinite waiting.
3809  *
3810  * If @reset_on_progress is true, the timeout will be reset whenever any
3811  * printer has been seen to make some forward progress.
3812  *
3813  * Context: Process context. May sleep while acquiring console lock.
3814  * Return: true if all usable printers are caught up.
3815  */
pr_flush(int timeout_ms,bool reset_on_progress)3816 static bool pr_flush(int timeout_ms, bool reset_on_progress)
3817 {
3818 	return __pr_flush(NULL, timeout_ms, reset_on_progress);
3819 }
3820 
3821 /*
3822  * Delayed printk version, for scheduler-internal messages:
3823  */
3824 #define PRINTK_PENDING_WAKEUP	0x01
3825 #define PRINTK_PENDING_OUTPUT	0x02
3826 
3827 static DEFINE_PER_CPU(int, printk_pending);
3828 
wake_up_klogd_work_func(struct irq_work * irq_work)3829 static void wake_up_klogd_work_func(struct irq_work *irq_work)
3830 {
3831 	int pending = this_cpu_xchg(printk_pending, 0);
3832 
3833 	if (pending & PRINTK_PENDING_OUTPUT) {
3834 		/* If trylock fails, someone else is doing the printing */
3835 		if (console_trylock())
3836 			console_unlock();
3837 	}
3838 
3839 	if (pending & PRINTK_PENDING_WAKEUP)
3840 		wake_up_interruptible(&log_wait);
3841 }
3842 
3843 static DEFINE_PER_CPU(struct irq_work, wake_up_klogd_work) =
3844 	IRQ_WORK_INIT_LAZY(wake_up_klogd_work_func);
3845 
__wake_up_klogd(int val)3846 static void __wake_up_klogd(int val)
3847 {
3848 	if (!printk_percpu_data_ready())
3849 		return;
3850 
3851 	preempt_disable();
3852 	/*
3853 	 * Guarantee any new records can be seen by tasks preparing to wait
3854 	 * before this context checks if the wait queue is empty.
3855 	 *
3856 	 * The full memory barrier within wq_has_sleeper() pairs with the full
3857 	 * memory barrier within set_current_state() of
3858 	 * prepare_to_wait_event(), which is called after ___wait_event() adds
3859 	 * the waiter but before it has checked the wait condition.
3860 	 *
3861 	 * This pairs with devkmsg_read:A and syslog_print:A.
3862 	 */
3863 	if (wq_has_sleeper(&log_wait) || /* LMM(__wake_up_klogd:A) */
3864 	    (val & PRINTK_PENDING_OUTPUT)) {
3865 		this_cpu_or(printk_pending, val);
3866 		irq_work_queue(this_cpu_ptr(&wake_up_klogd_work));
3867 	}
3868 	preempt_enable();
3869 }
3870 
3871 /**
3872  * wake_up_klogd - Wake kernel logging daemon
3873  *
3874  * Use this function when new records have been added to the ringbuffer
3875  * and the console printing of those records has already occurred or is
3876  * known to be handled by some other context. This function will only
3877  * wake the logging daemon.
3878  *
3879  * Context: Any context.
3880  */
wake_up_klogd(void)3881 void wake_up_klogd(void)
3882 {
3883 	__wake_up_klogd(PRINTK_PENDING_WAKEUP);
3884 }
3885 
3886 /**
3887  * defer_console_output - Wake kernel logging daemon and trigger
3888  *	console printing in a deferred context
3889  *
3890  * Use this function when new records have been added to the ringbuffer,
3891  * this context is responsible for console printing those records, but
3892  * the current context is not allowed to perform the console printing.
3893  * Trigger an irq_work context to perform the console printing. This
3894  * function also wakes the logging daemon.
3895  *
3896  * Context: Any context.
3897  */
defer_console_output(void)3898 void defer_console_output(void)
3899 {
3900 	/*
3901 	 * New messages may have been added directly to the ringbuffer
3902 	 * using vprintk_store(), so wake any waiters as well.
3903 	 */
3904 	__wake_up_klogd(PRINTK_PENDING_WAKEUP | PRINTK_PENDING_OUTPUT);
3905 }
3906 
printk_trigger_flush(void)3907 void printk_trigger_flush(void)
3908 {
3909 	defer_console_output();
3910 }
3911 
vprintk_deferred(const char * fmt,va_list args)3912 int vprintk_deferred(const char *fmt, va_list args)
3913 {
3914 	return vprintk_emit(0, LOGLEVEL_SCHED, NULL, fmt, args);
3915 }
3916 
_printk_deferred(const char * fmt,...)3917 int _printk_deferred(const char *fmt, ...)
3918 {
3919 	va_list args;
3920 	int r;
3921 
3922 	va_start(args, fmt);
3923 	r = vprintk_deferred(fmt, args);
3924 	va_end(args);
3925 
3926 	return r;
3927 }
3928 
3929 /*
3930  * printk rate limiting, lifted from the networking subsystem.
3931  *
3932  * This enforces a rate limit: not more than 10 kernel messages
3933  * every 5s to make a denial-of-service attack impossible.
3934  */
3935 DEFINE_RATELIMIT_STATE(printk_ratelimit_state, 5 * HZ, 10);
3936 
__printk_ratelimit(const char * func)3937 int __printk_ratelimit(const char *func)
3938 {
3939 	return ___ratelimit(&printk_ratelimit_state, func);
3940 }
3941 EXPORT_SYMBOL(__printk_ratelimit);
3942 
3943 /**
3944  * printk_timed_ratelimit - caller-controlled printk ratelimiting
3945  * @caller_jiffies: pointer to caller's state
3946  * @interval_msecs: minimum interval between prints
3947  *
3948  * printk_timed_ratelimit() returns true if more than @interval_msecs
3949  * milliseconds have elapsed since the last time printk_timed_ratelimit()
3950  * returned true.
3951  */
printk_timed_ratelimit(unsigned long * caller_jiffies,unsigned int interval_msecs)3952 bool printk_timed_ratelimit(unsigned long *caller_jiffies,
3953 			unsigned int interval_msecs)
3954 {
3955 	unsigned long elapsed = jiffies - *caller_jiffies;
3956 
3957 	if (*caller_jiffies && elapsed <= msecs_to_jiffies(interval_msecs))
3958 		return false;
3959 
3960 	*caller_jiffies = jiffies;
3961 	return true;
3962 }
3963 EXPORT_SYMBOL(printk_timed_ratelimit);
3964 
3965 static DEFINE_SPINLOCK(dump_list_lock);
3966 static LIST_HEAD(dump_list);
3967 
3968 /**
3969  * kmsg_dump_register - register a kernel log dumper.
3970  * @dumper: pointer to the kmsg_dumper structure
3971  *
3972  * Adds a kernel log dumper to the system. The dump callback in the
3973  * structure will be called when the kernel oopses or panics and must be
3974  * set. Returns zero on success and %-EINVAL or %-EBUSY otherwise.
3975  */
kmsg_dump_register(struct kmsg_dumper * dumper)3976 int kmsg_dump_register(struct kmsg_dumper *dumper)
3977 {
3978 	unsigned long flags;
3979 	int err = -EBUSY;
3980 
3981 	/* The dump callback needs to be set */
3982 	if (!dumper->dump)
3983 		return -EINVAL;
3984 
3985 	spin_lock_irqsave(&dump_list_lock, flags);
3986 	/* Don't allow registering multiple times */
3987 	if (!dumper->registered) {
3988 		dumper->registered = 1;
3989 		list_add_tail_rcu(&dumper->list, &dump_list);
3990 		err = 0;
3991 	}
3992 	spin_unlock_irqrestore(&dump_list_lock, flags);
3993 
3994 	return err;
3995 }
3996 EXPORT_SYMBOL_GPL(kmsg_dump_register);
3997 
3998 /**
3999  * kmsg_dump_unregister - unregister a kmsg dumper.
4000  * @dumper: pointer to the kmsg_dumper structure
4001  *
4002  * Removes a dump device from the system. Returns zero on success and
4003  * %-EINVAL otherwise.
4004  */
kmsg_dump_unregister(struct kmsg_dumper * dumper)4005 int kmsg_dump_unregister(struct kmsg_dumper *dumper)
4006 {
4007 	unsigned long flags;
4008 	int err = -EINVAL;
4009 
4010 	spin_lock_irqsave(&dump_list_lock, flags);
4011 	if (dumper->registered) {
4012 		dumper->registered = 0;
4013 		list_del_rcu(&dumper->list);
4014 		err = 0;
4015 	}
4016 	spin_unlock_irqrestore(&dump_list_lock, flags);
4017 	synchronize_rcu();
4018 
4019 	return err;
4020 }
4021 EXPORT_SYMBOL_GPL(kmsg_dump_unregister);
4022 
4023 static bool always_kmsg_dump;
4024 module_param_named(always_kmsg_dump, always_kmsg_dump, bool, S_IRUGO | S_IWUSR);
4025 
kmsg_dump_reason_str(enum kmsg_dump_reason reason)4026 const char *kmsg_dump_reason_str(enum kmsg_dump_reason reason)
4027 {
4028 	switch (reason) {
4029 	case KMSG_DUMP_PANIC:
4030 		return "Panic";
4031 	case KMSG_DUMP_OOPS:
4032 		return "Oops";
4033 	case KMSG_DUMP_EMERG:
4034 		return "Emergency";
4035 	case KMSG_DUMP_SHUTDOWN:
4036 		return "Shutdown";
4037 	default:
4038 		return "Unknown";
4039 	}
4040 }
4041 EXPORT_SYMBOL_GPL(kmsg_dump_reason_str);
4042 
4043 /**
4044  * kmsg_dump - dump kernel log to kernel message dumpers.
4045  * @reason: the reason (oops, panic etc) for dumping
4046  *
4047  * Call each of the registered dumper's dump() callback, which can
4048  * retrieve the kmsg records with kmsg_dump_get_line() or
4049  * kmsg_dump_get_buffer().
4050  */
kmsg_dump(enum kmsg_dump_reason reason)4051 void kmsg_dump(enum kmsg_dump_reason reason)
4052 {
4053 	struct kmsg_dumper *dumper;
4054 
4055 	rcu_read_lock();
4056 	list_for_each_entry_rcu(dumper, &dump_list, list) {
4057 		enum kmsg_dump_reason max_reason = dumper->max_reason;
4058 
4059 		/*
4060 		 * If client has not provided a specific max_reason, default
4061 		 * to KMSG_DUMP_OOPS, unless always_kmsg_dump was set.
4062 		 */
4063 		if (max_reason == KMSG_DUMP_UNDEF) {
4064 			max_reason = always_kmsg_dump ? KMSG_DUMP_MAX :
4065 							KMSG_DUMP_OOPS;
4066 		}
4067 		if (reason > max_reason)
4068 			continue;
4069 
4070 		/* invoke dumper which will iterate over records */
4071 		dumper->dump(dumper, reason);
4072 	}
4073 	rcu_read_unlock();
4074 }
4075 
4076 /**
4077  * kmsg_dump_get_line - retrieve one kmsg log line
4078  * @iter: kmsg dump iterator
4079  * @syslog: include the "<4>" prefixes
4080  * @line: buffer to copy the line to
4081  * @size: maximum size of the buffer
4082  * @len: length of line placed into buffer
4083  *
4084  * Start at the beginning of the kmsg buffer, with the oldest kmsg
4085  * record, and copy one record into the provided buffer.
4086  *
4087  * Consecutive calls will return the next available record moving
4088  * towards the end of the buffer with the youngest messages.
4089  *
4090  * A return value of FALSE indicates that there are no more records to
4091  * read.
4092  */
kmsg_dump_get_line(struct kmsg_dump_iter * iter,bool syslog,char * line,size_t size,size_t * len)4093 bool kmsg_dump_get_line(struct kmsg_dump_iter *iter, bool syslog,
4094 			char *line, size_t size, size_t *len)
4095 {
4096 	u64 min_seq = latched_seq_read_nolock(&clear_seq);
4097 	struct printk_info info;
4098 	unsigned int line_count;
4099 	struct printk_record r;
4100 	size_t l = 0;
4101 	bool ret = false;
4102 
4103 	if (iter->cur_seq < min_seq)
4104 		iter->cur_seq = min_seq;
4105 
4106 	prb_rec_init_rd(&r, &info, line, size);
4107 
4108 	/* Read text or count text lines? */
4109 	if (line) {
4110 		if (!prb_read_valid(prb, iter->cur_seq, &r))
4111 			goto out;
4112 		l = record_print_text(&r, syslog, printk_time);
4113 	} else {
4114 		if (!prb_read_valid_info(prb, iter->cur_seq,
4115 					 &info, &line_count)) {
4116 			goto out;
4117 		}
4118 		l = get_record_print_text_size(&info, line_count, syslog,
4119 					       printk_time);
4120 
4121 	}
4122 
4123 	iter->cur_seq = r.info->seq + 1;
4124 	ret = true;
4125 out:
4126 	if (len)
4127 		*len = l;
4128 	return ret;
4129 }
4130 EXPORT_SYMBOL_GPL(kmsg_dump_get_line);
4131 
4132 /**
4133  * kmsg_dump_get_buffer - copy kmsg log lines
4134  * @iter: kmsg dump iterator
4135  * @syslog: include the "<4>" prefixes
4136  * @buf: buffer to copy the line to
4137  * @size: maximum size of the buffer
4138  * @len_out: length of line placed into buffer
4139  *
4140  * Start at the end of the kmsg buffer and fill the provided buffer
4141  * with as many of the *youngest* kmsg records that fit into it.
4142  * If the buffer is large enough, all available kmsg records will be
4143  * copied with a single call.
4144  *
4145  * Consecutive calls will fill the buffer with the next block of
4146  * available older records, not including the earlier retrieved ones.
4147  *
4148  * A return value of FALSE indicates that there are no more records to
4149  * read.
4150  */
kmsg_dump_get_buffer(struct kmsg_dump_iter * iter,bool syslog,char * buf,size_t size,size_t * len_out)4151 bool kmsg_dump_get_buffer(struct kmsg_dump_iter *iter, bool syslog,
4152 			  char *buf, size_t size, size_t *len_out)
4153 {
4154 	u64 min_seq = latched_seq_read_nolock(&clear_seq);
4155 	struct printk_info info;
4156 	struct printk_record r;
4157 	u64 seq;
4158 	u64 next_seq;
4159 	size_t len = 0;
4160 	bool ret = false;
4161 	bool time = printk_time;
4162 
4163 	if (!buf || !size)
4164 		goto out;
4165 
4166 	if (iter->cur_seq < min_seq)
4167 		iter->cur_seq = min_seq;
4168 
4169 	if (prb_read_valid_info(prb, iter->cur_seq, &info, NULL)) {
4170 		if (info.seq != iter->cur_seq) {
4171 			/* messages are gone, move to first available one */
4172 			iter->cur_seq = info.seq;
4173 		}
4174 	}
4175 
4176 	/* last entry */
4177 	if (iter->cur_seq >= iter->next_seq)
4178 		goto out;
4179 
4180 	/*
4181 	 * Find first record that fits, including all following records,
4182 	 * into the user-provided buffer for this dump. Pass in size-1
4183 	 * because this function (by way of record_print_text()) will
4184 	 * not write more than size-1 bytes of text into @buf.
4185 	 */
4186 	seq = find_first_fitting_seq(iter->cur_seq, iter->next_seq,
4187 				     size - 1, syslog, time);
4188 
4189 	/*
4190 	 * Next kmsg_dump_get_buffer() invocation will dump block of
4191 	 * older records stored right before this one.
4192 	 */
4193 	next_seq = seq;
4194 
4195 	prb_rec_init_rd(&r, &info, buf, size);
4196 
4197 	len = 0;
4198 	prb_for_each_record(seq, prb, seq, &r) {
4199 		if (r.info->seq >= iter->next_seq)
4200 			break;
4201 
4202 		len += record_print_text(&r, syslog, time);
4203 
4204 		/* Adjust record to store to remaining buffer space. */
4205 		prb_rec_init_rd(&r, &info, buf + len, size - len);
4206 	}
4207 
4208 	iter->next_seq = next_seq;
4209 	ret = true;
4210 out:
4211 	if (len_out)
4212 		*len_out = len;
4213 	return ret;
4214 }
4215 EXPORT_SYMBOL_GPL(kmsg_dump_get_buffer);
4216 
4217 /**
4218  * kmsg_dump_rewind - reset the iterator
4219  * @iter: kmsg dump iterator
4220  *
4221  * Reset the dumper's iterator so that kmsg_dump_get_line() and
4222  * kmsg_dump_get_buffer() can be called again and used multiple
4223  * times within the same dumper.dump() callback.
4224  */
kmsg_dump_rewind(struct kmsg_dump_iter * iter)4225 void kmsg_dump_rewind(struct kmsg_dump_iter *iter)
4226 {
4227 	iter->cur_seq = latched_seq_read_nolock(&clear_seq);
4228 	iter->next_seq = prb_next_seq(prb);
4229 }
4230 EXPORT_SYMBOL_GPL(kmsg_dump_rewind);
4231 
4232 #endif
4233 
4234 #ifdef CONFIG_SMP
4235 static atomic_t printk_cpu_sync_owner = ATOMIC_INIT(-1);
4236 static atomic_t printk_cpu_sync_nested = ATOMIC_INIT(0);
4237 
4238 /**
4239  * __printk_cpu_sync_wait() - Busy wait until the printk cpu-reentrant
4240  *                            spinning lock is not owned by any CPU.
4241  *
4242  * Context: Any context.
4243  */
__printk_cpu_sync_wait(void)4244 void __printk_cpu_sync_wait(void)
4245 {
4246 	do {
4247 		cpu_relax();
4248 	} while (atomic_read(&printk_cpu_sync_owner) != -1);
4249 }
4250 EXPORT_SYMBOL(__printk_cpu_sync_wait);
4251 
4252 /**
4253  * __printk_cpu_sync_try_get() - Try to acquire the printk cpu-reentrant
4254  *                               spinning lock.
4255  *
4256  * If no processor has the lock, the calling processor takes the lock and
4257  * becomes the owner. If the calling processor is already the owner of the
4258  * lock, this function succeeds immediately.
4259  *
4260  * Context: Any context. Expects interrupts to be disabled.
4261  * Return: 1 on success, otherwise 0.
4262  */
__printk_cpu_sync_try_get(void)4263 int __printk_cpu_sync_try_get(void)
4264 {
4265 	int cpu;
4266 	int old;
4267 
4268 	cpu = smp_processor_id();
4269 
4270 	/*
4271 	 * Guarantee loads and stores from this CPU when it is the lock owner
4272 	 * are _not_ visible to the previous lock owner. This pairs with
4273 	 * __printk_cpu_sync_put:B.
4274 	 *
4275 	 * Memory barrier involvement:
4276 	 *
4277 	 * If __printk_cpu_sync_try_get:A reads from __printk_cpu_sync_put:B,
4278 	 * then __printk_cpu_sync_put:A can never read from
4279 	 * __printk_cpu_sync_try_get:B.
4280 	 *
4281 	 * Relies on:
4282 	 *
4283 	 * RELEASE from __printk_cpu_sync_put:A to __printk_cpu_sync_put:B
4284 	 * of the previous CPU
4285 	 *    matching
4286 	 * ACQUIRE from __printk_cpu_sync_try_get:A to
4287 	 * __printk_cpu_sync_try_get:B of this CPU
4288 	 */
4289 	old = atomic_cmpxchg_acquire(&printk_cpu_sync_owner, -1,
4290 				     cpu); /* LMM(__printk_cpu_sync_try_get:A) */
4291 	if (old == -1) {
4292 		/*
4293 		 * This CPU is now the owner and begins loading/storing
4294 		 * data: LMM(__printk_cpu_sync_try_get:B)
4295 		 */
4296 		return 1;
4297 
4298 	} else if (old == cpu) {
4299 		/* This CPU is already the owner. */
4300 		atomic_inc(&printk_cpu_sync_nested);
4301 		return 1;
4302 	}
4303 
4304 	return 0;
4305 }
4306 EXPORT_SYMBOL(__printk_cpu_sync_try_get);
4307 
4308 /**
4309  * __printk_cpu_sync_put() - Release the printk cpu-reentrant spinning lock.
4310  *
4311  * The calling processor must be the owner of the lock.
4312  *
4313  * Context: Any context. Expects interrupts to be disabled.
4314  */
__printk_cpu_sync_put(void)4315 void __printk_cpu_sync_put(void)
4316 {
4317 	if (atomic_read(&printk_cpu_sync_nested)) {
4318 		atomic_dec(&printk_cpu_sync_nested);
4319 		return;
4320 	}
4321 
4322 	/*
4323 	 * This CPU is finished loading/storing data:
4324 	 * LMM(__printk_cpu_sync_put:A)
4325 	 */
4326 
4327 	/*
4328 	 * Guarantee loads and stores from this CPU when it was the
4329 	 * lock owner are visible to the next lock owner. This pairs
4330 	 * with __printk_cpu_sync_try_get:A.
4331 	 *
4332 	 * Memory barrier involvement:
4333 	 *
4334 	 * If __printk_cpu_sync_try_get:A reads from __printk_cpu_sync_put:B,
4335 	 * then __printk_cpu_sync_try_get:B reads from __printk_cpu_sync_put:A.
4336 	 *
4337 	 * Relies on:
4338 	 *
4339 	 * RELEASE from __printk_cpu_sync_put:A to __printk_cpu_sync_put:B
4340 	 * of this CPU
4341 	 *    matching
4342 	 * ACQUIRE from __printk_cpu_sync_try_get:A to
4343 	 * __printk_cpu_sync_try_get:B of the next CPU
4344 	 */
4345 	atomic_set_release(&printk_cpu_sync_owner,
4346 			   -1); /* LMM(__printk_cpu_sync_put:B) */
4347 }
4348 EXPORT_SYMBOL(__printk_cpu_sync_put);
4349 #endif /* CONFIG_SMP */
4350