1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * linux/kernel/printk.c
4 *
5 * Copyright (C) 1991, 1992 Linus Torvalds
6 *
7 * Modified to make sys_syslog() more flexible: added commands to
8 * return the last 4k of kernel messages, regardless of whether
9 * they've been read or not. Added option to suppress kernel printk's
10 * to the console. Added hook for sending the console messages
11 * elsewhere, in preparation for a serial line console (someday).
12 * Ted Ts'o, 2/11/93.
13 * Modified for sysctl support, 1/8/97, Chris Horn.
14 * Fixed SMP synchronization, 08/08/99, Manfred Spraul
15 * manfred@colorfullife.com
16 * Rewrote bits to get rid of console_lock
17 * 01Mar01 Andrew Morton
18 */
19
20 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
21
22 #include <linux/kernel.h>
23 #include <linux/mm.h>
24 #include <linux/tty.h>
25 #include <linux/tty_driver.h>
26 #include <linux/console.h>
27 #include <linux/init.h>
28 #include <linux/jiffies.h>
29 #include <linux/nmi.h>
30 #include <linux/module.h>
31 #include <linux/moduleparam.h>
32 #include <linux/delay.h>
33 #include <linux/smp.h>
34 #include <linux/security.h>
35 #include <linux/memblock.h>
36 #include <linux/syscalls.h>
37 #include <linux/crash_core.h>
38 #include <linux/ratelimit.h>
39 #include <linux/kmsg_dump.h>
40 #include <linux/syslog.h>
41 #include <linux/cpu.h>
42 #include <linux/rculist.h>
43 #include <linux/poll.h>
44 #include <linux/irq_work.h>
45 #include <linux/ctype.h>
46 #include <linux/uio.h>
47 #include <linux/sched/clock.h>
48 #include <linux/sched/debug.h>
49 #include <linux/sched/task_stack.h>
50
51 #include <linux/uaccess.h>
52 #include <asm/sections.h>
53
54 #include <trace/events/initcall.h>
55 #define CREATE_TRACE_POINTS
56 #include <trace/events/printk.h>
57
58 #include "printk_ringbuffer.h"
59 #include "console_cmdline.h"
60 #include "braille.h"
61 #include "internal.h"
62
63 int console_printk[4] = {
64 CONSOLE_LOGLEVEL_DEFAULT, /* console_loglevel */
65 MESSAGE_LOGLEVEL_DEFAULT, /* default_message_loglevel */
66 CONSOLE_LOGLEVEL_MIN, /* minimum_console_loglevel */
67 CONSOLE_LOGLEVEL_DEFAULT, /* default_console_loglevel */
68 };
69 EXPORT_SYMBOL_GPL(console_printk);
70
71 atomic_t ignore_console_lock_warning __read_mostly = ATOMIC_INIT(0);
72 EXPORT_SYMBOL(ignore_console_lock_warning);
73
74 EXPORT_TRACEPOINT_SYMBOL_GPL(console);
75
76 /*
77 * Low level drivers may need that to know if they can schedule in
78 * their unblank() callback or not. So let's export it.
79 */
80 int oops_in_progress;
81 EXPORT_SYMBOL(oops_in_progress);
82
83 /*
84 * console_mutex protects console_list updates and console->flags updates.
85 * The flags are synchronized only for consoles that are registered, i.e.
86 * accessible via the console list.
87 */
88 static DEFINE_MUTEX(console_mutex);
89
90 /*
91 * console_sem protects updates to console->seq
92 * and also provides serialization for console printing.
93 */
94 static DEFINE_SEMAPHORE(console_sem, 1);
95 HLIST_HEAD(console_list);
96 EXPORT_SYMBOL_GPL(console_list);
97 DEFINE_STATIC_SRCU(console_srcu);
98
99 /*
100 * System may need to suppress printk message under certain
101 * circumstances, like after kernel panic happens.
102 */
103 int __read_mostly suppress_printk;
104
105 /*
106 * During panic, heavy printk by other CPUs can delay the
107 * panic and risk deadlock on console resources.
108 */
109 static int __read_mostly suppress_panic_printk;
110
111 #ifdef CONFIG_LOCKDEP
112 static struct lockdep_map console_lock_dep_map = {
113 .name = "console_lock"
114 };
115
lockdep_assert_console_list_lock_held(void)116 void lockdep_assert_console_list_lock_held(void)
117 {
118 lockdep_assert_held(&console_mutex);
119 }
120 EXPORT_SYMBOL(lockdep_assert_console_list_lock_held);
121 #endif
122
123 #ifdef CONFIG_DEBUG_LOCK_ALLOC
console_srcu_read_lock_is_held(void)124 bool console_srcu_read_lock_is_held(void)
125 {
126 return srcu_read_lock_held(&console_srcu);
127 }
128 EXPORT_SYMBOL(console_srcu_read_lock_is_held);
129 #endif
130
131 enum devkmsg_log_bits {
132 __DEVKMSG_LOG_BIT_ON = 0,
133 __DEVKMSG_LOG_BIT_OFF,
134 __DEVKMSG_LOG_BIT_LOCK,
135 };
136
137 enum devkmsg_log_masks {
138 DEVKMSG_LOG_MASK_ON = BIT(__DEVKMSG_LOG_BIT_ON),
139 DEVKMSG_LOG_MASK_OFF = BIT(__DEVKMSG_LOG_BIT_OFF),
140 DEVKMSG_LOG_MASK_LOCK = BIT(__DEVKMSG_LOG_BIT_LOCK),
141 };
142
143 /* Keep both the 'on' and 'off' bits clear, i.e. ratelimit by default: */
144 #define DEVKMSG_LOG_MASK_DEFAULT 0
145
146 static unsigned int __read_mostly devkmsg_log = DEVKMSG_LOG_MASK_DEFAULT;
147
__control_devkmsg(char * str)148 static int __control_devkmsg(char *str)
149 {
150 size_t len;
151
152 if (!str)
153 return -EINVAL;
154
155 len = str_has_prefix(str, "on");
156 if (len) {
157 devkmsg_log = DEVKMSG_LOG_MASK_ON;
158 return len;
159 }
160
161 len = str_has_prefix(str, "off");
162 if (len) {
163 devkmsg_log = DEVKMSG_LOG_MASK_OFF;
164 return len;
165 }
166
167 len = str_has_prefix(str, "ratelimit");
168 if (len) {
169 devkmsg_log = DEVKMSG_LOG_MASK_DEFAULT;
170 return len;
171 }
172
173 return -EINVAL;
174 }
175
control_devkmsg(char * str)176 static int __init control_devkmsg(char *str)
177 {
178 if (__control_devkmsg(str) < 0) {
179 pr_warn("printk.devkmsg: bad option string '%s'\n", str);
180 return 1;
181 }
182
183 /*
184 * Set sysctl string accordingly:
185 */
186 if (devkmsg_log == DEVKMSG_LOG_MASK_ON)
187 strcpy(devkmsg_log_str, "on");
188 else if (devkmsg_log == DEVKMSG_LOG_MASK_OFF)
189 strcpy(devkmsg_log_str, "off");
190 /* else "ratelimit" which is set by default. */
191
192 /*
193 * Sysctl cannot change it anymore. The kernel command line setting of
194 * this parameter is to force the setting to be permanent throughout the
195 * runtime of the system. This is a precation measure against userspace
196 * trying to be a smarta** and attempting to change it up on us.
197 */
198 devkmsg_log |= DEVKMSG_LOG_MASK_LOCK;
199
200 return 1;
201 }
202 __setup("printk.devkmsg=", control_devkmsg);
203
204 char devkmsg_log_str[DEVKMSG_STR_MAX_SIZE] = "ratelimit";
205 #if defined(CONFIG_PRINTK) && defined(CONFIG_SYSCTL)
devkmsg_sysctl_set_loglvl(struct ctl_table * table,int write,void * buffer,size_t * lenp,loff_t * ppos)206 int devkmsg_sysctl_set_loglvl(struct ctl_table *table, int write,
207 void *buffer, size_t *lenp, loff_t *ppos)
208 {
209 char old_str[DEVKMSG_STR_MAX_SIZE];
210 unsigned int old;
211 int err;
212
213 if (write) {
214 if (devkmsg_log & DEVKMSG_LOG_MASK_LOCK)
215 return -EINVAL;
216
217 old = devkmsg_log;
218 strncpy(old_str, devkmsg_log_str, DEVKMSG_STR_MAX_SIZE);
219 }
220
221 err = proc_dostring(table, write, buffer, lenp, ppos);
222 if (err)
223 return err;
224
225 if (write) {
226 err = __control_devkmsg(devkmsg_log_str);
227
228 /*
229 * Do not accept an unknown string OR a known string with
230 * trailing crap...
231 */
232 if (err < 0 || (err + 1 != *lenp)) {
233
234 /* ... and restore old setting. */
235 devkmsg_log = old;
236 strncpy(devkmsg_log_str, old_str, DEVKMSG_STR_MAX_SIZE);
237
238 return -EINVAL;
239 }
240 }
241
242 return 0;
243 }
244 #endif /* CONFIG_PRINTK && CONFIG_SYSCTL */
245
246 /**
247 * console_list_lock - Lock the console list
248 *
249 * For console list or console->flags updates
250 */
console_list_lock(void)251 void console_list_lock(void)
252 {
253 /*
254 * In unregister_console() and console_force_preferred_locked(),
255 * synchronize_srcu() is called with the console_list_lock held.
256 * Therefore it is not allowed that the console_list_lock is taken
257 * with the srcu_lock held.
258 *
259 * Detecting if this context is really in the read-side critical
260 * section is only possible if the appropriate debug options are
261 * enabled.
262 */
263 WARN_ON_ONCE(debug_lockdep_rcu_enabled() &&
264 srcu_read_lock_held(&console_srcu));
265
266 mutex_lock(&console_mutex);
267 }
268 EXPORT_SYMBOL(console_list_lock);
269
270 /**
271 * console_list_unlock - Unlock the console list
272 *
273 * Counterpart to console_list_lock()
274 */
console_list_unlock(void)275 void console_list_unlock(void)
276 {
277 mutex_unlock(&console_mutex);
278 }
279 EXPORT_SYMBOL(console_list_unlock);
280
281 /**
282 * console_srcu_read_lock - Register a new reader for the
283 * SRCU-protected console list
284 *
285 * Use for_each_console_srcu() to iterate the console list
286 *
287 * Context: Any context.
288 * Return: A cookie to pass to console_srcu_read_unlock().
289 */
console_srcu_read_lock(void)290 int console_srcu_read_lock(void)
291 {
292 return srcu_read_lock_nmisafe(&console_srcu);
293 }
294 EXPORT_SYMBOL(console_srcu_read_lock);
295
296 /**
297 * console_srcu_read_unlock - Unregister an old reader from
298 * the SRCU-protected console list
299 * @cookie: cookie returned from console_srcu_read_lock()
300 *
301 * Counterpart to console_srcu_read_lock()
302 */
console_srcu_read_unlock(int cookie)303 void console_srcu_read_unlock(int cookie)
304 {
305 srcu_read_unlock_nmisafe(&console_srcu, cookie);
306 }
307 EXPORT_SYMBOL(console_srcu_read_unlock);
308
309 /*
310 * Helper macros to handle lockdep when locking/unlocking console_sem. We use
311 * macros instead of functions so that _RET_IP_ contains useful information.
312 */
313 #define down_console_sem() do { \
314 down(&console_sem);\
315 mutex_acquire(&console_lock_dep_map, 0, 0, _RET_IP_);\
316 } while (0)
317
__down_trylock_console_sem(unsigned long ip)318 static int __down_trylock_console_sem(unsigned long ip)
319 {
320 int lock_failed;
321 unsigned long flags;
322
323 /*
324 * Here and in __up_console_sem() we need to be in safe mode,
325 * because spindump/WARN/etc from under console ->lock will
326 * deadlock in printk()->down_trylock_console_sem() otherwise.
327 */
328 printk_safe_enter_irqsave(flags);
329 lock_failed = down_trylock(&console_sem);
330 printk_safe_exit_irqrestore(flags);
331
332 if (lock_failed)
333 return 1;
334 mutex_acquire(&console_lock_dep_map, 0, 1, ip);
335 return 0;
336 }
337 #define down_trylock_console_sem() __down_trylock_console_sem(_RET_IP_)
338
__up_console_sem(unsigned long ip)339 static void __up_console_sem(unsigned long ip)
340 {
341 unsigned long flags;
342
343 mutex_release(&console_lock_dep_map, ip);
344
345 printk_safe_enter_irqsave(flags);
346 up(&console_sem);
347 printk_safe_exit_irqrestore(flags);
348 }
349 #define up_console_sem() __up_console_sem(_RET_IP_)
350
panic_in_progress(void)351 static bool panic_in_progress(void)
352 {
353 return unlikely(atomic_read(&panic_cpu) != PANIC_CPU_INVALID);
354 }
355
356 /*
357 * This is used for debugging the mess that is the VT code by
358 * keeping track if we have the console semaphore held. It's
359 * definitely not the perfect debug tool (we don't know if _WE_
360 * hold it and are racing, but it helps tracking those weird code
361 * paths in the console code where we end up in places I want
362 * locked without the console semaphore held).
363 */
364 static int console_locked;
365
366 /*
367 * Array of consoles built from command line options (console=)
368 */
369
370 #define MAX_CMDLINECONSOLES 8
371
372 static struct console_cmdline console_cmdline[MAX_CMDLINECONSOLES];
373
374 static int preferred_console = -1;
375 int console_set_on_cmdline;
376 EXPORT_SYMBOL(console_set_on_cmdline);
377
378 /* Flag: console code may call schedule() */
379 static int console_may_schedule;
380
381 enum con_msg_format_flags {
382 MSG_FORMAT_DEFAULT = 0,
383 MSG_FORMAT_SYSLOG = (1 << 0),
384 };
385
386 static int console_msg_format = MSG_FORMAT_DEFAULT;
387
388 /*
389 * The printk log buffer consists of a sequenced collection of records, each
390 * containing variable length message text. Every record also contains its
391 * own meta-data (@info).
392 *
393 * Every record meta-data carries the timestamp in microseconds, as well as
394 * the standard userspace syslog level and syslog facility. The usual kernel
395 * messages use LOG_KERN; userspace-injected messages always carry a matching
396 * syslog facility, by default LOG_USER. The origin of every message can be
397 * reliably determined that way.
398 *
399 * The human readable log message of a record is available in @text, the
400 * length of the message text in @text_len. The stored message is not
401 * terminated.
402 *
403 * Optionally, a record can carry a dictionary of properties (key/value
404 * pairs), to provide userspace with a machine-readable message context.
405 *
406 * Examples for well-defined, commonly used property names are:
407 * DEVICE=b12:8 device identifier
408 * b12:8 block dev_t
409 * c127:3 char dev_t
410 * n8 netdev ifindex
411 * +sound:card0 subsystem:devname
412 * SUBSYSTEM=pci driver-core subsystem name
413 *
414 * Valid characters in property names are [a-zA-Z0-9.-_]. Property names
415 * and values are terminated by a '\0' character.
416 *
417 * Example of record values:
418 * record.text_buf = "it's a line" (unterminated)
419 * record.info.seq = 56
420 * record.info.ts_nsec = 36863
421 * record.info.text_len = 11
422 * record.info.facility = 0 (LOG_KERN)
423 * record.info.flags = 0
424 * record.info.level = 3 (LOG_ERR)
425 * record.info.caller_id = 299 (task 299)
426 * record.info.dev_info.subsystem = "pci" (terminated)
427 * record.info.dev_info.device = "+pci:0000:00:01.0" (terminated)
428 *
429 * The 'struct printk_info' buffer must never be directly exported to
430 * userspace, it is a kernel-private implementation detail that might
431 * need to be changed in the future, when the requirements change.
432 *
433 * /dev/kmsg exports the structured data in the following line format:
434 * "<level>,<sequnum>,<timestamp>,<contflag>[,additional_values, ... ];<message text>\n"
435 *
436 * Users of the export format should ignore possible additional values
437 * separated by ',', and find the message after the ';' character.
438 *
439 * The optional key/value pairs are attached as continuation lines starting
440 * with a space character and terminated by a newline. All possible
441 * non-prinatable characters are escaped in the "\xff" notation.
442 */
443
444 /* syslog_lock protects syslog_* variables and write access to clear_seq. */
445 static DEFINE_MUTEX(syslog_lock);
446
447 #ifdef CONFIG_PRINTK
448 DECLARE_WAIT_QUEUE_HEAD(log_wait);
449 /* All 3 protected by @syslog_lock. */
450 /* the next printk record to read by syslog(READ) or /proc/kmsg */
451 static u64 syslog_seq;
452 static size_t syslog_partial;
453 static bool syslog_time;
454
455 struct latched_seq {
456 seqcount_latch_t latch;
457 u64 val[2];
458 };
459
460 /*
461 * The next printk record to read after the last 'clear' command. There are
462 * two copies (updated with seqcount_latch) so that reads can locklessly
463 * access a valid value. Writers are synchronized by @syslog_lock.
464 */
465 static struct latched_seq clear_seq = {
466 .latch = SEQCNT_LATCH_ZERO(clear_seq.latch),
467 .val[0] = 0,
468 .val[1] = 0,
469 };
470
471 #define LOG_LEVEL(v) ((v) & 0x07)
472 #define LOG_FACILITY(v) ((v) >> 3 & 0xff)
473
474 /* record buffer */
475 #define LOG_ALIGN __alignof__(unsigned long)
476 #define __LOG_BUF_LEN (1 << CONFIG_LOG_BUF_SHIFT)
477 #define LOG_BUF_LEN_MAX (u32)(1 << 31)
478 static char __log_buf[__LOG_BUF_LEN] __aligned(LOG_ALIGN);
479 static char *log_buf = __log_buf;
480 static u32 log_buf_len = __LOG_BUF_LEN;
481
482 /*
483 * Define the average message size. This only affects the number of
484 * descriptors that will be available. Underestimating is better than
485 * overestimating (too many available descriptors is better than not enough).
486 */
487 #define PRB_AVGBITS 5 /* 32 character average length */
488
489 #if CONFIG_LOG_BUF_SHIFT <= PRB_AVGBITS
490 #error CONFIG_LOG_BUF_SHIFT value too small.
491 #endif
492 _DEFINE_PRINTKRB(printk_rb_static, CONFIG_LOG_BUF_SHIFT - PRB_AVGBITS,
493 PRB_AVGBITS, &__log_buf[0]);
494
495 static struct printk_ringbuffer printk_rb_dynamic;
496
497 static struct printk_ringbuffer *prb = &printk_rb_static;
498
499 /*
500 * We cannot access per-CPU data (e.g. per-CPU flush irq_work) before
501 * per_cpu_areas are initialised. This variable is set to true when
502 * it's safe to access per-CPU data.
503 */
504 static bool __printk_percpu_data_ready __ro_after_init;
505
printk_percpu_data_ready(void)506 bool printk_percpu_data_ready(void)
507 {
508 return __printk_percpu_data_ready;
509 }
510
511 /* Must be called under syslog_lock. */
latched_seq_write(struct latched_seq * ls,u64 val)512 static void latched_seq_write(struct latched_seq *ls, u64 val)
513 {
514 raw_write_seqcount_latch(&ls->latch);
515 ls->val[0] = val;
516 raw_write_seqcount_latch(&ls->latch);
517 ls->val[1] = val;
518 }
519
520 /* Can be called from any context. */
latched_seq_read_nolock(struct latched_seq * ls)521 static u64 latched_seq_read_nolock(struct latched_seq *ls)
522 {
523 unsigned int seq;
524 unsigned int idx;
525 u64 val;
526
527 do {
528 seq = raw_read_seqcount_latch(&ls->latch);
529 idx = seq & 0x1;
530 val = ls->val[idx];
531 } while (raw_read_seqcount_latch_retry(&ls->latch, seq));
532
533 return val;
534 }
535
536 /* Return log buffer address */
log_buf_addr_get(void)537 char *log_buf_addr_get(void)
538 {
539 return log_buf;
540 }
541
542 /* Return log buffer size */
log_buf_len_get(void)543 u32 log_buf_len_get(void)
544 {
545 return log_buf_len;
546 }
547
548 /*
549 * Define how much of the log buffer we could take at maximum. The value
550 * must be greater than two. Note that only half of the buffer is available
551 * when the index points to the middle.
552 */
553 #define MAX_LOG_TAKE_PART 4
554 static const char trunc_msg[] = "<truncated>";
555
truncate_msg(u16 * text_len,u16 * trunc_msg_len)556 static void truncate_msg(u16 *text_len, u16 *trunc_msg_len)
557 {
558 /*
559 * The message should not take the whole buffer. Otherwise, it might
560 * get removed too soon.
561 */
562 u32 max_text_len = log_buf_len / MAX_LOG_TAKE_PART;
563
564 if (*text_len > max_text_len)
565 *text_len = max_text_len;
566
567 /* enable the warning message (if there is room) */
568 *trunc_msg_len = strlen(trunc_msg);
569 if (*text_len >= *trunc_msg_len)
570 *text_len -= *trunc_msg_len;
571 else
572 *trunc_msg_len = 0;
573 }
574
575 int dmesg_restrict = IS_ENABLED(CONFIG_SECURITY_DMESG_RESTRICT);
576
syslog_action_restricted(int type)577 static int syslog_action_restricted(int type)
578 {
579 if (dmesg_restrict)
580 return 1;
581 /*
582 * Unless restricted, we allow "read all" and "get buffer size"
583 * for everybody.
584 */
585 return type != SYSLOG_ACTION_READ_ALL &&
586 type != SYSLOG_ACTION_SIZE_BUFFER;
587 }
588
check_syslog_permissions(int type,int source)589 static int check_syslog_permissions(int type, int source)
590 {
591 /*
592 * If this is from /proc/kmsg and we've already opened it, then we've
593 * already done the capabilities checks at open time.
594 */
595 if (source == SYSLOG_FROM_PROC && type != SYSLOG_ACTION_OPEN)
596 goto ok;
597
598 if (syslog_action_restricted(type)) {
599 if (capable(CAP_SYSLOG))
600 goto ok;
601 /*
602 * For historical reasons, accept CAP_SYS_ADMIN too, with
603 * a warning.
604 */
605 if (capable(CAP_SYS_ADMIN)) {
606 pr_warn_once("%s (%d): Attempt to access syslog with "
607 "CAP_SYS_ADMIN but no CAP_SYSLOG "
608 "(deprecated).\n",
609 current->comm, task_pid_nr(current));
610 goto ok;
611 }
612 return -EPERM;
613 }
614 ok:
615 return security_syslog(type);
616 }
617
append_char(char ** pp,char * e,char c)618 static void append_char(char **pp, char *e, char c)
619 {
620 if (*pp < e)
621 *(*pp)++ = c;
622 }
623
info_print_ext_header(char * buf,size_t size,struct printk_info * info)624 static ssize_t info_print_ext_header(char *buf, size_t size,
625 struct printk_info *info)
626 {
627 u64 ts_usec = info->ts_nsec;
628 char caller[20];
629 #ifdef CONFIG_PRINTK_CALLER
630 u32 id = info->caller_id;
631
632 snprintf(caller, sizeof(caller), ",caller=%c%u",
633 id & 0x80000000 ? 'C' : 'T', id & ~0x80000000);
634 #else
635 caller[0] = '\0';
636 #endif
637
638 do_div(ts_usec, 1000);
639
640 return scnprintf(buf, size, "%u,%llu,%llu,%c%s;",
641 (info->facility << 3) | info->level, info->seq,
642 ts_usec, info->flags & LOG_CONT ? 'c' : '-', caller);
643 }
644
msg_add_ext_text(char * buf,size_t size,const char * text,size_t text_len,unsigned char endc)645 static ssize_t msg_add_ext_text(char *buf, size_t size,
646 const char *text, size_t text_len,
647 unsigned char endc)
648 {
649 char *p = buf, *e = buf + size;
650 size_t i;
651
652 /* escape non-printable characters */
653 for (i = 0; i < text_len; i++) {
654 unsigned char c = text[i];
655
656 if (c < ' ' || c >= 127 || c == '\\')
657 p += scnprintf(p, e - p, "\\x%02x", c);
658 else
659 append_char(&p, e, c);
660 }
661 append_char(&p, e, endc);
662
663 return p - buf;
664 }
665
msg_add_dict_text(char * buf,size_t size,const char * key,const char * val)666 static ssize_t msg_add_dict_text(char *buf, size_t size,
667 const char *key, const char *val)
668 {
669 size_t val_len = strlen(val);
670 ssize_t len;
671
672 if (!val_len)
673 return 0;
674
675 len = msg_add_ext_text(buf, size, "", 0, ' '); /* dict prefix */
676 len += msg_add_ext_text(buf + len, size - len, key, strlen(key), '=');
677 len += msg_add_ext_text(buf + len, size - len, val, val_len, '\n');
678
679 return len;
680 }
681
msg_print_ext_body(char * buf,size_t size,char * text,size_t text_len,struct dev_printk_info * dev_info)682 static ssize_t msg_print_ext_body(char *buf, size_t size,
683 char *text, size_t text_len,
684 struct dev_printk_info *dev_info)
685 {
686 ssize_t len;
687
688 len = msg_add_ext_text(buf, size, text, text_len, '\n');
689
690 if (!dev_info)
691 goto out;
692
693 len += msg_add_dict_text(buf + len, size - len, "SUBSYSTEM",
694 dev_info->subsystem);
695 len += msg_add_dict_text(buf + len, size - len, "DEVICE",
696 dev_info->device);
697 out:
698 return len;
699 }
700
701 static bool printk_get_next_message(struct printk_message *pmsg, u64 seq,
702 bool is_extended, bool may_supress);
703
704 /* /dev/kmsg - userspace message inject/listen interface */
705 struct devkmsg_user {
706 atomic64_t seq;
707 struct ratelimit_state rs;
708 struct mutex lock;
709 struct printk_buffers pbufs;
710 };
711
712 static __printf(3, 4) __cold
devkmsg_emit(int facility,int level,const char * fmt,...)713 int devkmsg_emit(int facility, int level, const char *fmt, ...)
714 {
715 va_list args;
716 int r;
717
718 va_start(args, fmt);
719 r = vprintk_emit(facility, level, NULL, fmt, args);
720 va_end(args);
721
722 return r;
723 }
724
devkmsg_write(struct kiocb * iocb,struct iov_iter * from)725 static ssize_t devkmsg_write(struct kiocb *iocb, struct iov_iter *from)
726 {
727 char *buf, *line;
728 int level = default_message_loglevel;
729 int facility = 1; /* LOG_USER */
730 struct file *file = iocb->ki_filp;
731 struct devkmsg_user *user = file->private_data;
732 size_t len = iov_iter_count(from);
733 ssize_t ret = len;
734
735 if (len > PRINTKRB_RECORD_MAX)
736 return -EINVAL;
737
738 /* Ignore when user logging is disabled. */
739 if (devkmsg_log & DEVKMSG_LOG_MASK_OFF)
740 return len;
741
742 /* Ratelimit when not explicitly enabled. */
743 if (!(devkmsg_log & DEVKMSG_LOG_MASK_ON)) {
744 if (!___ratelimit(&user->rs, current->comm))
745 return ret;
746 }
747
748 buf = kmalloc(len+1, GFP_KERNEL);
749 if (buf == NULL)
750 return -ENOMEM;
751
752 buf[len] = '\0';
753 if (!copy_from_iter_full(buf, len, from)) {
754 kfree(buf);
755 return -EFAULT;
756 }
757
758 /*
759 * Extract and skip the syslog prefix <[0-9]*>. Coming from userspace
760 * the decimal value represents 32bit, the lower 3 bit are the log
761 * level, the rest are the log facility.
762 *
763 * If no prefix or no userspace facility is specified, we
764 * enforce LOG_USER, to be able to reliably distinguish
765 * kernel-generated messages from userspace-injected ones.
766 */
767 line = buf;
768 if (line[0] == '<') {
769 char *endp = NULL;
770 unsigned int u;
771
772 u = simple_strtoul(line + 1, &endp, 10);
773 if (endp && endp[0] == '>') {
774 level = LOG_LEVEL(u);
775 if (LOG_FACILITY(u) != 0)
776 facility = LOG_FACILITY(u);
777 endp++;
778 line = endp;
779 }
780 }
781
782 devkmsg_emit(facility, level, "%s", line);
783 kfree(buf);
784 return ret;
785 }
786
devkmsg_read(struct file * file,char __user * buf,size_t count,loff_t * ppos)787 static ssize_t devkmsg_read(struct file *file, char __user *buf,
788 size_t count, loff_t *ppos)
789 {
790 struct devkmsg_user *user = file->private_data;
791 char *outbuf = &user->pbufs.outbuf[0];
792 struct printk_message pmsg = {
793 .pbufs = &user->pbufs,
794 };
795 ssize_t ret;
796
797 ret = mutex_lock_interruptible(&user->lock);
798 if (ret)
799 return ret;
800
801 if (!printk_get_next_message(&pmsg, atomic64_read(&user->seq), true, false)) {
802 if (file->f_flags & O_NONBLOCK) {
803 ret = -EAGAIN;
804 goto out;
805 }
806
807 /*
808 * Guarantee this task is visible on the waitqueue before
809 * checking the wake condition.
810 *
811 * The full memory barrier within set_current_state() of
812 * prepare_to_wait_event() pairs with the full memory barrier
813 * within wq_has_sleeper().
814 *
815 * This pairs with __wake_up_klogd:A.
816 */
817 ret = wait_event_interruptible(log_wait,
818 printk_get_next_message(&pmsg, atomic64_read(&user->seq), true,
819 false)); /* LMM(devkmsg_read:A) */
820 if (ret)
821 goto out;
822 }
823
824 if (pmsg.dropped) {
825 /* our last seen message is gone, return error and reset */
826 atomic64_set(&user->seq, pmsg.seq);
827 ret = -EPIPE;
828 goto out;
829 }
830
831 atomic64_set(&user->seq, pmsg.seq + 1);
832
833 if (pmsg.outbuf_len > count) {
834 ret = -EINVAL;
835 goto out;
836 }
837
838 if (copy_to_user(buf, outbuf, pmsg.outbuf_len)) {
839 ret = -EFAULT;
840 goto out;
841 }
842 ret = pmsg.outbuf_len;
843 out:
844 mutex_unlock(&user->lock);
845 return ret;
846 }
847
848 /*
849 * Be careful when modifying this function!!!
850 *
851 * Only few operations are supported because the device works only with the
852 * entire variable length messages (records). Non-standard values are
853 * returned in the other cases and has been this way for quite some time.
854 * User space applications might depend on this behavior.
855 */
devkmsg_llseek(struct file * file,loff_t offset,int whence)856 static loff_t devkmsg_llseek(struct file *file, loff_t offset, int whence)
857 {
858 struct devkmsg_user *user = file->private_data;
859 loff_t ret = 0;
860
861 if (offset)
862 return -ESPIPE;
863
864 switch (whence) {
865 case SEEK_SET:
866 /* the first record */
867 atomic64_set(&user->seq, prb_first_valid_seq(prb));
868 break;
869 case SEEK_DATA:
870 /*
871 * The first record after the last SYSLOG_ACTION_CLEAR,
872 * like issued by 'dmesg -c'. Reading /dev/kmsg itself
873 * changes no global state, and does not clear anything.
874 */
875 atomic64_set(&user->seq, latched_seq_read_nolock(&clear_seq));
876 break;
877 case SEEK_END:
878 /* after the last record */
879 atomic64_set(&user->seq, prb_next_seq(prb));
880 break;
881 default:
882 ret = -EINVAL;
883 }
884 return ret;
885 }
886
devkmsg_poll(struct file * file,poll_table * wait)887 static __poll_t devkmsg_poll(struct file *file, poll_table *wait)
888 {
889 struct devkmsg_user *user = file->private_data;
890 struct printk_info info;
891 __poll_t ret = 0;
892
893 poll_wait(file, &log_wait, wait);
894
895 if (prb_read_valid_info(prb, atomic64_read(&user->seq), &info, NULL)) {
896 /* return error when data has vanished underneath us */
897 if (info.seq != atomic64_read(&user->seq))
898 ret = EPOLLIN|EPOLLRDNORM|EPOLLERR|EPOLLPRI;
899 else
900 ret = EPOLLIN|EPOLLRDNORM;
901 }
902
903 return ret;
904 }
905
devkmsg_open(struct inode * inode,struct file * file)906 static int devkmsg_open(struct inode *inode, struct file *file)
907 {
908 struct devkmsg_user *user;
909 int err;
910
911 if (devkmsg_log & DEVKMSG_LOG_MASK_OFF)
912 return -EPERM;
913
914 /* write-only does not need any file context */
915 if ((file->f_flags & O_ACCMODE) != O_WRONLY) {
916 err = check_syslog_permissions(SYSLOG_ACTION_READ_ALL,
917 SYSLOG_FROM_READER);
918 if (err)
919 return err;
920 }
921
922 user = kvmalloc(sizeof(struct devkmsg_user), GFP_KERNEL);
923 if (!user)
924 return -ENOMEM;
925
926 ratelimit_default_init(&user->rs);
927 ratelimit_set_flags(&user->rs, RATELIMIT_MSG_ON_RELEASE);
928
929 mutex_init(&user->lock);
930
931 atomic64_set(&user->seq, prb_first_valid_seq(prb));
932
933 file->private_data = user;
934 return 0;
935 }
936
devkmsg_release(struct inode * inode,struct file * file)937 static int devkmsg_release(struct inode *inode, struct file *file)
938 {
939 struct devkmsg_user *user = file->private_data;
940
941 ratelimit_state_exit(&user->rs);
942
943 mutex_destroy(&user->lock);
944 kvfree(user);
945 return 0;
946 }
947
948 const struct file_operations kmsg_fops = {
949 .open = devkmsg_open,
950 .read = devkmsg_read,
951 .write_iter = devkmsg_write,
952 .llseek = devkmsg_llseek,
953 .poll = devkmsg_poll,
954 .release = devkmsg_release,
955 };
956
957 #ifdef CONFIG_CRASH_CORE
958 /*
959 * This appends the listed symbols to /proc/vmcore
960 *
961 * /proc/vmcore is used by various utilities, like crash and makedumpfile to
962 * obtain access to symbols that are otherwise very difficult to locate. These
963 * symbols are specifically used so that utilities can access and extract the
964 * dmesg log from a vmcore file after a crash.
965 */
log_buf_vmcoreinfo_setup(void)966 void log_buf_vmcoreinfo_setup(void)
967 {
968 struct dev_printk_info *dev_info = NULL;
969
970 VMCOREINFO_SYMBOL(prb);
971 VMCOREINFO_SYMBOL(printk_rb_static);
972 VMCOREINFO_SYMBOL(clear_seq);
973
974 /*
975 * Export struct size and field offsets. User space tools can
976 * parse it and detect any changes to structure down the line.
977 */
978
979 VMCOREINFO_STRUCT_SIZE(printk_ringbuffer);
980 VMCOREINFO_OFFSET(printk_ringbuffer, desc_ring);
981 VMCOREINFO_OFFSET(printk_ringbuffer, text_data_ring);
982 VMCOREINFO_OFFSET(printk_ringbuffer, fail);
983
984 VMCOREINFO_STRUCT_SIZE(prb_desc_ring);
985 VMCOREINFO_OFFSET(prb_desc_ring, count_bits);
986 VMCOREINFO_OFFSET(prb_desc_ring, descs);
987 VMCOREINFO_OFFSET(prb_desc_ring, infos);
988 VMCOREINFO_OFFSET(prb_desc_ring, head_id);
989 VMCOREINFO_OFFSET(prb_desc_ring, tail_id);
990
991 VMCOREINFO_STRUCT_SIZE(prb_desc);
992 VMCOREINFO_OFFSET(prb_desc, state_var);
993 VMCOREINFO_OFFSET(prb_desc, text_blk_lpos);
994
995 VMCOREINFO_STRUCT_SIZE(prb_data_blk_lpos);
996 VMCOREINFO_OFFSET(prb_data_blk_lpos, begin);
997 VMCOREINFO_OFFSET(prb_data_blk_lpos, next);
998
999 VMCOREINFO_STRUCT_SIZE(printk_info);
1000 VMCOREINFO_OFFSET(printk_info, seq);
1001 VMCOREINFO_OFFSET(printk_info, ts_nsec);
1002 VMCOREINFO_OFFSET(printk_info, text_len);
1003 VMCOREINFO_OFFSET(printk_info, caller_id);
1004 VMCOREINFO_OFFSET(printk_info, dev_info);
1005
1006 VMCOREINFO_STRUCT_SIZE(dev_printk_info);
1007 VMCOREINFO_OFFSET(dev_printk_info, subsystem);
1008 VMCOREINFO_LENGTH(printk_info_subsystem, sizeof(dev_info->subsystem));
1009 VMCOREINFO_OFFSET(dev_printk_info, device);
1010 VMCOREINFO_LENGTH(printk_info_device, sizeof(dev_info->device));
1011
1012 VMCOREINFO_STRUCT_SIZE(prb_data_ring);
1013 VMCOREINFO_OFFSET(prb_data_ring, size_bits);
1014 VMCOREINFO_OFFSET(prb_data_ring, data);
1015 VMCOREINFO_OFFSET(prb_data_ring, head_lpos);
1016 VMCOREINFO_OFFSET(prb_data_ring, tail_lpos);
1017
1018 VMCOREINFO_SIZE(atomic_long_t);
1019 VMCOREINFO_TYPE_OFFSET(atomic_long_t, counter);
1020
1021 VMCOREINFO_STRUCT_SIZE(latched_seq);
1022 VMCOREINFO_OFFSET(latched_seq, val);
1023 }
1024 #endif
1025
1026 /* requested log_buf_len from kernel cmdline */
1027 static unsigned long __initdata new_log_buf_len;
1028
1029 /* we practice scaling the ring buffer by powers of 2 */
log_buf_len_update(u64 size)1030 static void __init log_buf_len_update(u64 size)
1031 {
1032 if (size > (u64)LOG_BUF_LEN_MAX) {
1033 size = (u64)LOG_BUF_LEN_MAX;
1034 pr_err("log_buf over 2G is not supported.\n");
1035 }
1036
1037 if (size)
1038 size = roundup_pow_of_two(size);
1039 if (size > log_buf_len)
1040 new_log_buf_len = (unsigned long)size;
1041 }
1042
1043 /* save requested log_buf_len since it's too early to process it */
log_buf_len_setup(char * str)1044 static int __init log_buf_len_setup(char *str)
1045 {
1046 u64 size;
1047
1048 if (!str)
1049 return -EINVAL;
1050
1051 size = memparse(str, &str);
1052
1053 log_buf_len_update(size);
1054
1055 return 0;
1056 }
1057 early_param("log_buf_len", log_buf_len_setup);
1058
1059 #ifdef CONFIG_SMP
1060 #define __LOG_CPU_MAX_BUF_LEN (1 << CONFIG_LOG_CPU_MAX_BUF_SHIFT)
1061
log_buf_add_cpu(void)1062 static void __init log_buf_add_cpu(void)
1063 {
1064 unsigned int cpu_extra;
1065
1066 /*
1067 * archs should set up cpu_possible_bits properly with
1068 * set_cpu_possible() after setup_arch() but just in
1069 * case lets ensure this is valid.
1070 */
1071 if (num_possible_cpus() == 1)
1072 return;
1073
1074 cpu_extra = (num_possible_cpus() - 1) * __LOG_CPU_MAX_BUF_LEN;
1075
1076 /* by default this will only continue through for large > 64 CPUs */
1077 if (cpu_extra <= __LOG_BUF_LEN / 2)
1078 return;
1079
1080 pr_info("log_buf_len individual max cpu contribution: %d bytes\n",
1081 __LOG_CPU_MAX_BUF_LEN);
1082 pr_info("log_buf_len total cpu_extra contributions: %d bytes\n",
1083 cpu_extra);
1084 pr_info("log_buf_len min size: %d bytes\n", __LOG_BUF_LEN);
1085
1086 log_buf_len_update(cpu_extra + __LOG_BUF_LEN);
1087 }
1088 #else /* !CONFIG_SMP */
log_buf_add_cpu(void)1089 static inline void log_buf_add_cpu(void) {}
1090 #endif /* CONFIG_SMP */
1091
set_percpu_data_ready(void)1092 static void __init set_percpu_data_ready(void)
1093 {
1094 __printk_percpu_data_ready = true;
1095 }
1096
add_to_rb(struct printk_ringbuffer * rb,struct printk_record * r)1097 static unsigned int __init add_to_rb(struct printk_ringbuffer *rb,
1098 struct printk_record *r)
1099 {
1100 struct prb_reserved_entry e;
1101 struct printk_record dest_r;
1102
1103 prb_rec_init_wr(&dest_r, r->info->text_len);
1104
1105 if (!prb_reserve(&e, rb, &dest_r))
1106 return 0;
1107
1108 memcpy(&dest_r.text_buf[0], &r->text_buf[0], r->info->text_len);
1109 dest_r.info->text_len = r->info->text_len;
1110 dest_r.info->facility = r->info->facility;
1111 dest_r.info->level = r->info->level;
1112 dest_r.info->flags = r->info->flags;
1113 dest_r.info->ts_nsec = r->info->ts_nsec;
1114 dest_r.info->caller_id = r->info->caller_id;
1115 memcpy(&dest_r.info->dev_info, &r->info->dev_info, sizeof(dest_r.info->dev_info));
1116
1117 prb_final_commit(&e);
1118
1119 return prb_record_text_space(&e);
1120 }
1121
1122 static char setup_text_buf[PRINTKRB_RECORD_MAX] __initdata;
1123
setup_log_buf(int early)1124 void __init setup_log_buf(int early)
1125 {
1126 struct printk_info *new_infos;
1127 unsigned int new_descs_count;
1128 struct prb_desc *new_descs;
1129 struct printk_info info;
1130 struct printk_record r;
1131 unsigned int text_size;
1132 size_t new_descs_size;
1133 size_t new_infos_size;
1134 unsigned long flags;
1135 char *new_log_buf;
1136 unsigned int free;
1137 u64 seq;
1138
1139 /*
1140 * Some archs call setup_log_buf() multiple times - first is very
1141 * early, e.g. from setup_arch(), and second - when percpu_areas
1142 * are initialised.
1143 */
1144 if (!early)
1145 set_percpu_data_ready();
1146
1147 if (log_buf != __log_buf)
1148 return;
1149
1150 if (!early && !new_log_buf_len)
1151 log_buf_add_cpu();
1152
1153 if (!new_log_buf_len)
1154 return;
1155
1156 new_descs_count = new_log_buf_len >> PRB_AVGBITS;
1157 if (new_descs_count == 0) {
1158 pr_err("new_log_buf_len: %lu too small\n", new_log_buf_len);
1159 return;
1160 }
1161
1162 new_log_buf = memblock_alloc(new_log_buf_len, LOG_ALIGN);
1163 if (unlikely(!new_log_buf)) {
1164 pr_err("log_buf_len: %lu text bytes not available\n",
1165 new_log_buf_len);
1166 return;
1167 }
1168
1169 new_descs_size = new_descs_count * sizeof(struct prb_desc);
1170 new_descs = memblock_alloc(new_descs_size, LOG_ALIGN);
1171 if (unlikely(!new_descs)) {
1172 pr_err("log_buf_len: %zu desc bytes not available\n",
1173 new_descs_size);
1174 goto err_free_log_buf;
1175 }
1176
1177 new_infos_size = new_descs_count * sizeof(struct printk_info);
1178 new_infos = memblock_alloc(new_infos_size, LOG_ALIGN);
1179 if (unlikely(!new_infos)) {
1180 pr_err("log_buf_len: %zu info bytes not available\n",
1181 new_infos_size);
1182 goto err_free_descs;
1183 }
1184
1185 prb_rec_init_rd(&r, &info, &setup_text_buf[0], sizeof(setup_text_buf));
1186
1187 prb_init(&printk_rb_dynamic,
1188 new_log_buf, ilog2(new_log_buf_len),
1189 new_descs, ilog2(new_descs_count),
1190 new_infos);
1191
1192 local_irq_save(flags);
1193
1194 log_buf_len = new_log_buf_len;
1195 log_buf = new_log_buf;
1196 new_log_buf_len = 0;
1197
1198 free = __LOG_BUF_LEN;
1199 prb_for_each_record(0, &printk_rb_static, seq, &r) {
1200 text_size = add_to_rb(&printk_rb_dynamic, &r);
1201 if (text_size > free)
1202 free = 0;
1203 else
1204 free -= text_size;
1205 }
1206
1207 prb = &printk_rb_dynamic;
1208
1209 local_irq_restore(flags);
1210
1211 /*
1212 * Copy any remaining messages that might have appeared from
1213 * NMI context after copying but before switching to the
1214 * dynamic buffer.
1215 */
1216 prb_for_each_record(seq, &printk_rb_static, seq, &r) {
1217 text_size = add_to_rb(&printk_rb_dynamic, &r);
1218 if (text_size > free)
1219 free = 0;
1220 else
1221 free -= text_size;
1222 }
1223
1224 if (seq != prb_next_seq(&printk_rb_static)) {
1225 pr_err("dropped %llu messages\n",
1226 prb_next_seq(&printk_rb_static) - seq);
1227 }
1228
1229 pr_info("log_buf_len: %u bytes\n", log_buf_len);
1230 pr_info("early log buf free: %u(%u%%)\n",
1231 free, (free * 100) / __LOG_BUF_LEN);
1232 return;
1233
1234 err_free_descs:
1235 memblock_free(new_descs, new_descs_size);
1236 err_free_log_buf:
1237 memblock_free(new_log_buf, new_log_buf_len);
1238 }
1239
1240 static bool __read_mostly ignore_loglevel;
1241
ignore_loglevel_setup(char * str)1242 static int __init ignore_loglevel_setup(char *str)
1243 {
1244 ignore_loglevel = true;
1245 pr_info("debug: ignoring loglevel setting.\n");
1246
1247 return 0;
1248 }
1249
1250 early_param("ignore_loglevel", ignore_loglevel_setup);
1251 module_param(ignore_loglevel, bool, S_IRUGO | S_IWUSR);
1252 MODULE_PARM_DESC(ignore_loglevel,
1253 "ignore loglevel setting (prints all kernel messages to the console)");
1254
suppress_message_printing(int level)1255 static bool suppress_message_printing(int level)
1256 {
1257 return (level >= console_loglevel && !ignore_loglevel);
1258 }
1259
1260 #ifdef CONFIG_BOOT_PRINTK_DELAY
1261
1262 static int boot_delay; /* msecs delay after each printk during bootup */
1263 static unsigned long long loops_per_msec; /* based on boot_delay */
1264
boot_delay_setup(char * str)1265 static int __init boot_delay_setup(char *str)
1266 {
1267 unsigned long lpj;
1268
1269 lpj = preset_lpj ? preset_lpj : 1000000; /* some guess */
1270 loops_per_msec = (unsigned long long)lpj / 1000 * HZ;
1271
1272 get_option(&str, &boot_delay);
1273 if (boot_delay > 10 * 1000)
1274 boot_delay = 0;
1275
1276 pr_debug("boot_delay: %u, preset_lpj: %ld, lpj: %lu, "
1277 "HZ: %d, loops_per_msec: %llu\n",
1278 boot_delay, preset_lpj, lpj, HZ, loops_per_msec);
1279 return 0;
1280 }
1281 early_param("boot_delay", boot_delay_setup);
1282
boot_delay_msec(int level)1283 static void boot_delay_msec(int level)
1284 {
1285 unsigned long long k;
1286 unsigned long timeout;
1287
1288 if ((boot_delay == 0 || system_state >= SYSTEM_RUNNING)
1289 || suppress_message_printing(level)) {
1290 return;
1291 }
1292
1293 k = (unsigned long long)loops_per_msec * boot_delay;
1294
1295 timeout = jiffies + msecs_to_jiffies(boot_delay);
1296 while (k) {
1297 k--;
1298 cpu_relax();
1299 /*
1300 * use (volatile) jiffies to prevent
1301 * compiler reduction; loop termination via jiffies
1302 * is secondary and may or may not happen.
1303 */
1304 if (time_after(jiffies, timeout))
1305 break;
1306 touch_nmi_watchdog();
1307 }
1308 }
1309 #else
boot_delay_msec(int level)1310 static inline void boot_delay_msec(int level)
1311 {
1312 }
1313 #endif
1314
1315 static bool printk_time = IS_ENABLED(CONFIG_PRINTK_TIME);
1316 module_param_named(time, printk_time, bool, S_IRUGO | S_IWUSR);
1317
print_syslog(unsigned int level,char * buf)1318 static size_t print_syslog(unsigned int level, char *buf)
1319 {
1320 return sprintf(buf, "<%u>", level);
1321 }
1322
print_time(u64 ts,char * buf)1323 static size_t print_time(u64 ts, char *buf)
1324 {
1325 unsigned long rem_nsec = do_div(ts, 1000000000);
1326
1327 return sprintf(buf, "[%5lu.%06lu]",
1328 (unsigned long)ts, rem_nsec / 1000);
1329 }
1330
1331 #ifdef CONFIG_PRINTK_CALLER
print_caller(u32 id,char * buf)1332 static size_t print_caller(u32 id, char *buf)
1333 {
1334 char caller[12];
1335
1336 snprintf(caller, sizeof(caller), "%c%u",
1337 id & 0x80000000 ? 'C' : 'T', id & ~0x80000000);
1338 return sprintf(buf, "[%6s]", caller);
1339 }
1340 #else
1341 #define print_caller(id, buf) 0
1342 #endif
1343
info_print_prefix(const struct printk_info * info,bool syslog,bool time,char * buf)1344 static size_t info_print_prefix(const struct printk_info *info, bool syslog,
1345 bool time, char *buf)
1346 {
1347 size_t len = 0;
1348
1349 if (syslog)
1350 len = print_syslog((info->facility << 3) | info->level, buf);
1351
1352 if (time)
1353 len += print_time(info->ts_nsec, buf + len);
1354
1355 len += print_caller(info->caller_id, buf + len);
1356
1357 if (IS_ENABLED(CONFIG_PRINTK_CALLER) || time) {
1358 buf[len++] = ' ';
1359 buf[len] = '\0';
1360 }
1361
1362 return len;
1363 }
1364
1365 /*
1366 * Prepare the record for printing. The text is shifted within the given
1367 * buffer to avoid a need for another one. The following operations are
1368 * done:
1369 *
1370 * - Add prefix for each line.
1371 * - Drop truncated lines that no longer fit into the buffer.
1372 * - Add the trailing newline that has been removed in vprintk_store().
1373 * - Add a string terminator.
1374 *
1375 * Since the produced string is always terminated, the maximum possible
1376 * return value is @r->text_buf_size - 1;
1377 *
1378 * Return: The length of the updated/prepared text, including the added
1379 * prefixes and the newline. The terminator is not counted. The dropped
1380 * line(s) are not counted.
1381 */
record_print_text(struct printk_record * r,bool syslog,bool time)1382 static size_t record_print_text(struct printk_record *r, bool syslog,
1383 bool time)
1384 {
1385 size_t text_len = r->info->text_len;
1386 size_t buf_size = r->text_buf_size;
1387 char *text = r->text_buf;
1388 char prefix[PRINTK_PREFIX_MAX];
1389 bool truncated = false;
1390 size_t prefix_len;
1391 size_t line_len;
1392 size_t len = 0;
1393 char *next;
1394
1395 /*
1396 * If the message was truncated because the buffer was not large
1397 * enough, treat the available text as if it were the full text.
1398 */
1399 if (text_len > buf_size)
1400 text_len = buf_size;
1401
1402 prefix_len = info_print_prefix(r->info, syslog, time, prefix);
1403
1404 /*
1405 * @text_len: bytes of unprocessed text
1406 * @line_len: bytes of current line _without_ newline
1407 * @text: pointer to beginning of current line
1408 * @len: number of bytes prepared in r->text_buf
1409 */
1410 for (;;) {
1411 next = memchr(text, '\n', text_len);
1412 if (next) {
1413 line_len = next - text;
1414 } else {
1415 /* Drop truncated line(s). */
1416 if (truncated)
1417 break;
1418 line_len = text_len;
1419 }
1420
1421 /*
1422 * Truncate the text if there is not enough space to add the
1423 * prefix and a trailing newline and a terminator.
1424 */
1425 if (len + prefix_len + text_len + 1 + 1 > buf_size) {
1426 /* Drop even the current line if no space. */
1427 if (len + prefix_len + line_len + 1 + 1 > buf_size)
1428 break;
1429
1430 text_len = buf_size - len - prefix_len - 1 - 1;
1431 truncated = true;
1432 }
1433
1434 memmove(text + prefix_len, text, text_len);
1435 memcpy(text, prefix, prefix_len);
1436
1437 /*
1438 * Increment the prepared length to include the text and
1439 * prefix that were just moved+copied. Also increment for the
1440 * newline at the end of this line. If this is the last line,
1441 * there is no newline, but it will be added immediately below.
1442 */
1443 len += prefix_len + line_len + 1;
1444 if (text_len == line_len) {
1445 /*
1446 * This is the last line. Add the trailing newline
1447 * removed in vprintk_store().
1448 */
1449 text[prefix_len + line_len] = '\n';
1450 break;
1451 }
1452
1453 /*
1454 * Advance beyond the added prefix and the related line with
1455 * its newline.
1456 */
1457 text += prefix_len + line_len + 1;
1458
1459 /*
1460 * The remaining text has only decreased by the line with its
1461 * newline.
1462 *
1463 * Note that @text_len can become zero. It happens when @text
1464 * ended with a newline (either due to truncation or the
1465 * original string ending with "\n\n"). The loop is correctly
1466 * repeated and (if not truncated) an empty line with a prefix
1467 * will be prepared.
1468 */
1469 text_len -= line_len + 1;
1470 }
1471
1472 /*
1473 * If a buffer was provided, it will be terminated. Space for the
1474 * string terminator is guaranteed to be available. The terminator is
1475 * not counted in the return value.
1476 */
1477 if (buf_size > 0)
1478 r->text_buf[len] = 0;
1479
1480 return len;
1481 }
1482
get_record_print_text_size(struct printk_info * info,unsigned int line_count,bool syslog,bool time)1483 static size_t get_record_print_text_size(struct printk_info *info,
1484 unsigned int line_count,
1485 bool syslog, bool time)
1486 {
1487 char prefix[PRINTK_PREFIX_MAX];
1488 size_t prefix_len;
1489
1490 prefix_len = info_print_prefix(info, syslog, time, prefix);
1491
1492 /*
1493 * Each line will be preceded with a prefix. The intermediate
1494 * newlines are already within the text, but a final trailing
1495 * newline will be added.
1496 */
1497 return ((prefix_len * line_count) + info->text_len + 1);
1498 }
1499
1500 /*
1501 * Beginning with @start_seq, find the first record where it and all following
1502 * records up to (but not including) @max_seq fit into @size.
1503 *
1504 * @max_seq is simply an upper bound and does not need to exist. If the caller
1505 * does not require an upper bound, -1 can be used for @max_seq.
1506 */
find_first_fitting_seq(u64 start_seq,u64 max_seq,size_t size,bool syslog,bool time)1507 static u64 find_first_fitting_seq(u64 start_seq, u64 max_seq, size_t size,
1508 bool syslog, bool time)
1509 {
1510 struct printk_info info;
1511 unsigned int line_count;
1512 size_t len = 0;
1513 u64 seq;
1514
1515 /* Determine the size of the records up to @max_seq. */
1516 prb_for_each_info(start_seq, prb, seq, &info, &line_count) {
1517 if (info.seq >= max_seq)
1518 break;
1519 len += get_record_print_text_size(&info, line_count, syslog, time);
1520 }
1521
1522 /*
1523 * Adjust the upper bound for the next loop to avoid subtracting
1524 * lengths that were never added.
1525 */
1526 if (seq < max_seq)
1527 max_seq = seq;
1528
1529 /*
1530 * Move first record forward until length fits into the buffer. Ignore
1531 * newest messages that were not counted in the above cycle. Messages
1532 * might appear and get lost in the meantime. This is a best effort
1533 * that prevents an infinite loop that could occur with a retry.
1534 */
1535 prb_for_each_info(start_seq, prb, seq, &info, &line_count) {
1536 if (len <= size || info.seq >= max_seq)
1537 break;
1538 len -= get_record_print_text_size(&info, line_count, syslog, time);
1539 }
1540
1541 return seq;
1542 }
1543
1544 /* The caller is responsible for making sure @size is greater than 0. */
syslog_print(char __user * buf,int size)1545 static int syslog_print(char __user *buf, int size)
1546 {
1547 struct printk_info info;
1548 struct printk_record r;
1549 char *text;
1550 int len = 0;
1551 u64 seq;
1552
1553 text = kmalloc(PRINTK_MESSAGE_MAX, GFP_KERNEL);
1554 if (!text)
1555 return -ENOMEM;
1556
1557 prb_rec_init_rd(&r, &info, text, PRINTK_MESSAGE_MAX);
1558
1559 mutex_lock(&syslog_lock);
1560
1561 /*
1562 * Wait for the @syslog_seq record to be available. @syslog_seq may
1563 * change while waiting.
1564 */
1565 do {
1566 seq = syslog_seq;
1567
1568 mutex_unlock(&syslog_lock);
1569 /*
1570 * Guarantee this task is visible on the waitqueue before
1571 * checking the wake condition.
1572 *
1573 * The full memory barrier within set_current_state() of
1574 * prepare_to_wait_event() pairs with the full memory barrier
1575 * within wq_has_sleeper().
1576 *
1577 * This pairs with __wake_up_klogd:A.
1578 */
1579 len = wait_event_interruptible(log_wait,
1580 prb_read_valid(prb, seq, NULL)); /* LMM(syslog_print:A) */
1581 mutex_lock(&syslog_lock);
1582
1583 if (len)
1584 goto out;
1585 } while (syslog_seq != seq);
1586
1587 /*
1588 * Copy records that fit into the buffer. The above cycle makes sure
1589 * that the first record is always available.
1590 */
1591 do {
1592 size_t n;
1593 size_t skip;
1594 int err;
1595
1596 if (!prb_read_valid(prb, syslog_seq, &r))
1597 break;
1598
1599 if (r.info->seq != syslog_seq) {
1600 /* message is gone, move to next valid one */
1601 syslog_seq = r.info->seq;
1602 syslog_partial = 0;
1603 }
1604
1605 /*
1606 * To keep reading/counting partial line consistent,
1607 * use printk_time value as of the beginning of a line.
1608 */
1609 if (!syslog_partial)
1610 syslog_time = printk_time;
1611
1612 skip = syslog_partial;
1613 n = record_print_text(&r, true, syslog_time);
1614 if (n - syslog_partial <= size) {
1615 /* message fits into buffer, move forward */
1616 syslog_seq = r.info->seq + 1;
1617 n -= syslog_partial;
1618 syslog_partial = 0;
1619 } else if (!len){
1620 /* partial read(), remember position */
1621 n = size;
1622 syslog_partial += n;
1623 } else
1624 n = 0;
1625
1626 if (!n)
1627 break;
1628
1629 mutex_unlock(&syslog_lock);
1630 err = copy_to_user(buf, text + skip, n);
1631 mutex_lock(&syslog_lock);
1632
1633 if (err) {
1634 if (!len)
1635 len = -EFAULT;
1636 break;
1637 }
1638
1639 len += n;
1640 size -= n;
1641 buf += n;
1642 } while (size);
1643 out:
1644 mutex_unlock(&syslog_lock);
1645 kfree(text);
1646 return len;
1647 }
1648
syslog_print_all(char __user * buf,int size,bool clear)1649 static int syslog_print_all(char __user *buf, int size, bool clear)
1650 {
1651 struct printk_info info;
1652 struct printk_record r;
1653 char *text;
1654 int len = 0;
1655 u64 seq;
1656 bool time;
1657
1658 text = kmalloc(PRINTK_MESSAGE_MAX, GFP_KERNEL);
1659 if (!text)
1660 return -ENOMEM;
1661
1662 time = printk_time;
1663 /*
1664 * Find first record that fits, including all following records,
1665 * into the user-provided buffer for this dump.
1666 */
1667 seq = find_first_fitting_seq(latched_seq_read_nolock(&clear_seq), -1,
1668 size, true, time);
1669
1670 prb_rec_init_rd(&r, &info, text, PRINTK_MESSAGE_MAX);
1671
1672 len = 0;
1673 prb_for_each_record(seq, prb, seq, &r) {
1674 int textlen;
1675
1676 textlen = record_print_text(&r, true, time);
1677
1678 if (len + textlen > size) {
1679 seq--;
1680 break;
1681 }
1682
1683 if (copy_to_user(buf + len, text, textlen))
1684 len = -EFAULT;
1685 else
1686 len += textlen;
1687
1688 if (len < 0)
1689 break;
1690 }
1691
1692 if (clear) {
1693 mutex_lock(&syslog_lock);
1694 latched_seq_write(&clear_seq, seq);
1695 mutex_unlock(&syslog_lock);
1696 }
1697
1698 kfree(text);
1699 return len;
1700 }
1701
syslog_clear(void)1702 static void syslog_clear(void)
1703 {
1704 mutex_lock(&syslog_lock);
1705 latched_seq_write(&clear_seq, prb_next_seq(prb));
1706 mutex_unlock(&syslog_lock);
1707 }
1708
do_syslog(int type,char __user * buf,int len,int source)1709 int do_syslog(int type, char __user *buf, int len, int source)
1710 {
1711 struct printk_info info;
1712 bool clear = false;
1713 static int saved_console_loglevel = LOGLEVEL_DEFAULT;
1714 int error;
1715
1716 error = check_syslog_permissions(type, source);
1717 if (error)
1718 return error;
1719
1720 switch (type) {
1721 case SYSLOG_ACTION_CLOSE: /* Close log */
1722 break;
1723 case SYSLOG_ACTION_OPEN: /* Open log */
1724 break;
1725 case SYSLOG_ACTION_READ: /* Read from log */
1726 if (!buf || len < 0)
1727 return -EINVAL;
1728 if (!len)
1729 return 0;
1730 if (!access_ok(buf, len))
1731 return -EFAULT;
1732 error = syslog_print(buf, len);
1733 break;
1734 /* Read/clear last kernel messages */
1735 case SYSLOG_ACTION_READ_CLEAR:
1736 clear = true;
1737 fallthrough;
1738 /* Read last kernel messages */
1739 case SYSLOG_ACTION_READ_ALL:
1740 if (!buf || len < 0)
1741 return -EINVAL;
1742 if (!len)
1743 return 0;
1744 if (!access_ok(buf, len))
1745 return -EFAULT;
1746 error = syslog_print_all(buf, len, clear);
1747 break;
1748 /* Clear ring buffer */
1749 case SYSLOG_ACTION_CLEAR:
1750 syslog_clear();
1751 break;
1752 /* Disable logging to console */
1753 case SYSLOG_ACTION_CONSOLE_OFF:
1754 if (saved_console_loglevel == LOGLEVEL_DEFAULT)
1755 saved_console_loglevel = console_loglevel;
1756 console_loglevel = minimum_console_loglevel;
1757 break;
1758 /* Enable logging to console */
1759 case SYSLOG_ACTION_CONSOLE_ON:
1760 if (saved_console_loglevel != LOGLEVEL_DEFAULT) {
1761 console_loglevel = saved_console_loglevel;
1762 saved_console_loglevel = LOGLEVEL_DEFAULT;
1763 }
1764 break;
1765 /* Set level of messages printed to console */
1766 case SYSLOG_ACTION_CONSOLE_LEVEL:
1767 if (len < 1 || len > 8)
1768 return -EINVAL;
1769 if (len < minimum_console_loglevel)
1770 len = minimum_console_loglevel;
1771 console_loglevel = len;
1772 /* Implicitly re-enable logging to console */
1773 saved_console_loglevel = LOGLEVEL_DEFAULT;
1774 break;
1775 /* Number of chars in the log buffer */
1776 case SYSLOG_ACTION_SIZE_UNREAD:
1777 mutex_lock(&syslog_lock);
1778 if (!prb_read_valid_info(prb, syslog_seq, &info, NULL)) {
1779 /* No unread messages. */
1780 mutex_unlock(&syslog_lock);
1781 return 0;
1782 }
1783 if (info.seq != syslog_seq) {
1784 /* messages are gone, move to first one */
1785 syslog_seq = info.seq;
1786 syslog_partial = 0;
1787 }
1788 if (source == SYSLOG_FROM_PROC) {
1789 /*
1790 * Short-cut for poll(/"proc/kmsg") which simply checks
1791 * for pending data, not the size; return the count of
1792 * records, not the length.
1793 */
1794 error = prb_next_seq(prb) - syslog_seq;
1795 } else {
1796 bool time = syslog_partial ? syslog_time : printk_time;
1797 unsigned int line_count;
1798 u64 seq;
1799
1800 prb_for_each_info(syslog_seq, prb, seq, &info,
1801 &line_count) {
1802 error += get_record_print_text_size(&info, line_count,
1803 true, time);
1804 time = printk_time;
1805 }
1806 error -= syslog_partial;
1807 }
1808 mutex_unlock(&syslog_lock);
1809 break;
1810 /* Size of the log buffer */
1811 case SYSLOG_ACTION_SIZE_BUFFER:
1812 error = log_buf_len;
1813 break;
1814 default:
1815 error = -EINVAL;
1816 break;
1817 }
1818
1819 return error;
1820 }
1821
SYSCALL_DEFINE3(syslog,int,type,char __user *,buf,int,len)1822 SYSCALL_DEFINE3(syslog, int, type, char __user *, buf, int, len)
1823 {
1824 return do_syslog(type, buf, len, SYSLOG_FROM_READER);
1825 }
1826
1827 /*
1828 * Special console_lock variants that help to reduce the risk of soft-lockups.
1829 * They allow to pass console_lock to another printk() call using a busy wait.
1830 */
1831
1832 #ifdef CONFIG_LOCKDEP
1833 static struct lockdep_map console_owner_dep_map = {
1834 .name = "console_owner"
1835 };
1836 #endif
1837
1838 static DEFINE_RAW_SPINLOCK(console_owner_lock);
1839 static struct task_struct *console_owner;
1840 static bool console_waiter;
1841
1842 /**
1843 * console_lock_spinning_enable - mark beginning of code where another
1844 * thread might safely busy wait
1845 *
1846 * This basically converts console_lock into a spinlock. This marks
1847 * the section where the console_lock owner can not sleep, because
1848 * there may be a waiter spinning (like a spinlock). Also it must be
1849 * ready to hand over the lock at the end of the section.
1850 */
console_lock_spinning_enable(void)1851 static void console_lock_spinning_enable(void)
1852 {
1853 raw_spin_lock(&console_owner_lock);
1854 console_owner = current;
1855 raw_spin_unlock(&console_owner_lock);
1856
1857 /* The waiter may spin on us after setting console_owner */
1858 spin_acquire(&console_owner_dep_map, 0, 0, _THIS_IP_);
1859 }
1860
1861 /**
1862 * console_lock_spinning_disable_and_check - mark end of code where another
1863 * thread was able to busy wait and check if there is a waiter
1864 * @cookie: cookie returned from console_srcu_read_lock()
1865 *
1866 * This is called at the end of the section where spinning is allowed.
1867 * It has two functions. First, it is a signal that it is no longer
1868 * safe to start busy waiting for the lock. Second, it checks if
1869 * there is a busy waiter and passes the lock rights to her.
1870 *
1871 * Important: Callers lose both the console_lock and the SRCU read lock if
1872 * there was a busy waiter. They must not touch items synchronized by
1873 * console_lock or SRCU read lock in this case.
1874 *
1875 * Return: 1 if the lock rights were passed, 0 otherwise.
1876 */
console_lock_spinning_disable_and_check(int cookie)1877 static int console_lock_spinning_disable_and_check(int cookie)
1878 {
1879 int waiter;
1880
1881 raw_spin_lock(&console_owner_lock);
1882 waiter = READ_ONCE(console_waiter);
1883 console_owner = NULL;
1884 raw_spin_unlock(&console_owner_lock);
1885
1886 if (!waiter) {
1887 spin_release(&console_owner_dep_map, _THIS_IP_);
1888 return 0;
1889 }
1890
1891 /* The waiter is now free to continue */
1892 WRITE_ONCE(console_waiter, false);
1893
1894 spin_release(&console_owner_dep_map, _THIS_IP_);
1895
1896 /*
1897 * Preserve lockdep lock ordering. Release the SRCU read lock before
1898 * releasing the console_lock.
1899 */
1900 console_srcu_read_unlock(cookie);
1901
1902 /*
1903 * Hand off console_lock to waiter. The waiter will perform
1904 * the up(). After this, the waiter is the console_lock owner.
1905 */
1906 mutex_release(&console_lock_dep_map, _THIS_IP_);
1907 return 1;
1908 }
1909
1910 /**
1911 * console_trylock_spinning - try to get console_lock by busy waiting
1912 *
1913 * This allows to busy wait for the console_lock when the current
1914 * owner is running in specially marked sections. It means that
1915 * the current owner is running and cannot reschedule until it
1916 * is ready to lose the lock.
1917 *
1918 * Return: 1 if we got the lock, 0 othrewise
1919 */
console_trylock_spinning(void)1920 static int console_trylock_spinning(void)
1921 {
1922 struct task_struct *owner = NULL;
1923 bool waiter;
1924 bool spin = false;
1925 unsigned long flags;
1926
1927 if (console_trylock())
1928 return 1;
1929
1930 /*
1931 * It's unsafe to spin once a panic has begun. If we are the
1932 * panic CPU, we may have already halted the owner of the
1933 * console_sem. If we are not the panic CPU, then we should
1934 * avoid taking console_sem, so the panic CPU has a better
1935 * chance of cleanly acquiring it later.
1936 */
1937 if (panic_in_progress())
1938 return 0;
1939
1940 printk_safe_enter_irqsave(flags);
1941
1942 raw_spin_lock(&console_owner_lock);
1943 owner = READ_ONCE(console_owner);
1944 waiter = READ_ONCE(console_waiter);
1945 if (!waiter && owner && owner != current) {
1946 WRITE_ONCE(console_waiter, true);
1947 spin = true;
1948 }
1949 raw_spin_unlock(&console_owner_lock);
1950
1951 /*
1952 * If there is an active printk() writing to the
1953 * consoles, instead of having it write our data too,
1954 * see if we can offload that load from the active
1955 * printer, and do some printing ourselves.
1956 * Go into a spin only if there isn't already a waiter
1957 * spinning, and there is an active printer, and
1958 * that active printer isn't us (recursive printk?).
1959 */
1960 if (!spin) {
1961 printk_safe_exit_irqrestore(flags);
1962 return 0;
1963 }
1964
1965 /* We spin waiting for the owner to release us */
1966 spin_acquire(&console_owner_dep_map, 0, 0, _THIS_IP_);
1967 /* Owner will clear console_waiter on hand off */
1968 while (READ_ONCE(console_waiter))
1969 cpu_relax();
1970 spin_release(&console_owner_dep_map, _THIS_IP_);
1971
1972 printk_safe_exit_irqrestore(flags);
1973 /*
1974 * The owner passed the console lock to us.
1975 * Since we did not spin on console lock, annotate
1976 * this as a trylock. Otherwise lockdep will
1977 * complain.
1978 */
1979 mutex_acquire(&console_lock_dep_map, 0, 1, _THIS_IP_);
1980
1981 return 1;
1982 }
1983
1984 /*
1985 * Recursion is tracked separately on each CPU. If NMIs are supported, an
1986 * additional NMI context per CPU is also separately tracked. Until per-CPU
1987 * is available, a separate "early tracking" is performed.
1988 */
1989 static DEFINE_PER_CPU(u8, printk_count);
1990 static u8 printk_count_early;
1991 #ifdef CONFIG_HAVE_NMI
1992 static DEFINE_PER_CPU(u8, printk_count_nmi);
1993 static u8 printk_count_nmi_early;
1994 #endif
1995
1996 /*
1997 * Recursion is limited to keep the output sane. printk() should not require
1998 * more than 1 level of recursion (allowing, for example, printk() to trigger
1999 * a WARN), but a higher value is used in case some printk-internal errors
2000 * exist, such as the ringbuffer validation checks failing.
2001 */
2002 #define PRINTK_MAX_RECURSION 3
2003
2004 /*
2005 * Return a pointer to the dedicated counter for the CPU+context of the
2006 * caller.
2007 */
__printk_recursion_counter(void)2008 static u8 *__printk_recursion_counter(void)
2009 {
2010 #ifdef CONFIG_HAVE_NMI
2011 if (in_nmi()) {
2012 if (printk_percpu_data_ready())
2013 return this_cpu_ptr(&printk_count_nmi);
2014 return &printk_count_nmi_early;
2015 }
2016 #endif
2017 if (printk_percpu_data_ready())
2018 return this_cpu_ptr(&printk_count);
2019 return &printk_count_early;
2020 }
2021
2022 /*
2023 * Enter recursion tracking. Interrupts are disabled to simplify tracking.
2024 * The caller must check the boolean return value to see if the recursion is
2025 * allowed. On failure, interrupts are not disabled.
2026 *
2027 * @recursion_ptr must be a variable of type (u8 *) and is the same variable
2028 * that is passed to printk_exit_irqrestore().
2029 */
2030 #define printk_enter_irqsave(recursion_ptr, flags) \
2031 ({ \
2032 bool success = true; \
2033 \
2034 typecheck(u8 *, recursion_ptr); \
2035 local_irq_save(flags); \
2036 (recursion_ptr) = __printk_recursion_counter(); \
2037 if (*(recursion_ptr) > PRINTK_MAX_RECURSION) { \
2038 local_irq_restore(flags); \
2039 success = false; \
2040 } else { \
2041 (*(recursion_ptr))++; \
2042 } \
2043 success; \
2044 })
2045
2046 /* Exit recursion tracking, restoring interrupts. */
2047 #define printk_exit_irqrestore(recursion_ptr, flags) \
2048 do { \
2049 typecheck(u8 *, recursion_ptr); \
2050 (*(recursion_ptr))--; \
2051 local_irq_restore(flags); \
2052 } while (0)
2053
2054 int printk_delay_msec __read_mostly;
2055
printk_delay(int level)2056 static inline void printk_delay(int level)
2057 {
2058 boot_delay_msec(level);
2059
2060 if (unlikely(printk_delay_msec)) {
2061 int m = printk_delay_msec;
2062
2063 while (m--) {
2064 mdelay(1);
2065 touch_nmi_watchdog();
2066 }
2067 }
2068 }
2069
printk_caller_id(void)2070 static inline u32 printk_caller_id(void)
2071 {
2072 return in_task() ? task_pid_nr(current) :
2073 0x80000000 + smp_processor_id();
2074 }
2075
2076 /**
2077 * printk_parse_prefix - Parse level and control flags.
2078 *
2079 * @text: The terminated text message.
2080 * @level: A pointer to the current level value, will be updated.
2081 * @flags: A pointer to the current printk_info flags, will be updated.
2082 *
2083 * @level may be NULL if the caller is not interested in the parsed value.
2084 * Otherwise the variable pointed to by @level must be set to
2085 * LOGLEVEL_DEFAULT in order to be updated with the parsed value.
2086 *
2087 * @flags may be NULL if the caller is not interested in the parsed value.
2088 * Otherwise the variable pointed to by @flags will be OR'd with the parsed
2089 * value.
2090 *
2091 * Return: The length of the parsed level and control flags.
2092 */
printk_parse_prefix(const char * text,int * level,enum printk_info_flags * flags)2093 u16 printk_parse_prefix(const char *text, int *level,
2094 enum printk_info_flags *flags)
2095 {
2096 u16 prefix_len = 0;
2097 int kern_level;
2098
2099 while (*text) {
2100 kern_level = printk_get_level(text);
2101 if (!kern_level)
2102 break;
2103
2104 switch (kern_level) {
2105 case '0' ... '7':
2106 if (level && *level == LOGLEVEL_DEFAULT)
2107 *level = kern_level - '0';
2108 break;
2109 case 'c': /* KERN_CONT */
2110 if (flags)
2111 *flags |= LOG_CONT;
2112 }
2113
2114 prefix_len += 2;
2115 text += 2;
2116 }
2117
2118 return prefix_len;
2119 }
2120
2121 __printf(5, 0)
printk_sprint(char * text,u16 size,int facility,enum printk_info_flags * flags,const char * fmt,va_list args)2122 static u16 printk_sprint(char *text, u16 size, int facility,
2123 enum printk_info_flags *flags, const char *fmt,
2124 va_list args)
2125 {
2126 u16 text_len;
2127
2128 text_len = vscnprintf(text, size, fmt, args);
2129
2130 /* Mark and strip a trailing newline. */
2131 if (text_len && text[text_len - 1] == '\n') {
2132 text_len--;
2133 *flags |= LOG_NEWLINE;
2134 }
2135
2136 /* Strip log level and control flags. */
2137 if (facility == 0) {
2138 u16 prefix_len;
2139
2140 prefix_len = printk_parse_prefix(text, NULL, NULL);
2141 if (prefix_len) {
2142 text_len -= prefix_len;
2143 memmove(text, text + prefix_len, text_len);
2144 }
2145 }
2146
2147 trace_console(text, text_len);
2148
2149 return text_len;
2150 }
2151
2152 __printf(4, 0)
vprintk_store(int facility,int level,const struct dev_printk_info * dev_info,const char * fmt,va_list args)2153 int vprintk_store(int facility, int level,
2154 const struct dev_printk_info *dev_info,
2155 const char *fmt, va_list args)
2156 {
2157 struct prb_reserved_entry e;
2158 enum printk_info_flags flags = 0;
2159 struct printk_record r;
2160 unsigned long irqflags;
2161 u16 trunc_msg_len = 0;
2162 char prefix_buf[8];
2163 u8 *recursion_ptr;
2164 u16 reserve_size;
2165 va_list args2;
2166 u32 caller_id;
2167 u16 text_len;
2168 int ret = 0;
2169 u64 ts_nsec;
2170
2171 if (!printk_enter_irqsave(recursion_ptr, irqflags))
2172 return 0;
2173
2174 /*
2175 * Since the duration of printk() can vary depending on the message
2176 * and state of the ringbuffer, grab the timestamp now so that it is
2177 * close to the call of printk(). This provides a more deterministic
2178 * timestamp with respect to the caller.
2179 */
2180 ts_nsec = local_clock();
2181
2182 caller_id = printk_caller_id();
2183
2184 /*
2185 * The sprintf needs to come first since the syslog prefix might be
2186 * passed in as a parameter. An extra byte must be reserved so that
2187 * later the vscnprintf() into the reserved buffer has room for the
2188 * terminating '\0', which is not counted by vsnprintf().
2189 */
2190 va_copy(args2, args);
2191 reserve_size = vsnprintf(&prefix_buf[0], sizeof(prefix_buf), fmt, args2) + 1;
2192 va_end(args2);
2193
2194 if (reserve_size > PRINTKRB_RECORD_MAX)
2195 reserve_size = PRINTKRB_RECORD_MAX;
2196
2197 /* Extract log level or control flags. */
2198 if (facility == 0)
2199 printk_parse_prefix(&prefix_buf[0], &level, &flags);
2200
2201 if (level == LOGLEVEL_DEFAULT)
2202 level = default_message_loglevel;
2203
2204 if (dev_info)
2205 flags |= LOG_NEWLINE;
2206
2207 if (flags & LOG_CONT) {
2208 prb_rec_init_wr(&r, reserve_size);
2209 if (prb_reserve_in_last(&e, prb, &r, caller_id, PRINTKRB_RECORD_MAX)) {
2210 text_len = printk_sprint(&r.text_buf[r.info->text_len], reserve_size,
2211 facility, &flags, fmt, args);
2212 r.info->text_len += text_len;
2213
2214 if (flags & LOG_NEWLINE) {
2215 r.info->flags |= LOG_NEWLINE;
2216 prb_final_commit(&e);
2217 } else {
2218 prb_commit(&e);
2219 }
2220
2221 ret = text_len;
2222 goto out;
2223 }
2224 }
2225
2226 /*
2227 * Explicitly initialize the record before every prb_reserve() call.
2228 * prb_reserve_in_last() and prb_reserve() purposely invalidate the
2229 * structure when they fail.
2230 */
2231 prb_rec_init_wr(&r, reserve_size);
2232 if (!prb_reserve(&e, prb, &r)) {
2233 /* truncate the message if it is too long for empty buffer */
2234 truncate_msg(&reserve_size, &trunc_msg_len);
2235
2236 prb_rec_init_wr(&r, reserve_size + trunc_msg_len);
2237 if (!prb_reserve(&e, prb, &r))
2238 goto out;
2239 }
2240
2241 /* fill message */
2242 text_len = printk_sprint(&r.text_buf[0], reserve_size, facility, &flags, fmt, args);
2243 if (trunc_msg_len)
2244 memcpy(&r.text_buf[text_len], trunc_msg, trunc_msg_len);
2245 r.info->text_len = text_len + trunc_msg_len;
2246 r.info->facility = facility;
2247 r.info->level = level & 7;
2248 r.info->flags = flags & 0x1f;
2249 r.info->ts_nsec = ts_nsec;
2250 r.info->caller_id = caller_id;
2251 if (dev_info)
2252 memcpy(&r.info->dev_info, dev_info, sizeof(r.info->dev_info));
2253
2254 /* A message without a trailing newline can be continued. */
2255 if (!(flags & LOG_NEWLINE))
2256 prb_commit(&e);
2257 else
2258 prb_final_commit(&e);
2259
2260 ret = text_len + trunc_msg_len;
2261 out:
2262 printk_exit_irqrestore(recursion_ptr, irqflags);
2263 return ret;
2264 }
2265
vprintk_emit(int facility,int level,const struct dev_printk_info * dev_info,const char * fmt,va_list args)2266 asmlinkage int vprintk_emit(int facility, int level,
2267 const struct dev_printk_info *dev_info,
2268 const char *fmt, va_list args)
2269 {
2270 int printed_len;
2271 bool in_sched = false;
2272
2273 /* Suppress unimportant messages after panic happens */
2274 if (unlikely(suppress_printk))
2275 return 0;
2276
2277 if (unlikely(suppress_panic_printk) &&
2278 atomic_read(&panic_cpu) != raw_smp_processor_id())
2279 return 0;
2280
2281 if (level == LOGLEVEL_SCHED) {
2282 level = LOGLEVEL_DEFAULT;
2283 in_sched = true;
2284 }
2285
2286 printk_delay(level);
2287
2288 printed_len = vprintk_store(facility, level, dev_info, fmt, args);
2289
2290 /* If called from the scheduler, we can not call up(). */
2291 if (!in_sched) {
2292 /*
2293 * The caller may be holding system-critical or
2294 * timing-sensitive locks. Disable preemption during
2295 * printing of all remaining records to all consoles so that
2296 * this context can return as soon as possible. Hopefully
2297 * another printk() caller will take over the printing.
2298 */
2299 preempt_disable();
2300 /*
2301 * Try to acquire and then immediately release the console
2302 * semaphore. The release will print out buffers. With the
2303 * spinning variant, this context tries to take over the
2304 * printing from another printing context.
2305 */
2306 if (console_trylock_spinning())
2307 console_unlock();
2308 preempt_enable();
2309 }
2310
2311 if (in_sched)
2312 defer_console_output();
2313 else
2314 wake_up_klogd();
2315
2316 return printed_len;
2317 }
2318 EXPORT_SYMBOL(vprintk_emit);
2319
vprintk_default(const char * fmt,va_list args)2320 int vprintk_default(const char *fmt, va_list args)
2321 {
2322 return vprintk_emit(0, LOGLEVEL_DEFAULT, NULL, fmt, args);
2323 }
2324 EXPORT_SYMBOL_GPL(vprintk_default);
2325
_printk(const char * fmt,...)2326 asmlinkage __visible int _printk(const char *fmt, ...)
2327 {
2328 va_list args;
2329 int r;
2330
2331 va_start(args, fmt);
2332 r = vprintk(fmt, args);
2333 va_end(args);
2334
2335 return r;
2336 }
2337 EXPORT_SYMBOL(_printk);
2338
2339 static bool pr_flush(int timeout_ms, bool reset_on_progress);
2340 static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progress);
2341
2342 #else /* CONFIG_PRINTK */
2343
2344 #define printk_time false
2345
2346 #define prb_read_valid(rb, seq, r) false
2347 #define prb_first_valid_seq(rb) 0
2348 #define prb_next_seq(rb) 0
2349
2350 static u64 syslog_seq;
2351
record_print_text(const struct printk_record * r,bool syslog,bool time)2352 static size_t record_print_text(const struct printk_record *r,
2353 bool syslog, bool time)
2354 {
2355 return 0;
2356 }
info_print_ext_header(char * buf,size_t size,struct printk_info * info)2357 static ssize_t info_print_ext_header(char *buf, size_t size,
2358 struct printk_info *info)
2359 {
2360 return 0;
2361 }
msg_print_ext_body(char * buf,size_t size,char * text,size_t text_len,struct dev_printk_info * dev_info)2362 static ssize_t msg_print_ext_body(char *buf, size_t size,
2363 char *text, size_t text_len,
2364 struct dev_printk_info *dev_info) { return 0; }
console_lock_spinning_enable(void)2365 static void console_lock_spinning_enable(void) { }
console_lock_spinning_disable_and_check(int cookie)2366 static int console_lock_spinning_disable_and_check(int cookie) { return 0; }
suppress_message_printing(int level)2367 static bool suppress_message_printing(int level) { return false; }
pr_flush(int timeout_ms,bool reset_on_progress)2368 static bool pr_flush(int timeout_ms, bool reset_on_progress) { return true; }
__pr_flush(struct console * con,int timeout_ms,bool reset_on_progress)2369 static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progress) { return true; }
2370
2371 #endif /* CONFIG_PRINTK */
2372
2373 #ifdef CONFIG_EARLY_PRINTK
2374 struct console *early_console;
2375
early_printk(const char * fmt,...)2376 asmlinkage __visible void early_printk(const char *fmt, ...)
2377 {
2378 va_list ap;
2379 char buf[512];
2380 int n;
2381
2382 if (!early_console)
2383 return;
2384
2385 va_start(ap, fmt);
2386 n = vscnprintf(buf, sizeof(buf), fmt, ap);
2387 va_end(ap);
2388
2389 early_console->write(early_console, buf, n);
2390 }
2391 #endif
2392
set_user_specified(struct console_cmdline * c,bool user_specified)2393 static void set_user_specified(struct console_cmdline *c, bool user_specified)
2394 {
2395 if (!user_specified)
2396 return;
2397
2398 /*
2399 * @c console was defined by the user on the command line.
2400 * Do not clear when added twice also by SPCR or the device tree.
2401 */
2402 c->user_specified = true;
2403 /* At least one console defined by the user on the command line. */
2404 console_set_on_cmdline = 1;
2405 }
2406
__add_preferred_console(char * name,int idx,char * options,char * brl_options,bool user_specified)2407 static int __add_preferred_console(char *name, int idx, char *options,
2408 char *brl_options, bool user_specified)
2409 {
2410 struct console_cmdline *c;
2411 int i;
2412
2413 /*
2414 * See if this tty is not yet registered, and
2415 * if we have a slot free.
2416 */
2417 for (i = 0, c = console_cmdline;
2418 i < MAX_CMDLINECONSOLES && c->name[0];
2419 i++, c++) {
2420 if (strcmp(c->name, name) == 0 && c->index == idx) {
2421 if (!brl_options)
2422 preferred_console = i;
2423 set_user_specified(c, user_specified);
2424 return 0;
2425 }
2426 }
2427 if (i == MAX_CMDLINECONSOLES)
2428 return -E2BIG;
2429 if (!brl_options)
2430 preferred_console = i;
2431 strscpy(c->name, name, sizeof(c->name));
2432 c->options = options;
2433 set_user_specified(c, user_specified);
2434 braille_set_options(c, brl_options);
2435
2436 c->index = idx;
2437 return 0;
2438 }
2439
console_msg_format_setup(char * str)2440 static int __init console_msg_format_setup(char *str)
2441 {
2442 if (!strcmp(str, "syslog"))
2443 console_msg_format = MSG_FORMAT_SYSLOG;
2444 if (!strcmp(str, "default"))
2445 console_msg_format = MSG_FORMAT_DEFAULT;
2446 return 1;
2447 }
2448 __setup("console_msg_format=", console_msg_format_setup);
2449
2450 /*
2451 * Set up a console. Called via do_early_param() in init/main.c
2452 * for each "console=" parameter in the boot command line.
2453 */
console_setup(char * str)2454 static int __init console_setup(char *str)
2455 {
2456 char buf[sizeof(console_cmdline[0].name) + 4]; /* 4 for "ttyS" */
2457 char *s, *options, *brl_options = NULL;
2458 int idx;
2459
2460 /*
2461 * console="" or console=null have been suggested as a way to
2462 * disable console output. Use ttynull that has been created
2463 * for exactly this purpose.
2464 */
2465 if (str[0] == 0 || strcmp(str, "null") == 0) {
2466 __add_preferred_console("ttynull", 0, NULL, NULL, true);
2467 return 1;
2468 }
2469
2470 if (_braille_console_setup(&str, &brl_options))
2471 return 1;
2472
2473 /*
2474 * Decode str into name, index, options.
2475 */
2476 if (str[0] >= '0' && str[0] <= '9') {
2477 strcpy(buf, "ttyS");
2478 strncpy(buf + 4, str, sizeof(buf) - 5);
2479 } else {
2480 strncpy(buf, str, sizeof(buf) - 1);
2481 }
2482 buf[sizeof(buf) - 1] = 0;
2483 options = strchr(str, ',');
2484 if (options)
2485 *(options++) = 0;
2486 #ifdef __sparc__
2487 if (!strcmp(str, "ttya"))
2488 strcpy(buf, "ttyS0");
2489 if (!strcmp(str, "ttyb"))
2490 strcpy(buf, "ttyS1");
2491 #endif
2492 for (s = buf; *s; s++)
2493 if (isdigit(*s) || *s == ',')
2494 break;
2495 idx = simple_strtoul(s, NULL, 10);
2496 *s = 0;
2497
2498 __add_preferred_console(buf, idx, options, brl_options, true);
2499 return 1;
2500 }
2501 __setup("console=", console_setup);
2502
2503 /**
2504 * add_preferred_console - add a device to the list of preferred consoles.
2505 * @name: device name
2506 * @idx: device index
2507 * @options: options for this console
2508 *
2509 * The last preferred console added will be used for kernel messages
2510 * and stdin/out/err for init. Normally this is used by console_setup
2511 * above to handle user-supplied console arguments; however it can also
2512 * be used by arch-specific code either to override the user or more
2513 * commonly to provide a default console (ie from PROM variables) when
2514 * the user has not supplied one.
2515 */
add_preferred_console(char * name,int idx,char * options)2516 int add_preferred_console(char *name, int idx, char *options)
2517 {
2518 return __add_preferred_console(name, idx, options, NULL, false);
2519 }
2520
2521 bool console_suspend_enabled = true;
2522 EXPORT_SYMBOL(console_suspend_enabled);
2523
console_suspend_disable(char * str)2524 static int __init console_suspend_disable(char *str)
2525 {
2526 console_suspend_enabled = false;
2527 return 1;
2528 }
2529 __setup("no_console_suspend", console_suspend_disable);
2530 module_param_named(console_suspend, console_suspend_enabled,
2531 bool, S_IRUGO | S_IWUSR);
2532 MODULE_PARM_DESC(console_suspend, "suspend console during suspend"
2533 " and hibernate operations");
2534
2535 static bool printk_console_no_auto_verbose;
2536
console_verbose(void)2537 void console_verbose(void)
2538 {
2539 if (console_loglevel && !printk_console_no_auto_verbose)
2540 console_loglevel = CONSOLE_LOGLEVEL_MOTORMOUTH;
2541 }
2542 EXPORT_SYMBOL_GPL(console_verbose);
2543
2544 module_param_named(console_no_auto_verbose, printk_console_no_auto_verbose, bool, 0644);
2545 MODULE_PARM_DESC(console_no_auto_verbose, "Disable console loglevel raise to highest on oops/panic/etc");
2546
2547 /**
2548 * suspend_console - suspend the console subsystem
2549 *
2550 * This disables printk() while we go into suspend states
2551 */
suspend_console(void)2552 void suspend_console(void)
2553 {
2554 struct console *con;
2555
2556 if (!console_suspend_enabled)
2557 return;
2558 pr_info("Suspending console(s) (use no_console_suspend to debug)\n");
2559 pr_flush(1000, true);
2560
2561 console_list_lock();
2562 for_each_console(con)
2563 console_srcu_write_flags(con, con->flags | CON_SUSPENDED);
2564 console_list_unlock();
2565
2566 /*
2567 * Ensure that all SRCU list walks have completed. All printing
2568 * contexts must be able to see that they are suspended so that it
2569 * is guaranteed that all printing has stopped when this function
2570 * completes.
2571 */
2572 synchronize_srcu(&console_srcu);
2573 }
2574
resume_console(void)2575 void resume_console(void)
2576 {
2577 struct console *con;
2578
2579 if (!console_suspend_enabled)
2580 return;
2581
2582 console_list_lock();
2583 for_each_console(con)
2584 console_srcu_write_flags(con, con->flags & ~CON_SUSPENDED);
2585 console_list_unlock();
2586
2587 /*
2588 * Ensure that all SRCU list walks have completed. All printing
2589 * contexts must be able to see they are no longer suspended so
2590 * that they are guaranteed to wake up and resume printing.
2591 */
2592 synchronize_srcu(&console_srcu);
2593
2594 pr_flush(1000, true);
2595 }
2596
2597 /**
2598 * console_cpu_notify - print deferred console messages after CPU hotplug
2599 * @cpu: unused
2600 *
2601 * If printk() is called from a CPU that is not online yet, the messages
2602 * will be printed on the console only if there are CON_ANYTIME consoles.
2603 * This function is called when a new CPU comes online (or fails to come
2604 * up) or goes offline.
2605 */
console_cpu_notify(unsigned int cpu)2606 static int console_cpu_notify(unsigned int cpu)
2607 {
2608 if (!cpuhp_tasks_frozen) {
2609 /* If trylock fails, someone else is doing the printing */
2610 if (console_trylock())
2611 console_unlock();
2612 }
2613 return 0;
2614 }
2615
2616 /*
2617 * Return true if a panic is in progress on a remote CPU.
2618 *
2619 * On true, the local CPU should immediately release any printing resources
2620 * that may be needed by the panic CPU.
2621 */
other_cpu_in_panic(void)2622 bool other_cpu_in_panic(void)
2623 {
2624 if (!panic_in_progress())
2625 return false;
2626
2627 /*
2628 * We can use raw_smp_processor_id() here because it is impossible for
2629 * the task to be migrated to the panic_cpu, or away from it. If
2630 * panic_cpu has already been set, and we're not currently executing on
2631 * that CPU, then we never will be.
2632 */
2633 return atomic_read(&panic_cpu) != raw_smp_processor_id();
2634 }
2635
2636 /**
2637 * console_lock - block the console subsystem from printing
2638 *
2639 * Acquires a lock which guarantees that no consoles will
2640 * be in or enter their write() callback.
2641 *
2642 * Can sleep, returns nothing.
2643 */
console_lock(void)2644 void console_lock(void)
2645 {
2646 might_sleep();
2647
2648 /* On panic, the console_lock must be left to the panic cpu. */
2649 while (other_cpu_in_panic())
2650 msleep(1000);
2651
2652 down_console_sem();
2653 console_locked = 1;
2654 console_may_schedule = 1;
2655 }
2656 EXPORT_SYMBOL(console_lock);
2657
2658 /**
2659 * console_trylock - try to block the console subsystem from printing
2660 *
2661 * Try to acquire a lock which guarantees that no consoles will
2662 * be in or enter their write() callback.
2663 *
2664 * returns 1 on success, and 0 on failure to acquire the lock.
2665 */
console_trylock(void)2666 int console_trylock(void)
2667 {
2668 /* On panic, the console_lock must be left to the panic cpu. */
2669 if (other_cpu_in_panic())
2670 return 0;
2671 if (down_trylock_console_sem())
2672 return 0;
2673 console_locked = 1;
2674 console_may_schedule = 0;
2675 return 1;
2676 }
2677 EXPORT_SYMBOL(console_trylock);
2678
is_console_locked(void)2679 int is_console_locked(void)
2680 {
2681 return console_locked;
2682 }
2683 EXPORT_SYMBOL(is_console_locked);
2684
2685 /*
2686 * Check if the given console is currently capable and allowed to print
2687 * records.
2688 *
2689 * Requires the console_srcu_read_lock.
2690 */
console_is_usable(struct console * con)2691 static inline bool console_is_usable(struct console *con)
2692 {
2693 short flags = console_srcu_read_flags(con);
2694
2695 if (!(flags & CON_ENABLED))
2696 return false;
2697
2698 if ((flags & CON_SUSPENDED))
2699 return false;
2700
2701 if (!con->write)
2702 return false;
2703
2704 /*
2705 * Console drivers may assume that per-cpu resources have been
2706 * allocated. So unless they're explicitly marked as being able to
2707 * cope (CON_ANYTIME) don't call them until this CPU is officially up.
2708 */
2709 if (!cpu_online(raw_smp_processor_id()) && !(flags & CON_ANYTIME))
2710 return false;
2711
2712 return true;
2713 }
2714
__console_unlock(void)2715 static void __console_unlock(void)
2716 {
2717 console_locked = 0;
2718 up_console_sem();
2719 }
2720
2721 /*
2722 * Prepend the message in @pmsg->pbufs->outbuf with a "dropped message". This
2723 * is achieved by shifting the existing message over and inserting the dropped
2724 * message.
2725 *
2726 * @pmsg is the printk message to prepend.
2727 *
2728 * @dropped is the dropped count to report in the dropped message.
2729 *
2730 * If the message text in @pmsg->pbufs->outbuf does not have enough space for
2731 * the dropped message, the message text will be sufficiently truncated.
2732 *
2733 * If @pmsg->pbufs->outbuf is modified, @pmsg->outbuf_len is updated.
2734 */
2735 #ifdef CONFIG_PRINTK
console_prepend_dropped(struct printk_message * pmsg,unsigned long dropped)2736 static void console_prepend_dropped(struct printk_message *pmsg, unsigned long dropped)
2737 {
2738 struct printk_buffers *pbufs = pmsg->pbufs;
2739 const size_t scratchbuf_sz = sizeof(pbufs->scratchbuf);
2740 const size_t outbuf_sz = sizeof(pbufs->outbuf);
2741 char *scratchbuf = &pbufs->scratchbuf[0];
2742 char *outbuf = &pbufs->outbuf[0];
2743 size_t len;
2744
2745 len = scnprintf(scratchbuf, scratchbuf_sz,
2746 "** %lu printk messages dropped **\n", dropped);
2747
2748 /*
2749 * Make sure outbuf is sufficiently large before prepending.
2750 * Keep at least the prefix when the message must be truncated.
2751 * It is a rather theoretical problem when someone tries to
2752 * use a minimalist buffer.
2753 */
2754 if (WARN_ON_ONCE(len + PRINTK_PREFIX_MAX >= outbuf_sz))
2755 return;
2756
2757 if (pmsg->outbuf_len + len >= outbuf_sz) {
2758 /* Truncate the message, but keep it terminated. */
2759 pmsg->outbuf_len = outbuf_sz - (len + 1);
2760 outbuf[pmsg->outbuf_len] = 0;
2761 }
2762
2763 memmove(outbuf + len, outbuf, pmsg->outbuf_len + 1);
2764 memcpy(outbuf, scratchbuf, len);
2765 pmsg->outbuf_len += len;
2766 }
2767 #else
2768 #define console_prepend_dropped(pmsg, dropped)
2769 #endif /* CONFIG_PRINTK */
2770
2771 /*
2772 * Read and format the specified record (or a later record if the specified
2773 * record is not available).
2774 *
2775 * @pmsg will contain the formatted result. @pmsg->pbufs must point to a
2776 * struct printk_buffers.
2777 *
2778 * @seq is the record to read and format. If it is not available, the next
2779 * valid record is read.
2780 *
2781 * @is_extended specifies if the message should be formatted for extended
2782 * console output.
2783 *
2784 * @may_supress specifies if records may be skipped based on loglevel.
2785 *
2786 * Returns false if no record is available. Otherwise true and all fields
2787 * of @pmsg are valid. (See the documentation of struct printk_message
2788 * for information about the @pmsg fields.)
2789 */
printk_get_next_message(struct printk_message * pmsg,u64 seq,bool is_extended,bool may_suppress)2790 static bool printk_get_next_message(struct printk_message *pmsg, u64 seq,
2791 bool is_extended, bool may_suppress)
2792 {
2793 static int panic_console_dropped;
2794
2795 struct printk_buffers *pbufs = pmsg->pbufs;
2796 const size_t scratchbuf_sz = sizeof(pbufs->scratchbuf);
2797 const size_t outbuf_sz = sizeof(pbufs->outbuf);
2798 char *scratchbuf = &pbufs->scratchbuf[0];
2799 char *outbuf = &pbufs->outbuf[0];
2800 struct printk_info info;
2801 struct printk_record r;
2802 size_t len = 0;
2803
2804 /*
2805 * Formatting extended messages requires a separate buffer, so use the
2806 * scratch buffer to read in the ringbuffer text.
2807 *
2808 * Formatting normal messages is done in-place, so read the ringbuffer
2809 * text directly into the output buffer.
2810 */
2811 if (is_extended)
2812 prb_rec_init_rd(&r, &info, scratchbuf, scratchbuf_sz);
2813 else
2814 prb_rec_init_rd(&r, &info, outbuf, outbuf_sz);
2815
2816 if (!prb_read_valid(prb, seq, &r))
2817 return false;
2818
2819 pmsg->seq = r.info->seq;
2820 pmsg->dropped = r.info->seq - seq;
2821
2822 /*
2823 * Check for dropped messages in panic here so that printk
2824 * suppression can occur as early as possible if necessary.
2825 */
2826 if (pmsg->dropped &&
2827 panic_in_progress() &&
2828 panic_console_dropped++ > 10) {
2829 suppress_panic_printk = 1;
2830 pr_warn_once("Too many dropped messages. Suppress messages on non-panic CPUs to prevent livelock.\n");
2831 }
2832
2833 /* Skip record that has level above the console loglevel. */
2834 if (may_suppress && suppress_message_printing(r.info->level))
2835 goto out;
2836
2837 if (is_extended) {
2838 len = info_print_ext_header(outbuf, outbuf_sz, r.info);
2839 len += msg_print_ext_body(outbuf + len, outbuf_sz - len,
2840 &r.text_buf[0], r.info->text_len, &r.info->dev_info);
2841 } else {
2842 len = record_print_text(&r, console_msg_format & MSG_FORMAT_SYSLOG, printk_time);
2843 }
2844 out:
2845 pmsg->outbuf_len = len;
2846 return true;
2847 }
2848
2849 /*
2850 * Print one record for the given console. The record printed is whatever
2851 * record is the next available record for the given console.
2852 *
2853 * @handover will be set to true if a printk waiter has taken over the
2854 * console_lock, in which case the caller is no longer holding both the
2855 * console_lock and the SRCU read lock. Otherwise it is set to false.
2856 *
2857 * @cookie is the cookie from the SRCU read lock.
2858 *
2859 * Returns false if the given console has no next record to print, otherwise
2860 * true.
2861 *
2862 * Requires the console_lock and the SRCU read lock.
2863 */
console_emit_next_record(struct console * con,bool * handover,int cookie)2864 static bool console_emit_next_record(struct console *con, bool *handover, int cookie)
2865 {
2866 static struct printk_buffers pbufs;
2867
2868 bool is_extended = console_srcu_read_flags(con) & CON_EXTENDED;
2869 char *outbuf = &pbufs.outbuf[0];
2870 struct printk_message pmsg = {
2871 .pbufs = &pbufs,
2872 };
2873 unsigned long flags;
2874
2875 *handover = false;
2876
2877 if (!printk_get_next_message(&pmsg, con->seq, is_extended, true))
2878 return false;
2879
2880 con->dropped += pmsg.dropped;
2881
2882 /* Skip messages of formatted length 0. */
2883 if (pmsg.outbuf_len == 0) {
2884 con->seq = pmsg.seq + 1;
2885 goto skip;
2886 }
2887
2888 if (con->dropped && !is_extended) {
2889 console_prepend_dropped(&pmsg, con->dropped);
2890 con->dropped = 0;
2891 }
2892
2893 /*
2894 * While actively printing out messages, if another printk()
2895 * were to occur on another CPU, it may wait for this one to
2896 * finish. This task can not be preempted if there is a
2897 * waiter waiting to take over.
2898 *
2899 * Interrupts are disabled because the hand over to a waiter
2900 * must not be interrupted until the hand over is completed
2901 * (@console_waiter is cleared).
2902 */
2903 printk_safe_enter_irqsave(flags);
2904 console_lock_spinning_enable();
2905
2906 /* Do not trace print latency. */
2907 stop_critical_timings();
2908
2909 /* Write everything out to the hardware. */
2910 con->write(con, outbuf, pmsg.outbuf_len);
2911
2912 start_critical_timings();
2913
2914 con->seq = pmsg.seq + 1;
2915
2916 *handover = console_lock_spinning_disable_and_check(cookie);
2917 printk_safe_exit_irqrestore(flags);
2918 skip:
2919 return true;
2920 }
2921
2922 /*
2923 * Print out all remaining records to all consoles.
2924 *
2925 * @do_cond_resched is set by the caller. It can be true only in schedulable
2926 * context.
2927 *
2928 * @next_seq is set to the sequence number after the last available record.
2929 * The value is valid only when this function returns true. It means that all
2930 * usable consoles are completely flushed.
2931 *
2932 * @handover will be set to true if a printk waiter has taken over the
2933 * console_lock, in which case the caller is no longer holding the
2934 * console_lock. Otherwise it is set to false.
2935 *
2936 * Returns true when there was at least one usable console and all messages
2937 * were flushed to all usable consoles. A returned false informs the caller
2938 * that everything was not flushed (either there were no usable consoles or
2939 * another context has taken over printing or it is a panic situation and this
2940 * is not the panic CPU). Regardless the reason, the caller should assume it
2941 * is not useful to immediately try again.
2942 *
2943 * Requires the console_lock.
2944 */
console_flush_all(bool do_cond_resched,u64 * next_seq,bool * handover)2945 static bool console_flush_all(bool do_cond_resched, u64 *next_seq, bool *handover)
2946 {
2947 bool any_usable = false;
2948 struct console *con;
2949 bool any_progress;
2950 int cookie;
2951
2952 *next_seq = 0;
2953 *handover = false;
2954
2955 do {
2956 any_progress = false;
2957
2958 cookie = console_srcu_read_lock();
2959 for_each_console_srcu(con) {
2960 bool progress;
2961
2962 if (!console_is_usable(con))
2963 continue;
2964 any_usable = true;
2965
2966 progress = console_emit_next_record(con, handover, cookie);
2967
2968 /*
2969 * If a handover has occurred, the SRCU read lock
2970 * is already released.
2971 */
2972 if (*handover)
2973 return false;
2974
2975 /* Track the next of the highest seq flushed. */
2976 if (con->seq > *next_seq)
2977 *next_seq = con->seq;
2978
2979 if (!progress)
2980 continue;
2981 any_progress = true;
2982
2983 /* Allow panic_cpu to take over the consoles safely. */
2984 if (other_cpu_in_panic())
2985 goto abandon;
2986
2987 if (do_cond_resched)
2988 cond_resched();
2989 }
2990 console_srcu_read_unlock(cookie);
2991 } while (any_progress);
2992
2993 return any_usable;
2994
2995 abandon:
2996 console_srcu_read_unlock(cookie);
2997 return false;
2998 }
2999
3000 /**
3001 * console_unlock - unblock the console subsystem from printing
3002 *
3003 * Releases the console_lock which the caller holds to block printing of
3004 * the console subsystem.
3005 *
3006 * While the console_lock was held, console output may have been buffered
3007 * by printk(). If this is the case, console_unlock(); emits
3008 * the output prior to releasing the lock.
3009 *
3010 * console_unlock(); may be called from any context.
3011 */
console_unlock(void)3012 void console_unlock(void)
3013 {
3014 bool do_cond_resched;
3015 bool handover;
3016 bool flushed;
3017 u64 next_seq;
3018
3019 /*
3020 * Console drivers are called with interrupts disabled, so
3021 * @console_may_schedule should be cleared before; however, we may
3022 * end up dumping a lot of lines, for example, if called from
3023 * console registration path, and should invoke cond_resched()
3024 * between lines if allowable. Not doing so can cause a very long
3025 * scheduling stall on a slow console leading to RCU stall and
3026 * softlockup warnings which exacerbate the issue with more
3027 * messages practically incapacitating the system. Therefore, create
3028 * a local to use for the printing loop.
3029 */
3030 do_cond_resched = console_may_schedule;
3031
3032 do {
3033 console_may_schedule = 0;
3034
3035 flushed = console_flush_all(do_cond_resched, &next_seq, &handover);
3036 if (!handover)
3037 __console_unlock();
3038
3039 /*
3040 * Abort if there was a failure to flush all messages to all
3041 * usable consoles. Either it is not possible to flush (in
3042 * which case it would be an infinite loop of retrying) or
3043 * another context has taken over printing.
3044 */
3045 if (!flushed)
3046 break;
3047
3048 /*
3049 * Some context may have added new records after
3050 * console_flush_all() but before unlocking the console.
3051 * Re-check if there is a new record to flush. If the trylock
3052 * fails, another context is already handling the printing.
3053 */
3054 } while (prb_read_valid(prb, next_seq, NULL) && console_trylock());
3055 }
3056 EXPORT_SYMBOL(console_unlock);
3057
3058 /**
3059 * console_conditional_schedule - yield the CPU if required
3060 *
3061 * If the console code is currently allowed to sleep, and
3062 * if this CPU should yield the CPU to another task, do
3063 * so here.
3064 *
3065 * Must be called within console_lock();.
3066 */
console_conditional_schedule(void)3067 void __sched console_conditional_schedule(void)
3068 {
3069 if (console_may_schedule)
3070 cond_resched();
3071 }
3072 EXPORT_SYMBOL(console_conditional_schedule);
3073
console_unblank(void)3074 void console_unblank(void)
3075 {
3076 bool found_unblank = false;
3077 struct console *c;
3078 int cookie;
3079
3080 /*
3081 * First check if there are any consoles implementing the unblank()
3082 * callback. If not, there is no reason to continue and take the
3083 * console lock, which in particular can be dangerous if
3084 * @oops_in_progress is set.
3085 */
3086 cookie = console_srcu_read_lock();
3087 for_each_console_srcu(c) {
3088 if ((console_srcu_read_flags(c) & CON_ENABLED) && c->unblank) {
3089 found_unblank = true;
3090 break;
3091 }
3092 }
3093 console_srcu_read_unlock(cookie);
3094 if (!found_unblank)
3095 return;
3096
3097 /*
3098 * Stop console printing because the unblank() callback may
3099 * assume the console is not within its write() callback.
3100 *
3101 * If @oops_in_progress is set, this may be an atomic context.
3102 * In that case, attempt a trylock as best-effort.
3103 */
3104 if (oops_in_progress) {
3105 /* Semaphores are not NMI-safe. */
3106 if (in_nmi())
3107 return;
3108
3109 /*
3110 * Attempting to trylock the console lock can deadlock
3111 * if another CPU was stopped while modifying the
3112 * semaphore. "Hope and pray" that this is not the
3113 * current situation.
3114 */
3115 if (down_trylock_console_sem() != 0)
3116 return;
3117 } else
3118 console_lock();
3119
3120 console_locked = 1;
3121 console_may_schedule = 0;
3122
3123 cookie = console_srcu_read_lock();
3124 for_each_console_srcu(c) {
3125 if ((console_srcu_read_flags(c) & CON_ENABLED) && c->unblank)
3126 c->unblank();
3127 }
3128 console_srcu_read_unlock(cookie);
3129
3130 console_unlock();
3131
3132 if (!oops_in_progress)
3133 pr_flush(1000, true);
3134 }
3135
3136 /**
3137 * console_flush_on_panic - flush console content on panic
3138 * @mode: flush all messages in buffer or just the pending ones
3139 *
3140 * Immediately output all pending messages no matter what.
3141 */
console_flush_on_panic(enum con_flush_mode mode)3142 void console_flush_on_panic(enum con_flush_mode mode)
3143 {
3144 bool handover;
3145 u64 next_seq;
3146
3147 /*
3148 * Ignore the console lock and flush out the messages. Attempting a
3149 * trylock would not be useful because:
3150 *
3151 * - if it is contended, it must be ignored anyway
3152 * - console_lock() and console_trylock() block and fail
3153 * respectively in panic for non-panic CPUs
3154 * - semaphores are not NMI-safe
3155 */
3156
3157 /*
3158 * If another context is holding the console lock,
3159 * @console_may_schedule might be set. Clear it so that
3160 * this context does not call cond_resched() while flushing.
3161 */
3162 console_may_schedule = 0;
3163
3164 if (mode == CONSOLE_REPLAY_ALL) {
3165 struct console *c;
3166 int cookie;
3167 u64 seq;
3168
3169 seq = prb_first_valid_seq(prb);
3170
3171 cookie = console_srcu_read_lock();
3172 for_each_console_srcu(c) {
3173 /*
3174 * This is an unsynchronized assignment, but the
3175 * kernel is in "hope and pray" mode anyway.
3176 */
3177 c->seq = seq;
3178 }
3179 console_srcu_read_unlock(cookie);
3180 }
3181
3182 console_flush_all(false, &next_seq, &handover);
3183 }
3184
3185 /*
3186 * Return the console tty driver structure and its associated index
3187 */
console_device(int * index)3188 struct tty_driver *console_device(int *index)
3189 {
3190 struct console *c;
3191 struct tty_driver *driver = NULL;
3192 int cookie;
3193
3194 /*
3195 * Take console_lock to serialize device() callback with
3196 * other console operations. For example, fg_console is
3197 * modified under console_lock when switching vt.
3198 */
3199 console_lock();
3200
3201 cookie = console_srcu_read_lock();
3202 for_each_console_srcu(c) {
3203 if (!c->device)
3204 continue;
3205 driver = c->device(c, index);
3206 if (driver)
3207 break;
3208 }
3209 console_srcu_read_unlock(cookie);
3210
3211 console_unlock();
3212 return driver;
3213 }
3214
3215 /*
3216 * Prevent further output on the passed console device so that (for example)
3217 * serial drivers can disable console output before suspending a port, and can
3218 * re-enable output afterwards.
3219 */
console_stop(struct console * console)3220 void console_stop(struct console *console)
3221 {
3222 __pr_flush(console, 1000, true);
3223 console_list_lock();
3224 console_srcu_write_flags(console, console->flags & ~CON_ENABLED);
3225 console_list_unlock();
3226
3227 /*
3228 * Ensure that all SRCU list walks have completed. All contexts must
3229 * be able to see that this console is disabled so that (for example)
3230 * the caller can suspend the port without risk of another context
3231 * using the port.
3232 */
3233 synchronize_srcu(&console_srcu);
3234 }
3235 EXPORT_SYMBOL(console_stop);
3236
console_start(struct console * console)3237 void console_start(struct console *console)
3238 {
3239 console_list_lock();
3240 console_srcu_write_flags(console, console->flags | CON_ENABLED);
3241 console_list_unlock();
3242 __pr_flush(console, 1000, true);
3243 }
3244 EXPORT_SYMBOL(console_start);
3245
3246 static int __read_mostly keep_bootcon;
3247
keep_bootcon_setup(char * str)3248 static int __init keep_bootcon_setup(char *str)
3249 {
3250 keep_bootcon = 1;
3251 pr_info("debug: skip boot console de-registration.\n");
3252
3253 return 0;
3254 }
3255
3256 early_param("keep_bootcon", keep_bootcon_setup);
3257
3258 /*
3259 * This is called by register_console() to try to match
3260 * the newly registered console with any of the ones selected
3261 * by either the command line or add_preferred_console() and
3262 * setup/enable it.
3263 *
3264 * Care need to be taken with consoles that are statically
3265 * enabled such as netconsole
3266 */
try_enable_preferred_console(struct console * newcon,bool user_specified)3267 static int try_enable_preferred_console(struct console *newcon,
3268 bool user_specified)
3269 {
3270 struct console_cmdline *c;
3271 int i, err;
3272
3273 for (i = 0, c = console_cmdline;
3274 i < MAX_CMDLINECONSOLES && c->name[0];
3275 i++, c++) {
3276 if (c->user_specified != user_specified)
3277 continue;
3278 if (!newcon->match ||
3279 newcon->match(newcon, c->name, c->index, c->options) != 0) {
3280 /* default matching */
3281 BUILD_BUG_ON(sizeof(c->name) != sizeof(newcon->name));
3282 if (strcmp(c->name, newcon->name) != 0)
3283 continue;
3284 if (newcon->index >= 0 &&
3285 newcon->index != c->index)
3286 continue;
3287 if (newcon->index < 0)
3288 newcon->index = c->index;
3289
3290 if (_braille_register_console(newcon, c))
3291 return 0;
3292
3293 if (newcon->setup &&
3294 (err = newcon->setup(newcon, c->options)) != 0)
3295 return err;
3296 }
3297 newcon->flags |= CON_ENABLED;
3298 if (i == preferred_console)
3299 newcon->flags |= CON_CONSDEV;
3300 return 0;
3301 }
3302
3303 /*
3304 * Some consoles, such as pstore and netconsole, can be enabled even
3305 * without matching. Accept the pre-enabled consoles only when match()
3306 * and setup() had a chance to be called.
3307 */
3308 if (newcon->flags & CON_ENABLED && c->user_specified == user_specified)
3309 return 0;
3310
3311 return -ENOENT;
3312 }
3313
3314 /* Try to enable the console unconditionally */
try_enable_default_console(struct console * newcon)3315 static void try_enable_default_console(struct console *newcon)
3316 {
3317 if (newcon->index < 0)
3318 newcon->index = 0;
3319
3320 if (newcon->setup && newcon->setup(newcon, NULL) != 0)
3321 return;
3322
3323 newcon->flags |= CON_ENABLED;
3324
3325 if (newcon->device)
3326 newcon->flags |= CON_CONSDEV;
3327 }
3328
3329 #define con_printk(lvl, con, fmt, ...) \
3330 printk(lvl pr_fmt("%sconsole [%s%d] " fmt), \
3331 (con->flags & CON_BOOT) ? "boot" : "", \
3332 con->name, con->index, ##__VA_ARGS__)
3333
console_init_seq(struct console * newcon,bool bootcon_registered)3334 static void console_init_seq(struct console *newcon, bool bootcon_registered)
3335 {
3336 struct console *con;
3337 bool handover;
3338
3339 if (newcon->flags & (CON_PRINTBUFFER | CON_BOOT)) {
3340 /* Get a consistent copy of @syslog_seq. */
3341 mutex_lock(&syslog_lock);
3342 newcon->seq = syslog_seq;
3343 mutex_unlock(&syslog_lock);
3344 } else {
3345 /* Begin with next message added to ringbuffer. */
3346 newcon->seq = prb_next_seq(prb);
3347
3348 /*
3349 * If any enabled boot consoles are due to be unregistered
3350 * shortly, some may not be caught up and may be the same
3351 * device as @newcon. Since it is not known which boot console
3352 * is the same device, flush all consoles and, if necessary,
3353 * start with the message of the enabled boot console that is
3354 * the furthest behind.
3355 */
3356 if (bootcon_registered && !keep_bootcon) {
3357 /*
3358 * Hold the console_lock to stop console printing and
3359 * guarantee safe access to console->seq.
3360 */
3361 console_lock();
3362
3363 /*
3364 * Flush all consoles and set the console to start at
3365 * the next unprinted sequence number.
3366 */
3367 if (!console_flush_all(true, &newcon->seq, &handover)) {
3368 /*
3369 * Flushing failed. Just choose the lowest
3370 * sequence of the enabled boot consoles.
3371 */
3372
3373 /*
3374 * If there was a handover, this context no
3375 * longer holds the console_lock.
3376 */
3377 if (handover)
3378 console_lock();
3379
3380 newcon->seq = prb_next_seq(prb);
3381 for_each_console(con) {
3382 if ((con->flags & CON_BOOT) &&
3383 (con->flags & CON_ENABLED) &&
3384 con->seq < newcon->seq) {
3385 newcon->seq = con->seq;
3386 }
3387 }
3388 }
3389
3390 console_unlock();
3391 }
3392 }
3393 }
3394
3395 #define console_first() \
3396 hlist_entry(console_list.first, struct console, node)
3397
3398 static int unregister_console_locked(struct console *console);
3399
3400 /*
3401 * The console driver calls this routine during kernel initialization
3402 * to register the console printing procedure with printk() and to
3403 * print any messages that were printed by the kernel before the
3404 * console driver was initialized.
3405 *
3406 * This can happen pretty early during the boot process (because of
3407 * early_printk) - sometimes before setup_arch() completes - be careful
3408 * of what kernel features are used - they may not be initialised yet.
3409 *
3410 * There are two types of consoles - bootconsoles (early_printk) and
3411 * "real" consoles (everything which is not a bootconsole) which are
3412 * handled differently.
3413 * - Any number of bootconsoles can be registered at any time.
3414 * - As soon as a "real" console is registered, all bootconsoles
3415 * will be unregistered automatically.
3416 * - Once a "real" console is registered, any attempt to register a
3417 * bootconsoles will be rejected
3418 */
register_console(struct console * newcon)3419 void register_console(struct console *newcon)
3420 {
3421 struct console *con;
3422 bool bootcon_registered = false;
3423 bool realcon_registered = false;
3424 int err;
3425
3426 console_list_lock();
3427
3428 for_each_console(con) {
3429 if (WARN(con == newcon, "console '%s%d' already registered\n",
3430 con->name, con->index)) {
3431 goto unlock;
3432 }
3433
3434 if (con->flags & CON_BOOT)
3435 bootcon_registered = true;
3436 else
3437 realcon_registered = true;
3438 }
3439
3440 /* Do not register boot consoles when there already is a real one. */
3441 if ((newcon->flags & CON_BOOT) && realcon_registered) {
3442 pr_info("Too late to register bootconsole %s%d\n",
3443 newcon->name, newcon->index);
3444 goto unlock;
3445 }
3446
3447 /*
3448 * See if we want to enable this console driver by default.
3449 *
3450 * Nope when a console is preferred by the command line, device
3451 * tree, or SPCR.
3452 *
3453 * The first real console with tty binding (driver) wins. More
3454 * consoles might get enabled before the right one is found.
3455 *
3456 * Note that a console with tty binding will have CON_CONSDEV
3457 * flag set and will be first in the list.
3458 */
3459 if (preferred_console < 0) {
3460 if (hlist_empty(&console_list) || !console_first()->device ||
3461 console_first()->flags & CON_BOOT) {
3462 try_enable_default_console(newcon);
3463 }
3464 }
3465
3466 /* See if this console matches one we selected on the command line */
3467 err = try_enable_preferred_console(newcon, true);
3468
3469 /* If not, try to match against the platform default(s) */
3470 if (err == -ENOENT)
3471 err = try_enable_preferred_console(newcon, false);
3472
3473 /* printk() messages are not printed to the Braille console. */
3474 if (err || newcon->flags & CON_BRL)
3475 goto unlock;
3476
3477 /*
3478 * If we have a bootconsole, and are switching to a real console,
3479 * don't print everything out again, since when the boot console, and
3480 * the real console are the same physical device, it's annoying to
3481 * see the beginning boot messages twice
3482 */
3483 if (bootcon_registered &&
3484 ((newcon->flags & (CON_CONSDEV | CON_BOOT)) == CON_CONSDEV)) {
3485 newcon->flags &= ~CON_PRINTBUFFER;
3486 }
3487
3488 newcon->dropped = 0;
3489 console_init_seq(newcon, bootcon_registered);
3490
3491 /*
3492 * Put this console in the list - keep the
3493 * preferred driver at the head of the list.
3494 */
3495 if (hlist_empty(&console_list)) {
3496 /* Ensure CON_CONSDEV is always set for the head. */
3497 newcon->flags |= CON_CONSDEV;
3498 hlist_add_head_rcu(&newcon->node, &console_list);
3499
3500 } else if (newcon->flags & CON_CONSDEV) {
3501 /* Only the new head can have CON_CONSDEV set. */
3502 console_srcu_write_flags(console_first(), console_first()->flags & ~CON_CONSDEV);
3503 hlist_add_head_rcu(&newcon->node, &console_list);
3504
3505 } else {
3506 hlist_add_behind_rcu(&newcon->node, console_list.first);
3507 }
3508
3509 /*
3510 * No need to synchronize SRCU here! The caller does not rely
3511 * on all contexts being able to see the new console before
3512 * register_console() completes.
3513 */
3514
3515 console_sysfs_notify();
3516
3517 /*
3518 * By unregistering the bootconsoles after we enable the real console
3519 * we get the "console xxx enabled" message on all the consoles -
3520 * boot consoles, real consoles, etc - this is to ensure that end
3521 * users know there might be something in the kernel's log buffer that
3522 * went to the bootconsole (that they do not see on the real console)
3523 */
3524 con_printk(KERN_INFO, newcon, "enabled\n");
3525 if (bootcon_registered &&
3526 ((newcon->flags & (CON_CONSDEV | CON_BOOT)) == CON_CONSDEV) &&
3527 !keep_bootcon) {
3528 struct hlist_node *tmp;
3529
3530 hlist_for_each_entry_safe(con, tmp, &console_list, node) {
3531 if (con->flags & CON_BOOT)
3532 unregister_console_locked(con);
3533 }
3534 }
3535 unlock:
3536 console_list_unlock();
3537 }
3538 EXPORT_SYMBOL(register_console);
3539
3540 /* Must be called under console_list_lock(). */
unregister_console_locked(struct console * console)3541 static int unregister_console_locked(struct console *console)
3542 {
3543 int res;
3544
3545 lockdep_assert_console_list_lock_held();
3546
3547 con_printk(KERN_INFO, console, "disabled\n");
3548
3549 res = _braille_unregister_console(console);
3550 if (res < 0)
3551 return res;
3552 if (res > 0)
3553 return 0;
3554
3555 /* Disable it unconditionally */
3556 console_srcu_write_flags(console, console->flags & ~CON_ENABLED);
3557
3558 if (!console_is_registered_locked(console))
3559 return -ENODEV;
3560
3561 hlist_del_init_rcu(&console->node);
3562
3563 /*
3564 * <HISTORICAL>
3565 * If this isn't the last console and it has CON_CONSDEV set, we
3566 * need to set it on the next preferred console.
3567 * </HISTORICAL>
3568 *
3569 * The above makes no sense as there is no guarantee that the next
3570 * console has any device attached. Oh well....
3571 */
3572 if (!hlist_empty(&console_list) && console->flags & CON_CONSDEV)
3573 console_srcu_write_flags(console_first(), console_first()->flags | CON_CONSDEV);
3574
3575 /*
3576 * Ensure that all SRCU list walks have completed. All contexts
3577 * must not be able to see this console in the list so that any
3578 * exit/cleanup routines can be performed safely.
3579 */
3580 synchronize_srcu(&console_srcu);
3581
3582 console_sysfs_notify();
3583
3584 if (console->exit)
3585 res = console->exit(console);
3586
3587 return res;
3588 }
3589
unregister_console(struct console * console)3590 int unregister_console(struct console *console)
3591 {
3592 int res;
3593
3594 console_list_lock();
3595 res = unregister_console_locked(console);
3596 console_list_unlock();
3597 return res;
3598 }
3599 EXPORT_SYMBOL(unregister_console);
3600
3601 /**
3602 * console_force_preferred_locked - force a registered console preferred
3603 * @con: The registered console to force preferred.
3604 *
3605 * Must be called under console_list_lock().
3606 */
console_force_preferred_locked(struct console * con)3607 void console_force_preferred_locked(struct console *con)
3608 {
3609 struct console *cur_pref_con;
3610
3611 if (!console_is_registered_locked(con))
3612 return;
3613
3614 cur_pref_con = console_first();
3615
3616 /* Already preferred? */
3617 if (cur_pref_con == con)
3618 return;
3619
3620 /*
3621 * Delete, but do not re-initialize the entry. This allows the console
3622 * to continue to appear registered (via any hlist_unhashed_lockless()
3623 * checks), even though it was briefly removed from the console list.
3624 */
3625 hlist_del_rcu(&con->node);
3626
3627 /*
3628 * Ensure that all SRCU list walks have completed so that the console
3629 * can be added to the beginning of the console list and its forward
3630 * list pointer can be re-initialized.
3631 */
3632 synchronize_srcu(&console_srcu);
3633
3634 con->flags |= CON_CONSDEV;
3635 WARN_ON(!con->device);
3636
3637 /* Only the new head can have CON_CONSDEV set. */
3638 console_srcu_write_flags(cur_pref_con, cur_pref_con->flags & ~CON_CONSDEV);
3639 hlist_add_head_rcu(&con->node, &console_list);
3640 }
3641 EXPORT_SYMBOL(console_force_preferred_locked);
3642
3643 /*
3644 * Initialize the console device. This is called *early*, so
3645 * we can't necessarily depend on lots of kernel help here.
3646 * Just do some early initializations, and do the complex setup
3647 * later.
3648 */
console_init(void)3649 void __init console_init(void)
3650 {
3651 int ret;
3652 initcall_t call;
3653 initcall_entry_t *ce;
3654
3655 /* Setup the default TTY line discipline. */
3656 n_tty_init();
3657
3658 /*
3659 * set up the console device so that later boot sequences can
3660 * inform about problems etc..
3661 */
3662 ce = __con_initcall_start;
3663 trace_initcall_level("console");
3664 while (ce < __con_initcall_end) {
3665 call = initcall_from_entry(ce);
3666 trace_initcall_start(call);
3667 ret = call();
3668 trace_initcall_finish(call, ret);
3669 ce++;
3670 }
3671 }
3672
3673 /*
3674 * Some boot consoles access data that is in the init section and which will
3675 * be discarded after the initcalls have been run. To make sure that no code
3676 * will access this data, unregister the boot consoles in a late initcall.
3677 *
3678 * If for some reason, such as deferred probe or the driver being a loadable
3679 * module, the real console hasn't registered yet at this point, there will
3680 * be a brief interval in which no messages are logged to the console, which
3681 * makes it difficult to diagnose problems that occur during this time.
3682 *
3683 * To mitigate this problem somewhat, only unregister consoles whose memory
3684 * intersects with the init section. Note that all other boot consoles will
3685 * get unregistered when the real preferred console is registered.
3686 */
printk_late_init(void)3687 static int __init printk_late_init(void)
3688 {
3689 struct hlist_node *tmp;
3690 struct console *con;
3691 int ret;
3692
3693 console_list_lock();
3694 hlist_for_each_entry_safe(con, tmp, &console_list, node) {
3695 if (!(con->flags & CON_BOOT))
3696 continue;
3697
3698 /* Check addresses that might be used for enabled consoles. */
3699 if (init_section_intersects(con, sizeof(*con)) ||
3700 init_section_contains(con->write, 0) ||
3701 init_section_contains(con->read, 0) ||
3702 init_section_contains(con->device, 0) ||
3703 init_section_contains(con->unblank, 0) ||
3704 init_section_contains(con->data, 0)) {
3705 /*
3706 * Please, consider moving the reported consoles out
3707 * of the init section.
3708 */
3709 pr_warn("bootconsole [%s%d] uses init memory and must be disabled even before the real one is ready\n",
3710 con->name, con->index);
3711 unregister_console_locked(con);
3712 }
3713 }
3714 console_list_unlock();
3715
3716 ret = cpuhp_setup_state_nocalls(CPUHP_PRINTK_DEAD, "printk:dead", NULL,
3717 console_cpu_notify);
3718 WARN_ON(ret < 0);
3719 ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "printk:online",
3720 console_cpu_notify, NULL);
3721 WARN_ON(ret < 0);
3722 printk_sysctl_init();
3723 return 0;
3724 }
3725 late_initcall(printk_late_init);
3726
3727 #if defined CONFIG_PRINTK
3728 /* If @con is specified, only wait for that console. Otherwise wait for all. */
__pr_flush(struct console * con,int timeout_ms,bool reset_on_progress)3729 static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progress)
3730 {
3731 int remaining = timeout_ms;
3732 struct console *c;
3733 u64 last_diff = 0;
3734 u64 printk_seq;
3735 int cookie;
3736 u64 diff;
3737 u64 seq;
3738
3739 might_sleep();
3740
3741 seq = prb_next_seq(prb);
3742
3743 /* Flush the consoles so that records up to @seq are printed. */
3744 console_lock();
3745 console_unlock();
3746
3747 for (;;) {
3748 diff = 0;
3749
3750 /*
3751 * Hold the console_lock to guarantee safe access to
3752 * console->seq. Releasing console_lock flushes more
3753 * records in case @seq is still not printed on all
3754 * usable consoles.
3755 */
3756 console_lock();
3757
3758 cookie = console_srcu_read_lock();
3759 for_each_console_srcu(c) {
3760 if (con && con != c)
3761 continue;
3762 /*
3763 * If consoles are not usable, it cannot be expected
3764 * that they make forward progress, so only increment
3765 * @diff for usable consoles.
3766 */
3767 if (!console_is_usable(c))
3768 continue;
3769 printk_seq = c->seq;
3770 if (printk_seq < seq)
3771 diff += seq - printk_seq;
3772 }
3773 console_srcu_read_unlock(cookie);
3774
3775 if (diff != last_diff && reset_on_progress)
3776 remaining = timeout_ms;
3777
3778 console_unlock();
3779
3780 /* Note: @diff is 0 if there are no usable consoles. */
3781 if (diff == 0 || remaining == 0)
3782 break;
3783
3784 if (remaining < 0) {
3785 /* no timeout limit */
3786 msleep(100);
3787 } else if (remaining < 100) {
3788 msleep(remaining);
3789 remaining = 0;
3790 } else {
3791 msleep(100);
3792 remaining -= 100;
3793 }
3794
3795 last_diff = diff;
3796 }
3797
3798 return (diff == 0);
3799 }
3800
3801 /**
3802 * pr_flush() - Wait for printing threads to catch up.
3803 *
3804 * @timeout_ms: The maximum time (in ms) to wait.
3805 * @reset_on_progress: Reset the timeout if forward progress is seen.
3806 *
3807 * A value of 0 for @timeout_ms means no waiting will occur. A value of -1
3808 * represents infinite waiting.
3809 *
3810 * If @reset_on_progress is true, the timeout will be reset whenever any
3811 * printer has been seen to make some forward progress.
3812 *
3813 * Context: Process context. May sleep while acquiring console lock.
3814 * Return: true if all usable printers are caught up.
3815 */
pr_flush(int timeout_ms,bool reset_on_progress)3816 static bool pr_flush(int timeout_ms, bool reset_on_progress)
3817 {
3818 return __pr_flush(NULL, timeout_ms, reset_on_progress);
3819 }
3820
3821 /*
3822 * Delayed printk version, for scheduler-internal messages:
3823 */
3824 #define PRINTK_PENDING_WAKEUP 0x01
3825 #define PRINTK_PENDING_OUTPUT 0x02
3826
3827 static DEFINE_PER_CPU(int, printk_pending);
3828
wake_up_klogd_work_func(struct irq_work * irq_work)3829 static void wake_up_klogd_work_func(struct irq_work *irq_work)
3830 {
3831 int pending = this_cpu_xchg(printk_pending, 0);
3832
3833 if (pending & PRINTK_PENDING_OUTPUT) {
3834 /* If trylock fails, someone else is doing the printing */
3835 if (console_trylock())
3836 console_unlock();
3837 }
3838
3839 if (pending & PRINTK_PENDING_WAKEUP)
3840 wake_up_interruptible(&log_wait);
3841 }
3842
3843 static DEFINE_PER_CPU(struct irq_work, wake_up_klogd_work) =
3844 IRQ_WORK_INIT_LAZY(wake_up_klogd_work_func);
3845
__wake_up_klogd(int val)3846 static void __wake_up_klogd(int val)
3847 {
3848 if (!printk_percpu_data_ready())
3849 return;
3850
3851 preempt_disable();
3852 /*
3853 * Guarantee any new records can be seen by tasks preparing to wait
3854 * before this context checks if the wait queue is empty.
3855 *
3856 * The full memory barrier within wq_has_sleeper() pairs with the full
3857 * memory barrier within set_current_state() of
3858 * prepare_to_wait_event(), which is called after ___wait_event() adds
3859 * the waiter but before it has checked the wait condition.
3860 *
3861 * This pairs with devkmsg_read:A and syslog_print:A.
3862 */
3863 if (wq_has_sleeper(&log_wait) || /* LMM(__wake_up_klogd:A) */
3864 (val & PRINTK_PENDING_OUTPUT)) {
3865 this_cpu_or(printk_pending, val);
3866 irq_work_queue(this_cpu_ptr(&wake_up_klogd_work));
3867 }
3868 preempt_enable();
3869 }
3870
3871 /**
3872 * wake_up_klogd - Wake kernel logging daemon
3873 *
3874 * Use this function when new records have been added to the ringbuffer
3875 * and the console printing of those records has already occurred or is
3876 * known to be handled by some other context. This function will only
3877 * wake the logging daemon.
3878 *
3879 * Context: Any context.
3880 */
wake_up_klogd(void)3881 void wake_up_klogd(void)
3882 {
3883 __wake_up_klogd(PRINTK_PENDING_WAKEUP);
3884 }
3885
3886 /**
3887 * defer_console_output - Wake kernel logging daemon and trigger
3888 * console printing in a deferred context
3889 *
3890 * Use this function when new records have been added to the ringbuffer,
3891 * this context is responsible for console printing those records, but
3892 * the current context is not allowed to perform the console printing.
3893 * Trigger an irq_work context to perform the console printing. This
3894 * function also wakes the logging daemon.
3895 *
3896 * Context: Any context.
3897 */
defer_console_output(void)3898 void defer_console_output(void)
3899 {
3900 /*
3901 * New messages may have been added directly to the ringbuffer
3902 * using vprintk_store(), so wake any waiters as well.
3903 */
3904 __wake_up_klogd(PRINTK_PENDING_WAKEUP | PRINTK_PENDING_OUTPUT);
3905 }
3906
printk_trigger_flush(void)3907 void printk_trigger_flush(void)
3908 {
3909 defer_console_output();
3910 }
3911
vprintk_deferred(const char * fmt,va_list args)3912 int vprintk_deferred(const char *fmt, va_list args)
3913 {
3914 return vprintk_emit(0, LOGLEVEL_SCHED, NULL, fmt, args);
3915 }
3916
_printk_deferred(const char * fmt,...)3917 int _printk_deferred(const char *fmt, ...)
3918 {
3919 va_list args;
3920 int r;
3921
3922 va_start(args, fmt);
3923 r = vprintk_deferred(fmt, args);
3924 va_end(args);
3925
3926 return r;
3927 }
3928
3929 /*
3930 * printk rate limiting, lifted from the networking subsystem.
3931 *
3932 * This enforces a rate limit: not more than 10 kernel messages
3933 * every 5s to make a denial-of-service attack impossible.
3934 */
3935 DEFINE_RATELIMIT_STATE(printk_ratelimit_state, 5 * HZ, 10);
3936
__printk_ratelimit(const char * func)3937 int __printk_ratelimit(const char *func)
3938 {
3939 return ___ratelimit(&printk_ratelimit_state, func);
3940 }
3941 EXPORT_SYMBOL(__printk_ratelimit);
3942
3943 /**
3944 * printk_timed_ratelimit - caller-controlled printk ratelimiting
3945 * @caller_jiffies: pointer to caller's state
3946 * @interval_msecs: minimum interval between prints
3947 *
3948 * printk_timed_ratelimit() returns true if more than @interval_msecs
3949 * milliseconds have elapsed since the last time printk_timed_ratelimit()
3950 * returned true.
3951 */
printk_timed_ratelimit(unsigned long * caller_jiffies,unsigned int interval_msecs)3952 bool printk_timed_ratelimit(unsigned long *caller_jiffies,
3953 unsigned int interval_msecs)
3954 {
3955 unsigned long elapsed = jiffies - *caller_jiffies;
3956
3957 if (*caller_jiffies && elapsed <= msecs_to_jiffies(interval_msecs))
3958 return false;
3959
3960 *caller_jiffies = jiffies;
3961 return true;
3962 }
3963 EXPORT_SYMBOL(printk_timed_ratelimit);
3964
3965 static DEFINE_SPINLOCK(dump_list_lock);
3966 static LIST_HEAD(dump_list);
3967
3968 /**
3969 * kmsg_dump_register - register a kernel log dumper.
3970 * @dumper: pointer to the kmsg_dumper structure
3971 *
3972 * Adds a kernel log dumper to the system. The dump callback in the
3973 * structure will be called when the kernel oopses or panics and must be
3974 * set. Returns zero on success and %-EINVAL or %-EBUSY otherwise.
3975 */
kmsg_dump_register(struct kmsg_dumper * dumper)3976 int kmsg_dump_register(struct kmsg_dumper *dumper)
3977 {
3978 unsigned long flags;
3979 int err = -EBUSY;
3980
3981 /* The dump callback needs to be set */
3982 if (!dumper->dump)
3983 return -EINVAL;
3984
3985 spin_lock_irqsave(&dump_list_lock, flags);
3986 /* Don't allow registering multiple times */
3987 if (!dumper->registered) {
3988 dumper->registered = 1;
3989 list_add_tail_rcu(&dumper->list, &dump_list);
3990 err = 0;
3991 }
3992 spin_unlock_irqrestore(&dump_list_lock, flags);
3993
3994 return err;
3995 }
3996 EXPORT_SYMBOL_GPL(kmsg_dump_register);
3997
3998 /**
3999 * kmsg_dump_unregister - unregister a kmsg dumper.
4000 * @dumper: pointer to the kmsg_dumper structure
4001 *
4002 * Removes a dump device from the system. Returns zero on success and
4003 * %-EINVAL otherwise.
4004 */
kmsg_dump_unregister(struct kmsg_dumper * dumper)4005 int kmsg_dump_unregister(struct kmsg_dumper *dumper)
4006 {
4007 unsigned long flags;
4008 int err = -EINVAL;
4009
4010 spin_lock_irqsave(&dump_list_lock, flags);
4011 if (dumper->registered) {
4012 dumper->registered = 0;
4013 list_del_rcu(&dumper->list);
4014 err = 0;
4015 }
4016 spin_unlock_irqrestore(&dump_list_lock, flags);
4017 synchronize_rcu();
4018
4019 return err;
4020 }
4021 EXPORT_SYMBOL_GPL(kmsg_dump_unregister);
4022
4023 static bool always_kmsg_dump;
4024 module_param_named(always_kmsg_dump, always_kmsg_dump, bool, S_IRUGO | S_IWUSR);
4025
kmsg_dump_reason_str(enum kmsg_dump_reason reason)4026 const char *kmsg_dump_reason_str(enum kmsg_dump_reason reason)
4027 {
4028 switch (reason) {
4029 case KMSG_DUMP_PANIC:
4030 return "Panic";
4031 case KMSG_DUMP_OOPS:
4032 return "Oops";
4033 case KMSG_DUMP_EMERG:
4034 return "Emergency";
4035 case KMSG_DUMP_SHUTDOWN:
4036 return "Shutdown";
4037 default:
4038 return "Unknown";
4039 }
4040 }
4041 EXPORT_SYMBOL_GPL(kmsg_dump_reason_str);
4042
4043 /**
4044 * kmsg_dump - dump kernel log to kernel message dumpers.
4045 * @reason: the reason (oops, panic etc) for dumping
4046 *
4047 * Call each of the registered dumper's dump() callback, which can
4048 * retrieve the kmsg records with kmsg_dump_get_line() or
4049 * kmsg_dump_get_buffer().
4050 */
kmsg_dump(enum kmsg_dump_reason reason)4051 void kmsg_dump(enum kmsg_dump_reason reason)
4052 {
4053 struct kmsg_dumper *dumper;
4054
4055 rcu_read_lock();
4056 list_for_each_entry_rcu(dumper, &dump_list, list) {
4057 enum kmsg_dump_reason max_reason = dumper->max_reason;
4058
4059 /*
4060 * If client has not provided a specific max_reason, default
4061 * to KMSG_DUMP_OOPS, unless always_kmsg_dump was set.
4062 */
4063 if (max_reason == KMSG_DUMP_UNDEF) {
4064 max_reason = always_kmsg_dump ? KMSG_DUMP_MAX :
4065 KMSG_DUMP_OOPS;
4066 }
4067 if (reason > max_reason)
4068 continue;
4069
4070 /* invoke dumper which will iterate over records */
4071 dumper->dump(dumper, reason);
4072 }
4073 rcu_read_unlock();
4074 }
4075
4076 /**
4077 * kmsg_dump_get_line - retrieve one kmsg log line
4078 * @iter: kmsg dump iterator
4079 * @syslog: include the "<4>" prefixes
4080 * @line: buffer to copy the line to
4081 * @size: maximum size of the buffer
4082 * @len: length of line placed into buffer
4083 *
4084 * Start at the beginning of the kmsg buffer, with the oldest kmsg
4085 * record, and copy one record into the provided buffer.
4086 *
4087 * Consecutive calls will return the next available record moving
4088 * towards the end of the buffer with the youngest messages.
4089 *
4090 * A return value of FALSE indicates that there are no more records to
4091 * read.
4092 */
kmsg_dump_get_line(struct kmsg_dump_iter * iter,bool syslog,char * line,size_t size,size_t * len)4093 bool kmsg_dump_get_line(struct kmsg_dump_iter *iter, bool syslog,
4094 char *line, size_t size, size_t *len)
4095 {
4096 u64 min_seq = latched_seq_read_nolock(&clear_seq);
4097 struct printk_info info;
4098 unsigned int line_count;
4099 struct printk_record r;
4100 size_t l = 0;
4101 bool ret = false;
4102
4103 if (iter->cur_seq < min_seq)
4104 iter->cur_seq = min_seq;
4105
4106 prb_rec_init_rd(&r, &info, line, size);
4107
4108 /* Read text or count text lines? */
4109 if (line) {
4110 if (!prb_read_valid(prb, iter->cur_seq, &r))
4111 goto out;
4112 l = record_print_text(&r, syslog, printk_time);
4113 } else {
4114 if (!prb_read_valid_info(prb, iter->cur_seq,
4115 &info, &line_count)) {
4116 goto out;
4117 }
4118 l = get_record_print_text_size(&info, line_count, syslog,
4119 printk_time);
4120
4121 }
4122
4123 iter->cur_seq = r.info->seq + 1;
4124 ret = true;
4125 out:
4126 if (len)
4127 *len = l;
4128 return ret;
4129 }
4130 EXPORT_SYMBOL_GPL(kmsg_dump_get_line);
4131
4132 /**
4133 * kmsg_dump_get_buffer - copy kmsg log lines
4134 * @iter: kmsg dump iterator
4135 * @syslog: include the "<4>" prefixes
4136 * @buf: buffer to copy the line to
4137 * @size: maximum size of the buffer
4138 * @len_out: length of line placed into buffer
4139 *
4140 * Start at the end of the kmsg buffer and fill the provided buffer
4141 * with as many of the *youngest* kmsg records that fit into it.
4142 * If the buffer is large enough, all available kmsg records will be
4143 * copied with a single call.
4144 *
4145 * Consecutive calls will fill the buffer with the next block of
4146 * available older records, not including the earlier retrieved ones.
4147 *
4148 * A return value of FALSE indicates that there are no more records to
4149 * read.
4150 */
kmsg_dump_get_buffer(struct kmsg_dump_iter * iter,bool syslog,char * buf,size_t size,size_t * len_out)4151 bool kmsg_dump_get_buffer(struct kmsg_dump_iter *iter, bool syslog,
4152 char *buf, size_t size, size_t *len_out)
4153 {
4154 u64 min_seq = latched_seq_read_nolock(&clear_seq);
4155 struct printk_info info;
4156 struct printk_record r;
4157 u64 seq;
4158 u64 next_seq;
4159 size_t len = 0;
4160 bool ret = false;
4161 bool time = printk_time;
4162
4163 if (!buf || !size)
4164 goto out;
4165
4166 if (iter->cur_seq < min_seq)
4167 iter->cur_seq = min_seq;
4168
4169 if (prb_read_valid_info(prb, iter->cur_seq, &info, NULL)) {
4170 if (info.seq != iter->cur_seq) {
4171 /* messages are gone, move to first available one */
4172 iter->cur_seq = info.seq;
4173 }
4174 }
4175
4176 /* last entry */
4177 if (iter->cur_seq >= iter->next_seq)
4178 goto out;
4179
4180 /*
4181 * Find first record that fits, including all following records,
4182 * into the user-provided buffer for this dump. Pass in size-1
4183 * because this function (by way of record_print_text()) will
4184 * not write more than size-1 bytes of text into @buf.
4185 */
4186 seq = find_first_fitting_seq(iter->cur_seq, iter->next_seq,
4187 size - 1, syslog, time);
4188
4189 /*
4190 * Next kmsg_dump_get_buffer() invocation will dump block of
4191 * older records stored right before this one.
4192 */
4193 next_seq = seq;
4194
4195 prb_rec_init_rd(&r, &info, buf, size);
4196
4197 len = 0;
4198 prb_for_each_record(seq, prb, seq, &r) {
4199 if (r.info->seq >= iter->next_seq)
4200 break;
4201
4202 len += record_print_text(&r, syslog, time);
4203
4204 /* Adjust record to store to remaining buffer space. */
4205 prb_rec_init_rd(&r, &info, buf + len, size - len);
4206 }
4207
4208 iter->next_seq = next_seq;
4209 ret = true;
4210 out:
4211 if (len_out)
4212 *len_out = len;
4213 return ret;
4214 }
4215 EXPORT_SYMBOL_GPL(kmsg_dump_get_buffer);
4216
4217 /**
4218 * kmsg_dump_rewind - reset the iterator
4219 * @iter: kmsg dump iterator
4220 *
4221 * Reset the dumper's iterator so that kmsg_dump_get_line() and
4222 * kmsg_dump_get_buffer() can be called again and used multiple
4223 * times within the same dumper.dump() callback.
4224 */
kmsg_dump_rewind(struct kmsg_dump_iter * iter)4225 void kmsg_dump_rewind(struct kmsg_dump_iter *iter)
4226 {
4227 iter->cur_seq = latched_seq_read_nolock(&clear_seq);
4228 iter->next_seq = prb_next_seq(prb);
4229 }
4230 EXPORT_SYMBOL_GPL(kmsg_dump_rewind);
4231
4232 #endif
4233
4234 #ifdef CONFIG_SMP
4235 static atomic_t printk_cpu_sync_owner = ATOMIC_INIT(-1);
4236 static atomic_t printk_cpu_sync_nested = ATOMIC_INIT(0);
4237
4238 /**
4239 * __printk_cpu_sync_wait() - Busy wait until the printk cpu-reentrant
4240 * spinning lock is not owned by any CPU.
4241 *
4242 * Context: Any context.
4243 */
__printk_cpu_sync_wait(void)4244 void __printk_cpu_sync_wait(void)
4245 {
4246 do {
4247 cpu_relax();
4248 } while (atomic_read(&printk_cpu_sync_owner) != -1);
4249 }
4250 EXPORT_SYMBOL(__printk_cpu_sync_wait);
4251
4252 /**
4253 * __printk_cpu_sync_try_get() - Try to acquire the printk cpu-reentrant
4254 * spinning lock.
4255 *
4256 * If no processor has the lock, the calling processor takes the lock and
4257 * becomes the owner. If the calling processor is already the owner of the
4258 * lock, this function succeeds immediately.
4259 *
4260 * Context: Any context. Expects interrupts to be disabled.
4261 * Return: 1 on success, otherwise 0.
4262 */
__printk_cpu_sync_try_get(void)4263 int __printk_cpu_sync_try_get(void)
4264 {
4265 int cpu;
4266 int old;
4267
4268 cpu = smp_processor_id();
4269
4270 /*
4271 * Guarantee loads and stores from this CPU when it is the lock owner
4272 * are _not_ visible to the previous lock owner. This pairs with
4273 * __printk_cpu_sync_put:B.
4274 *
4275 * Memory barrier involvement:
4276 *
4277 * If __printk_cpu_sync_try_get:A reads from __printk_cpu_sync_put:B,
4278 * then __printk_cpu_sync_put:A can never read from
4279 * __printk_cpu_sync_try_get:B.
4280 *
4281 * Relies on:
4282 *
4283 * RELEASE from __printk_cpu_sync_put:A to __printk_cpu_sync_put:B
4284 * of the previous CPU
4285 * matching
4286 * ACQUIRE from __printk_cpu_sync_try_get:A to
4287 * __printk_cpu_sync_try_get:B of this CPU
4288 */
4289 old = atomic_cmpxchg_acquire(&printk_cpu_sync_owner, -1,
4290 cpu); /* LMM(__printk_cpu_sync_try_get:A) */
4291 if (old == -1) {
4292 /*
4293 * This CPU is now the owner and begins loading/storing
4294 * data: LMM(__printk_cpu_sync_try_get:B)
4295 */
4296 return 1;
4297
4298 } else if (old == cpu) {
4299 /* This CPU is already the owner. */
4300 atomic_inc(&printk_cpu_sync_nested);
4301 return 1;
4302 }
4303
4304 return 0;
4305 }
4306 EXPORT_SYMBOL(__printk_cpu_sync_try_get);
4307
4308 /**
4309 * __printk_cpu_sync_put() - Release the printk cpu-reentrant spinning lock.
4310 *
4311 * The calling processor must be the owner of the lock.
4312 *
4313 * Context: Any context. Expects interrupts to be disabled.
4314 */
__printk_cpu_sync_put(void)4315 void __printk_cpu_sync_put(void)
4316 {
4317 if (atomic_read(&printk_cpu_sync_nested)) {
4318 atomic_dec(&printk_cpu_sync_nested);
4319 return;
4320 }
4321
4322 /*
4323 * This CPU is finished loading/storing data:
4324 * LMM(__printk_cpu_sync_put:A)
4325 */
4326
4327 /*
4328 * Guarantee loads and stores from this CPU when it was the
4329 * lock owner are visible to the next lock owner. This pairs
4330 * with __printk_cpu_sync_try_get:A.
4331 *
4332 * Memory barrier involvement:
4333 *
4334 * If __printk_cpu_sync_try_get:A reads from __printk_cpu_sync_put:B,
4335 * then __printk_cpu_sync_try_get:B reads from __printk_cpu_sync_put:A.
4336 *
4337 * Relies on:
4338 *
4339 * RELEASE from __printk_cpu_sync_put:A to __printk_cpu_sync_put:B
4340 * of this CPU
4341 * matching
4342 * ACQUIRE from __printk_cpu_sync_try_get:A to
4343 * __printk_cpu_sync_try_get:B of the next CPU
4344 */
4345 atomic_set_release(&printk_cpu_sync_owner,
4346 -1); /* LMM(__printk_cpu_sync_put:B) */
4347 }
4348 EXPORT_SYMBOL(__printk_cpu_sync_put);
4349 #endif /* CONFIG_SMP */
4350