1#!/usr/bin/perl -w
2# SPDX-License-Identifier: GPL-2.0-only
3#
4# Copyright 2010 - Steven Rostedt <srostedt@redhat.com>, Red Hat Inc.
5#
6
7use strict;
8use IPC::Open2;
9use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK);
10use File::Path qw(mkpath);
11use File::Copy qw(cp);
12use FileHandle;
13use FindBin;
14use IO::Handle;
15
16my $VERSION = "0.2";
17
18$| = 1;
19
20my %opt;
21my %repeat_tests;
22my %repeats;
23my %evals;
24
25#default opts
26my %default = (
27    "MAILER"			=> "sendmail",	# default mailer
28    "EMAIL_ON_ERROR"		=> 1,
29    "EMAIL_WHEN_FINISHED"	=> 1,
30    "EMAIL_WHEN_CANCELED"	=> 0,
31    "EMAIL_WHEN_STARTED"	=> 0,
32    "NUM_TESTS"			=> 1,
33    "TEST_TYPE"			=> "build",
34    "BUILD_TYPE"		=> "oldconfig",
35    "MAKE_CMD"			=> "make",
36    "CLOSE_CONSOLE_SIGNAL"	=> "INT",
37    "TIMEOUT"			=> 120,
38    "TMP_DIR"			=> "/tmp/ktest/\${MACHINE}",
39    "SLEEP_TIME"		=> 60,		# sleep time between tests
40    "BUILD_NOCLEAN"		=> 0,
41    "REBOOT_ON_ERROR"		=> 0,
42    "POWEROFF_ON_ERROR"		=> 0,
43    "REBOOT_ON_SUCCESS"		=> 1,
44    "POWEROFF_ON_SUCCESS"	=> 0,
45    "BUILD_OPTIONS"		=> "",
46    "BISECT_SLEEP_TIME"		=> 60,		# sleep time between bisects
47    "PATCHCHECK_SLEEP_TIME"	=> 60, 		# sleep time between patch checks
48    "CLEAR_LOG"			=> 0,
49    "BISECT_MANUAL"		=> 0,
50    "BISECT_SKIP"		=> 1,
51    "BISECT_TRIES"		=> 1,
52    "MIN_CONFIG_TYPE"		=> "boot",
53    "SUCCESS_LINE"		=> "login:",
54    "DETECT_TRIPLE_FAULT"	=> 1,
55    "NO_INSTALL"		=> 0,
56    "BOOTED_TIMEOUT"		=> 1,
57    "DIE_ON_FAILURE"		=> 1,
58    "SSH_EXEC"			=> "ssh \$SSH_USER\@\$MACHINE \$SSH_COMMAND",
59    "SCP_TO_TARGET"		=> "scp \$SRC_FILE \$SSH_USER\@\$MACHINE:\$DST_FILE",
60    "SCP_TO_TARGET_INSTALL"	=> "\${SCP_TO_TARGET}",
61    "REBOOT"			=> "ssh \$SSH_USER\@\$MACHINE reboot",
62    "REBOOT_RETURN_CODE"	=> 255,
63    "STOP_AFTER_SUCCESS"	=> 10,
64    "STOP_AFTER_FAILURE"	=> 60,
65    "STOP_TEST_AFTER"		=> 600,
66    "MAX_MONITOR_WAIT"		=> 1800,
67    "GRUB_REBOOT"		=> "grub2-reboot",
68    "GRUB_BLS_GET"		=> "grubby --info=ALL",
69    "SYSLINUX"			=> "extlinux",
70    "SYSLINUX_PATH"		=> "/boot/extlinux",
71    "CONNECT_TIMEOUT"		=> 25,
72
73# required, and we will ask users if they don't have them but we keep the default
74# value something that is common.
75    "REBOOT_TYPE"		=> "grub",
76    "LOCALVERSION"		=> "-test",
77    "SSH_USER"			=> "root",
78    "BUILD_TARGET"	 	=> "arch/x86/boot/bzImage",
79    "TARGET_IMAGE"		=> "/boot/vmlinuz-test",
80
81    "LOG_FILE"			=> undef,
82    "IGNORE_UNUSED"		=> 0,
83);
84
85my $test_log_start = 0;
86
87my $ktest_config = "ktest.conf";
88my $version;
89my $have_version = 0;
90my $machine;
91my $last_machine;
92my $ssh_user;
93my $tmpdir;
94my $builddir;
95my $outputdir;
96my $output_config;
97my $test_type;
98my $build_type;
99my $build_options;
100my $final_post_ktest;
101my $pre_ktest;
102my $post_ktest;
103my $pre_test;
104my $pre_test_die;
105my $post_test;
106my $pre_build;
107my $post_build;
108my $pre_build_die;
109my $post_build_die;
110my $reboot_type;
111my $reboot_script;
112my $power_cycle;
113my $reboot;
114my $reboot_return_code;
115my $reboot_on_error;
116my $switch_to_good;
117my $switch_to_test;
118my $poweroff_on_error;
119my $reboot_on_success;
120my $die_on_failure;
121my $powercycle_after_reboot;
122my $poweroff_after_halt;
123my $max_monitor_wait;
124my $ssh_exec;
125my $scp_to_target;
126my $scp_to_target_install;
127my $power_off;
128my $grub_menu;
129my $last_grub_menu;
130my $grub_file;
131my $grub_number;
132my $grub_reboot;
133my $grub_bls_get;
134my $syslinux;
135my $syslinux_path;
136my $syslinux_label;
137my $target;
138my $make;
139my $pre_install;
140my $post_install;
141my $no_install;
142my $noclean;
143my $minconfig;
144my $start_minconfig;
145my $start_minconfig_defined;
146my $output_minconfig;
147my $minconfig_type;
148my $use_output_minconfig;
149my $warnings_file;
150my $ignore_config;
151my $ignore_errors;
152my $addconfig;
153my $in_bisect = 0;
154my $bisect_bad_commit = "";
155my $reverse_bisect;
156my $bisect_manual;
157my $bisect_skip;
158my $bisect_tries;
159my $config_bisect_good;
160my $bisect_ret_good;
161my $bisect_ret_bad;
162my $bisect_ret_skip;
163my $bisect_ret_abort;
164my $bisect_ret_default;
165my $in_patchcheck = 0;
166my $run_test;
167my $buildlog;
168my $testlog;
169my $dmesg;
170my $monitor_fp;
171my $monitor_pid;
172my $monitor_cnt = 0;
173my $sleep_time;
174my $bisect_sleep_time;
175my $patchcheck_sleep_time;
176my $ignore_warnings;
177my $store_failures;
178my $store_successes;
179my $test_name;
180my $timeout;
181my $connect_timeout;
182my $config_bisect_exec;
183my $booted_timeout;
184my $detect_triplefault;
185my $console;
186my $close_console_signal;
187my $reboot_success_line;
188my $success_line;
189my $stop_after_success;
190my $stop_after_failure;
191my $stop_test_after;
192my $build_target;
193my $target_image;
194my $checkout;
195my $localversion;
196my $iteration = 0;
197my $successes = 0;
198my $stty_orig;
199my $run_command_status = 0;
200
201my $bisect_good;
202my $bisect_bad;
203my $bisect_type;
204my $bisect_start;
205my $bisect_replay;
206my $bisect_files;
207my $bisect_reverse;
208my $bisect_check;
209
210my $config_bisect;
211my $config_bisect_type;
212my $config_bisect_check;
213
214my $patchcheck_type;
215my $patchcheck_start;
216my $patchcheck_cherry;
217my $patchcheck_end;
218
219my $build_time;
220my $install_time;
221my $reboot_time;
222my $test_time;
223
224my $pwd;
225my $dirname = $FindBin::Bin;
226
227my $mailto;
228my $mailer;
229my $mail_path;
230my $mail_max_size;
231my $mail_command;
232my $email_on_error;
233my $email_when_finished;
234my $email_when_started;
235my $email_when_canceled;
236
237my $script_start_time = localtime();
238
239# set when a test is something other that just building or install
240# which would require more options.
241my $buildonly = 1;
242
243# tell build not to worry about warnings, even when WARNINGS_FILE is set
244my $warnings_ok = 0;
245
246# set when creating a new config
247my $newconfig = 0;
248
249my %entered_configs;
250my %config_help;
251my %variable;
252
253# force_config is the list of configs that we force enabled (or disabled)
254# in a .config file. The MIN_CONFIG and ADD_CONFIG configs.
255my %force_config;
256
257# do not force reboots on config problems
258my $no_reboot = 1;
259
260# reboot on success
261my $reboot_success = 0;
262
263my %option_map = (
264    "MAILTO"			=> \$mailto,
265    "MAILER"			=> \$mailer,
266    "MAIL_PATH"			=> \$mail_path,
267    "MAIL_MAX_SIZE"		=> \$mail_max_size,
268    "MAIL_COMMAND"		=> \$mail_command,
269    "EMAIL_ON_ERROR"		=> \$email_on_error,
270    "EMAIL_WHEN_FINISHED"	=> \$email_when_finished,
271    "EMAIL_WHEN_STARTED"	=> \$email_when_started,
272    "EMAIL_WHEN_CANCELED"	=> \$email_when_canceled,
273    "MACHINE"			=> \$machine,
274    "SSH_USER"			=> \$ssh_user,
275    "TMP_DIR"			=> \$tmpdir,
276    "OUTPUT_DIR"		=> \$outputdir,
277    "BUILD_DIR"			=> \$builddir,
278    "TEST_TYPE"			=> \$test_type,
279    "PRE_KTEST"			=> \$pre_ktest,
280    "POST_KTEST"		=> \$post_ktest,
281    "PRE_TEST"			=> \$pre_test,
282    "PRE_TEST_DIE"		=> \$pre_test_die,
283    "POST_TEST"			=> \$post_test,
284    "BUILD_TYPE"		=> \$build_type,
285    "BUILD_OPTIONS"		=> \$build_options,
286    "PRE_BUILD"			=> \$pre_build,
287    "POST_BUILD"		=> \$post_build,
288    "PRE_BUILD_DIE"		=> \$pre_build_die,
289    "POST_BUILD_DIE"		=> \$post_build_die,
290    "POWER_CYCLE"		=> \$power_cycle,
291    "REBOOT"			=> \$reboot,
292    "REBOOT_RETURN_CODE"	=> \$reboot_return_code,
293    "BUILD_NOCLEAN"		=> \$noclean,
294    "MIN_CONFIG"		=> \$minconfig,
295    "OUTPUT_MIN_CONFIG"		=> \$output_minconfig,
296    "START_MIN_CONFIG"		=> \$start_minconfig,
297    "MIN_CONFIG_TYPE"		=> \$minconfig_type,
298    "USE_OUTPUT_MIN_CONFIG"	=> \$use_output_minconfig,
299    "WARNINGS_FILE"		=> \$warnings_file,
300    "IGNORE_CONFIG"		=> \$ignore_config,
301    "TEST"			=> \$run_test,
302    "ADD_CONFIG"		=> \$addconfig,
303    "REBOOT_TYPE"		=> \$reboot_type,
304    "GRUB_MENU"			=> \$grub_menu,
305    "GRUB_FILE"			=> \$grub_file,
306    "GRUB_REBOOT"		=> \$grub_reboot,
307    "GRUB_BLS_GET"		=> \$grub_bls_get,
308    "SYSLINUX"			=> \$syslinux,
309    "SYSLINUX_PATH"		=> \$syslinux_path,
310    "SYSLINUX_LABEL"		=> \$syslinux_label,
311    "PRE_INSTALL"		=> \$pre_install,
312    "POST_INSTALL"		=> \$post_install,
313    "NO_INSTALL"		=> \$no_install,
314    "REBOOT_SCRIPT"		=> \$reboot_script,
315    "REBOOT_ON_ERROR"		=> \$reboot_on_error,
316    "SWITCH_TO_GOOD"		=> \$switch_to_good,
317    "SWITCH_TO_TEST"		=> \$switch_to_test,
318    "POWEROFF_ON_ERROR"		=> \$poweroff_on_error,
319    "REBOOT_ON_SUCCESS"		=> \$reboot_on_success,
320    "DIE_ON_FAILURE"		=> \$die_on_failure,
321    "POWER_OFF"			=> \$power_off,
322    "POWERCYCLE_AFTER_REBOOT"	=> \$powercycle_after_reboot,
323    "POWEROFF_AFTER_HALT"	=> \$poweroff_after_halt,
324    "MAX_MONITOR_WAIT"		=> \$max_monitor_wait,
325    "SLEEP_TIME"		=> \$sleep_time,
326    "BISECT_SLEEP_TIME"		=> \$bisect_sleep_time,
327    "PATCHCHECK_SLEEP_TIME"	=> \$patchcheck_sleep_time,
328    "IGNORE_WARNINGS"		=> \$ignore_warnings,
329    "IGNORE_ERRORS"		=> \$ignore_errors,
330    "BISECT_MANUAL"		=> \$bisect_manual,
331    "BISECT_SKIP"		=> \$bisect_skip,
332    "BISECT_TRIES"		=> \$bisect_tries,
333    "CONFIG_BISECT_GOOD"	=> \$config_bisect_good,
334    "BISECT_RET_GOOD"		=> \$bisect_ret_good,
335    "BISECT_RET_BAD"		=> \$bisect_ret_bad,
336    "BISECT_RET_SKIP"		=> \$bisect_ret_skip,
337    "BISECT_RET_ABORT"		=> \$bisect_ret_abort,
338    "BISECT_RET_DEFAULT"	=> \$bisect_ret_default,
339    "STORE_FAILURES"		=> \$store_failures,
340    "STORE_SUCCESSES"		=> \$store_successes,
341    "TEST_NAME"			=> \$test_name,
342    "TIMEOUT"			=> \$timeout,
343    "CONNECT_TIMEOUT"		=> \$connect_timeout,
344    "CONFIG_BISECT_EXEC"	=> \$config_bisect_exec,
345    "BOOTED_TIMEOUT"		=> \$booted_timeout,
346    "CONSOLE"			=> \$console,
347    "CLOSE_CONSOLE_SIGNAL"	=> \$close_console_signal,
348    "DETECT_TRIPLE_FAULT"	=> \$detect_triplefault,
349    "SUCCESS_LINE"		=> \$success_line,
350    "REBOOT_SUCCESS_LINE"	=> \$reboot_success_line,
351    "STOP_AFTER_SUCCESS"	=> \$stop_after_success,
352    "STOP_AFTER_FAILURE"	=> \$stop_after_failure,
353    "STOP_TEST_AFTER"		=> \$stop_test_after,
354    "BUILD_TARGET"		=> \$build_target,
355    "SSH_EXEC"			=> \$ssh_exec,
356    "SCP_TO_TARGET"		=> \$scp_to_target,
357    "SCP_TO_TARGET_INSTALL"	=> \$scp_to_target_install,
358    "CHECKOUT"			=> \$checkout,
359    "TARGET_IMAGE"		=> \$target_image,
360    "LOCALVERSION"		=> \$localversion,
361
362    "BISECT_GOOD"		=> \$bisect_good,
363    "BISECT_BAD"		=> \$bisect_bad,
364    "BISECT_TYPE"		=> \$bisect_type,
365    "BISECT_START"		=> \$bisect_start,
366    "BISECT_REPLAY"		=> \$bisect_replay,
367    "BISECT_FILES"		=> \$bisect_files,
368    "BISECT_REVERSE"		=> \$bisect_reverse,
369    "BISECT_CHECK"		=> \$bisect_check,
370
371    "CONFIG_BISECT"		=> \$config_bisect,
372    "CONFIG_BISECT_TYPE"	=> \$config_bisect_type,
373    "CONFIG_BISECT_CHECK"	=> \$config_bisect_check,
374
375    "PATCHCHECK_TYPE"		=> \$patchcheck_type,
376    "PATCHCHECK_START"		=> \$patchcheck_start,
377    "PATCHCHECK_CHERRY"		=> \$patchcheck_cherry,
378    "PATCHCHECK_END"		=> \$patchcheck_end,
379);
380
381# Options may be used by other options, record them.
382my %used_options;
383
384# default variables that can be used
385chomp ($variable{"PWD"} = `pwd`);
386$pwd = $variable{"PWD"};
387
388$config_help{"MACHINE"} = << "EOF"
389 The machine hostname that you will test.
390 For build only tests, it is still needed to differentiate log files.
391EOF
392    ;
393$config_help{"SSH_USER"} = << "EOF"
394 The box is expected to have ssh on normal bootup, provide the user
395  (most likely root, since you need privileged operations)
396EOF
397    ;
398$config_help{"BUILD_DIR"} = << "EOF"
399 The directory that contains the Linux source code (full path).
400 You can use \${PWD} that will be the path where ktest.pl is run, or use
401 \${THIS_DIR} which is assigned \${PWD} but may be changed later.
402EOF
403    ;
404$config_help{"OUTPUT_DIR"} = << "EOF"
405 The directory that the objects will be built (full path).
406 (can not be same as BUILD_DIR)
407 You can use \${PWD} that will be the path where ktest.pl is run, or use
408 \${THIS_DIR} which is assigned \${PWD} but may be changed later.
409EOF
410    ;
411$config_help{"BUILD_TARGET"} = << "EOF"
412 The location of the compiled file to copy to the target.
413 (relative to OUTPUT_DIR)
414EOF
415    ;
416$config_help{"BUILD_OPTIONS"} = << "EOF"
417 Options to add to \"make\" when building.
418 i.e.  -j20
419EOF
420    ;
421$config_help{"TARGET_IMAGE"} = << "EOF"
422 The place to put your image on the test machine.
423EOF
424    ;
425$config_help{"POWER_CYCLE"} = << "EOF"
426 A script or command to reboot the box.
427
428 Here is a digital loggers power switch example
429 POWER_CYCLE = wget --no-proxy -O /dev/null -q  --auth-no-challenge 'http://admin:admin\@power/outlet?5=CCL'
430
431 Here is an example to reboot a virtual box on the current host
432 with the name "Guest".
433 POWER_CYCLE = virsh destroy Guest; sleep 5; virsh start Guest
434EOF
435    ;
436$config_help{"CONSOLE"} = << "EOF"
437 The script or command that reads the console
438
439  If you use ttywatch server, something like the following would work.
440CONSOLE = nc -d localhost 3001
441
442 For a virtual machine with guest name "Guest".
443CONSOLE =  virsh console Guest
444EOF
445    ;
446$config_help{"LOCALVERSION"} = << "EOF"
447 Required version ending to differentiate the test
448 from other linux builds on the system.
449EOF
450    ;
451$config_help{"REBOOT_TYPE"} = << "EOF"
452 Way to reboot the box to the test kernel.
453 Only valid options so far are "grub", "grub2", "grub2bls", "syslinux", and "script".
454
455 If you specify grub, it will assume grub version 1
456 and will search in /boot/grub/menu.lst for the title \$GRUB_MENU
457 and select that target to reboot to the kernel. If this is not
458 your setup, then specify "script" and have a command or script
459 specified in REBOOT_SCRIPT to boot to the target.
460
461 The entry in /boot/grub/menu.lst must be entered in manually.
462 The test will not modify that file.
463
464 If you specify grub2, then you also need to specify both \$GRUB_MENU
465 and \$GRUB_FILE.
466
467 If you specify grub2bls, then you also need to specify \$GRUB_MENU.
468
469 If you specify syslinux, then you may use SYSLINUX to define the syslinux
470 command (defaults to extlinux), and SYSLINUX_PATH to specify the path to
471 the syslinux install (defaults to /boot/extlinux). But you have to specify
472 SYSLINUX_LABEL to define the label to boot to for the test kernel.
473EOF
474    ;
475$config_help{"GRUB_MENU"} = << "EOF"
476 The grub title name for the test kernel to boot
477 (Only mandatory if REBOOT_TYPE = grub or grub2)
478
479 Note, ktest.pl will not update the grub menu.lst, you need to
480 manually add an option for the test. ktest.pl will search
481 the grub menu.lst for this option to find what kernel to
482 reboot into.
483
484 For example, if in the /boot/grub/menu.lst the test kernel title has:
485 title Test Kernel
486 kernel vmlinuz-test
487 GRUB_MENU = Test Kernel
488
489 For grub2, a search of \$GRUB_FILE is performed for the lines
490 that begin with "menuentry". It will not detect submenus. The
491 menu must be a non-nested menu. Add the quotes used in the menu
492 to guarantee your selection, as the first menuentry with the content
493 of \$GRUB_MENU that is found will be used.
494
495 For grub2bls, \$GRUB_MENU is searched on the result of \$GRUB_BLS_GET
496 command for the lines that begin with "title".
497EOF
498    ;
499$config_help{"GRUB_FILE"} = << "EOF"
500 If grub2 is used, the full path for the grub.cfg file is placed
501 here. Use something like /boot/grub2/grub.cfg to search.
502EOF
503    ;
504$config_help{"SYSLINUX_LABEL"} = << "EOF"
505 If syslinux is used, the label that boots the target kernel must
506 be specified with SYSLINUX_LABEL.
507EOF
508    ;
509$config_help{"REBOOT_SCRIPT"} = << "EOF"
510 A script to reboot the target into the test kernel
511 (Only mandatory if REBOOT_TYPE = script)
512EOF
513    ;
514
515# used with process_expression()
516my $d = 0;
517
518# defined before get_test_name()
519my $in_die = 0;
520
521# defined before process_warning_line()
522my $check_build_re = ".*:.*(warning|error|Error):.*";
523my $utf8_quote = "\\x{e2}\\x{80}(\\x{98}|\\x{99})";
524
525# defined before child_finished()
526my $child_done;
527
528# config_ignore holds the configs that were set (or unset) for
529# a good config and we will ignore these configs for the rest
530# of a config bisect. These configs stay as they were.
531my %config_ignore;
532
533# config_set holds what all configs were set as.
534my %config_set;
535
536# config_off holds the set of configs that the bad config had disabled.
537# We need to record them and set them in the .config when running
538# olddefconfig, because olddefconfig keeps the defaults.
539my %config_off;
540
541# config_off_tmp holds a set of configs to turn off for now
542my @config_off_tmp;
543
544# config_list is the set of configs that are being tested
545my %config_list;
546my %null_config;
547
548my %dependency;
549
550# found above run_config_bisect()
551my $pass = 1;
552
553# found above add_dep()
554
555my %depends;
556my %depcount;
557my $iflevel = 0;
558my @ifdeps;
559
560# prevent recursion
561my %read_kconfigs;
562
563# found above test_this_config()
564my %min_configs;
565my %keep_configs;
566my %save_configs;
567my %processed_configs;
568my %nochange_config;
569
570#
571# These are first defined here, main function later on
572#
573sub run_command;
574sub start_monitor;
575sub end_monitor;
576sub wait_for_monitor;
577
578sub _logit {
579    if (defined($opt{"LOG_FILE"})) {
580	print LOG @_;
581    }
582}
583
584sub logit {
585    if (defined($opt{"LOG_FILE"})) {
586	_logit @_;
587    } else {
588	print @_;
589    }
590}
591
592sub doprint {
593    print @_;
594    _logit @_;
595}
596
597sub read_prompt {
598    my ($cancel, $prompt) = @_;
599
600    my $ans;
601
602    for (;;) {
603        if ($cancel) {
604	    print "$prompt [y/n/C] ";
605	} else {
606	    print "$prompt [Y/n] ";
607	}
608	$ans = <STDIN>;
609	chomp $ans;
610	if ($ans =~ /^\s*$/) {
611	    if ($cancel) {
612		$ans = "c";
613	    } else {
614		$ans = "y";
615	    }
616	}
617	last if ($ans =~ /^y$/i || $ans =~ /^n$/i);
618	if ($cancel) {
619	    last if ($ans =~ /^c$/i);
620	    print "Please answer either 'y', 'n' or 'c'.\n";
621	} else {
622	    print "Please answer either 'y' or 'n'.\n";
623	}
624    }
625    if ($ans =~ /^c/i) {
626	exit;
627    }
628    if ($ans !~ /^y$/i) {
629	return 0;
630    }
631    return 1;
632}
633
634sub read_yn {
635    my ($prompt) = @_;
636
637    return read_prompt 0, $prompt;
638}
639
640sub read_ync {
641    my ($prompt) = @_;
642
643    return read_prompt 1, $prompt;
644}
645
646sub get_mandatory_config {
647    my ($config) = @_;
648    my $ans;
649
650    return if (defined($opt{$config}));
651
652    if (defined($config_help{$config})) {
653	print "\n";
654	print $config_help{$config};
655    }
656
657    for (;;) {
658	print "$config = ";
659	if (defined($default{$config}) && length($default{$config})) {
660	    print "\[$default{$config}\] ";
661	}
662	$ans = <STDIN>;
663	$ans =~ s/^\s*(.*\S)\s*$/$1/;
664	if ($ans =~ /^\s*$/) {
665	    if ($default{$config}) {
666		$ans = $default{$config};
667	    } else {
668		print "Your answer can not be blank\n";
669		next;
670	    }
671	}
672	$entered_configs{$config} = ${ans};
673	last;
674    }
675}
676
677sub show_time {
678    my ($time) = @_;
679
680    my $hours = 0;
681    my $minutes = 0;
682
683    if ($time > 3600) {
684	$hours = int($time / 3600);
685	$time -= $hours * 3600;
686    }
687    if ($time > 60) {
688	$minutes = int($time / 60);
689	$time -= $minutes * 60;
690    }
691
692    if ($hours > 0) {
693	doprint "$hours hour";
694	doprint "s" if ($hours > 1);
695	doprint " ";
696    }
697
698    if ($minutes > 0) {
699	doprint "$minutes minute";
700	doprint "s" if ($minutes > 1);
701	doprint " ";
702    }
703
704    doprint "$time second";
705    doprint "s" if ($time != 1);
706}
707
708sub print_times {
709    doprint "\n";
710    if ($build_time) {
711	doprint "Build time:   ";
712	show_time($build_time);
713	doprint "\n";
714    }
715    if ($install_time) {
716	doprint "Install time: ";
717	show_time($install_time);
718	doprint "\n";
719    }
720    if ($reboot_time) {
721	doprint "Reboot time:  ";
722	show_time($reboot_time);
723	doprint "\n";
724    }
725    if ($test_time) {
726	doprint "Test time:    ";
727	show_time($test_time);
728	doprint "\n";
729    }
730    # reset for iterations like bisect
731    $build_time = 0;
732    $install_time = 0;
733    $reboot_time = 0;
734    $test_time = 0;
735}
736
737sub get_mandatory_configs {
738    get_mandatory_config("MACHINE");
739    get_mandatory_config("BUILD_DIR");
740    get_mandatory_config("OUTPUT_DIR");
741
742    if ($newconfig) {
743	get_mandatory_config("BUILD_OPTIONS");
744    }
745
746    # options required for other than just building a kernel
747    if (!$buildonly) {
748	get_mandatory_config("POWER_CYCLE");
749	get_mandatory_config("CONSOLE");
750    }
751
752    # options required for install and more
753    if ($buildonly != 1) {
754	get_mandatory_config("SSH_USER");
755	get_mandatory_config("BUILD_TARGET");
756	get_mandatory_config("TARGET_IMAGE");
757    }
758
759    get_mandatory_config("LOCALVERSION");
760
761    return if ($buildonly);
762
763    my $rtype = $opt{"REBOOT_TYPE"};
764
765    if (!defined($rtype)) {
766	if (!defined($opt{"GRUB_MENU"})) {
767	    get_mandatory_config("REBOOT_TYPE");
768	    $rtype = $entered_configs{"REBOOT_TYPE"};
769	} else {
770	    $rtype = "grub";
771	}
772    }
773
774    if (($rtype eq "grub") or ($rtype eq "grub2bls")) {
775	get_mandatory_config("GRUB_MENU");
776    }
777
778    if ($rtype eq "grub2") {
779	get_mandatory_config("GRUB_MENU");
780	get_mandatory_config("GRUB_FILE");
781    }
782
783    if ($rtype eq "syslinux") {
784	get_mandatory_config("SYSLINUX_LABEL");
785    }
786}
787
788sub process_variables {
789    my ($value, $remove_undef) = @_;
790    my $retval = "";
791
792    # We want to check for '\', and it is just easier
793    # to check the previous characet of '$' and not need
794    # to worry if '$' is the first character. By adding
795    # a space to $value, we can just check [^\\]\$ and
796    # it will still work.
797    $value = " $value";
798
799    while ($value =~ /(.*?[^\\])\$\{(.*?)\}(.*)/) {
800	my $begin = $1;
801	my $var = $2;
802	my $end = $3;
803	# append beginning of value to retval
804	$retval = "$retval$begin";
805	if (defined($variable{$var})) {
806	    $retval = "$retval$variable{$var}";
807	} elsif (defined($remove_undef) && $remove_undef) {
808	    # for if statements, any variable that is not defined,
809	    # we simple convert to 0
810	    $retval = "${retval}0";
811	} else {
812	    # put back the origin piece.
813	    $retval = "$retval\$\{$var\}";
814	    # This could be an option that is used later, save
815	    # it so we don't warn if this option is not one of
816	    # ktests options.
817	    $used_options{$var} = 1;
818	}
819	$value = $end;
820    }
821    $retval = "$retval$value";
822
823    # remove the space added in the beginning
824    $retval =~ s/ //;
825
826    return "$retval";
827}
828
829sub set_value {
830    my ($lvalue, $rvalue, $override, $overrides, $name) = @_;
831
832    my $prvalue = process_variables($rvalue);
833
834    if ($lvalue =~ /^(TEST|BISECT|CONFIG_BISECT)_TYPE(\[.*\])?$/ &&
835	$prvalue !~ /^(config_|)bisect$/ &&
836	$prvalue !~ /^build$/ &&
837	$buildonly) {
838
839	# Note if a test is something other than build, then we
840	# will need other mandatory options.
841	if ($prvalue ne "install") {
842	    $buildonly = 0;
843	} else {
844	    # install still limits some mandatory options.
845	    $buildonly = 2;
846	}
847    }
848
849    if (defined($opt{$lvalue})) {
850	if (!$override || defined(${$overrides}{$lvalue})) {
851	    my $extra = "";
852	    if ($override) {
853		$extra = "In the same override section!\n";
854	    }
855	    die "$name: $.: Option $lvalue defined more than once!\n$extra";
856	}
857	${$overrides}{$lvalue} = $prvalue;
858    }
859
860    $opt{$lvalue} = $prvalue;
861}
862
863sub set_eval {
864    my ($lvalue, $rvalue, $name) = @_;
865
866    my $prvalue = process_variables($rvalue);
867    my $arr;
868
869    if (defined($evals{$lvalue})) {
870	$arr = $evals{$lvalue};
871    } else {
872	$arr = [];
873	$evals{$lvalue} = $arr;
874    }
875
876    push @{$arr}, $rvalue;
877}
878
879sub set_variable {
880    my ($lvalue, $rvalue) = @_;
881
882    if ($rvalue =~ /^\s*$/) {
883	delete $variable{$lvalue};
884    } else {
885	$rvalue = process_variables($rvalue);
886	$variable{$lvalue} = $rvalue;
887    }
888}
889
890sub process_compare {
891    my ($lval, $cmp, $rval) = @_;
892
893    # remove whitespace
894
895    $lval =~ s/^\s*//;
896    $lval =~ s/\s*$//;
897
898    $rval =~ s/^\s*//;
899    $rval =~ s/\s*$//;
900
901    if ($cmp eq "==") {
902	return $lval eq $rval;
903    } elsif ($cmp eq "!=") {
904	return $lval ne $rval;
905    } elsif ($cmp eq "=~") {
906	return $lval =~ m/$rval/;
907    } elsif ($cmp eq "!~") {
908	return $lval !~ m/$rval/;
909    }
910
911    my $statement = "$lval $cmp $rval";
912    my $ret = eval $statement;
913
914    # $@ stores error of eval
915    if ($@) {
916	return -1;
917    }
918
919    return $ret;
920}
921
922sub value_defined {
923    my ($val) = @_;
924
925    return defined($variable{$2}) ||
926	defined($opt{$2});
927}
928
929sub process_expression {
930    my ($name, $val) = @_;
931
932    my $c = $d++;
933
934    while ($val =~ s/\(([^\(]*?)\)/\&\&\&\&VAL\&\&\&\&/) {
935	my $express = $1;
936
937	if (process_expression($name, $express)) {
938	    $val =~ s/\&\&\&\&VAL\&\&\&\&/ 1 /;
939	} else {
940	    $val =~ s/\&\&\&\&VAL\&\&\&\&/ 0 /;
941	}
942    }
943
944    $d--;
945    my $OR = "\\|\\|";
946    my $AND = "\\&\\&";
947
948    while ($val =~ s/^(.*?)($OR|$AND)//) {
949	my $express = $1;
950	my $op = $2;
951
952	if (process_expression($name, $express)) {
953	    if ($op eq "||") {
954		return 1;
955	    }
956	} else {
957	    if ($op eq "&&") {
958		return 0;
959	    }
960	}
961    }
962
963    if ($val =~ /(.*)(==|\!=|>=|<=|>|<|=~|\!~)(.*)/) {
964	my $ret = process_compare($1, $2, $3);
965	if ($ret < 0) {
966	    die "$name: $.: Unable to process comparison\n";
967	}
968	return $ret;
969    }
970
971    if ($val =~ /^\s*(NOT\s*)?DEFINED\s+(\S+)\s*$/) {
972	if (defined $1) {
973	    return !value_defined($2);
974	} else {
975	    return value_defined($2);
976	}
977    }
978
979    if ($val =~ s/^\s*NOT\s+(.*)//) {
980	my $express = $1;
981	my $ret = process_expression($name, $express);
982	return !$ret;
983    }
984
985    if ($val =~ /^\s*0\s*$/) {
986	return 0;
987    } elsif ($val =~ /^\s*\d+\s*$/) {
988	return 1;
989    }
990
991    die ("$name: $.: Undefined content $val in if statement\n");
992}
993
994sub process_if {
995    my ($name, $value) = @_;
996
997    # Convert variables and replace undefined ones with 0
998    my $val = process_variables($value, 1);
999    my $ret = process_expression $name, $val;
1000
1001    return $ret;
1002}
1003
1004sub __read_config {
1005    my ($config, $current_test_num) = @_;
1006
1007    my $in;
1008    open($in, $config) || die "can't read file $config";
1009
1010    my $name = $config;
1011    $name =~ s,.*/(.*),$1,;
1012
1013    my $test_num = $$current_test_num;
1014    my $default = 1;
1015    my $repeat = 1;
1016    my $num_tests_set = 0;
1017    my $skip = 0;
1018    my $rest;
1019    my $line;
1020    my $test_case = 0;
1021    my $if = 0;
1022    my $if_set = 0;
1023    my $override = 0;
1024
1025    my %overrides;
1026
1027    while (<$in>) {
1028
1029	# ignore blank lines and comments
1030	next if (/^\s*$/ || /\s*\#/);
1031
1032	if (/^\s*(TEST_START|DEFAULTS)\b(.*)/) {
1033
1034	    my $type = $1;
1035	    $rest = $2;
1036	    $line = $2;
1037
1038	    my $old_test_num;
1039	    my $old_repeat;
1040	    $override = 0;
1041
1042	    if ($type eq "TEST_START") {
1043		if ($num_tests_set) {
1044		    die "$name: $.: Can not specify both NUM_TESTS and TEST_START\n";
1045		}
1046
1047		$old_test_num = $test_num;
1048		$old_repeat = $repeat;
1049
1050		$test_num += $repeat;
1051		$default = 0;
1052		$repeat = 1;
1053	    } else {
1054		$default = 1;
1055	    }
1056
1057	    # If SKIP is anywhere in the line, the command will be skipped
1058	    if ($rest =~ s/\s+SKIP\b//) {
1059		$skip = 1;
1060	    } else {
1061		$test_case = 1;
1062		$skip = 0;
1063	    }
1064
1065	    if ($rest =~ s/\sELSE\b//) {
1066		if (!$if) {
1067		    die "$name: $.: ELSE found with out matching IF section\n$_";
1068		}
1069		$if = 0;
1070
1071		if ($if_set) {
1072		    $skip = 1;
1073		} else {
1074		    $skip = 0;
1075		}
1076	    }
1077
1078	    if ($rest =~ s/\sIF\s+(.*)//) {
1079		if (process_if($name, $1)) {
1080		    $if_set = 1;
1081		} else {
1082		    $skip = 1;
1083		}
1084		$if = 1;
1085	    } else {
1086		$if = 0;
1087		$if_set = 0;
1088	    }
1089
1090	    if (!$skip) {
1091		if ($type eq "TEST_START") {
1092		    if ($rest =~ s/\s+ITERATE\s+(\d+)//) {
1093			$repeat = $1;
1094			$repeat_tests{"$test_num"} = $repeat;
1095		    }
1096		} elsif ($rest =~ s/\sOVERRIDE\b//) {
1097		    # DEFAULT only
1098		    $override = 1;
1099		    # Clear previous overrides
1100		    %overrides = ();
1101		}
1102	    }
1103
1104	    if (!$skip && $rest !~ /^\s*$/) {
1105		die "$name: $.: Garbage found after $type\n$_";
1106	    }
1107
1108	    if ($skip && $type eq "TEST_START") {
1109		$test_num = $old_test_num;
1110		$repeat = $old_repeat;
1111	    }
1112	} elsif (/^\s*ELSE\b(.*)$/) {
1113	    if (!$if) {
1114		die "$name: $.: ELSE found with out matching IF section\n$_";
1115	    }
1116	    $rest = $1;
1117	    if ($if_set) {
1118		$skip = 1;
1119		$rest = "";
1120	    } else {
1121		$skip = 0;
1122
1123		if ($rest =~ /\sIF\s+(.*)/) {
1124		    # May be a ELSE IF section.
1125		    if (process_if($name, $1)) {
1126			$if_set = 1;
1127		    } else {
1128			$skip = 1;
1129		    }
1130		    $rest = "";
1131		} else {
1132		    $if = 0;
1133		}
1134	    }
1135
1136	    if ($rest !~ /^\s*$/) {
1137		die "$name: $.: Garbage found after DEFAULTS\n$_";
1138	    }
1139
1140	} elsif (/^\s*INCLUDE\s+(\S+)/) {
1141
1142	    next if ($skip);
1143
1144	    if (!$default) {
1145		die "$name: $.: INCLUDE can only be done in default sections\n$_";
1146	    }
1147
1148	    my $file = process_variables($1);
1149
1150	    if ($file !~ m,^/,) {
1151		# check the path of the config file first
1152		if ($config =~ m,(.*)/,) {
1153		    if (-f "$1/$file") {
1154			$file = "$1/$file";
1155		    }
1156		}
1157	    }
1158
1159	    if ( ! -r $file ) {
1160		die "$name: $.: Can't read file $file\n$_";
1161	    }
1162
1163	    if (__read_config($file, \$test_num)) {
1164		$test_case = 1;
1165	    }
1166
1167	} elsif (/^\s*([A-Z_\[\]\d]+)\s*=~\s*(.*?)\s*$/) {
1168
1169	    next if ($skip);
1170
1171	    my $lvalue = $1;
1172	    my $rvalue = $2;
1173
1174	    if ($default || $lvalue =~ /\[\d+\]$/) {
1175		set_eval($lvalue, $rvalue, $name);
1176	    } else {
1177		my $val = "$lvalue\[$test_num\]";
1178		set_eval($val, $rvalue, $name);
1179	    }
1180
1181	} elsif (/^\s*([A-Z_\[\]\d]+)\s*=\s*(.*?)\s*$/) {
1182
1183	    next if ($skip);
1184
1185	    my $lvalue = $1;
1186	    my $rvalue = $2;
1187
1188	    if (!$default &&
1189		($lvalue eq "NUM_TESTS" ||
1190		 $lvalue eq "LOG_FILE" ||
1191		 $lvalue eq "CLEAR_LOG")) {
1192		die "$name: $.: $lvalue must be set in DEFAULTS section\n";
1193	    }
1194
1195	    if ($lvalue eq "NUM_TESTS") {
1196		if ($test_num) {
1197		    die "$name: $.: Can not specify both NUM_TESTS and TEST_START\n";
1198		}
1199		if (!$default) {
1200		    die "$name: $.: NUM_TESTS must be set in default section\n";
1201		}
1202		$num_tests_set = 1;
1203	    }
1204
1205	    if ($default || $lvalue =~ /\[\d+\]$/) {
1206		set_value($lvalue, $rvalue, $override, \%overrides, $name);
1207	    } else {
1208		my $val = "$lvalue\[$test_num\]";
1209		set_value($val, $rvalue, $override, \%overrides, $name);
1210
1211		if ($repeat > 1) {
1212		    $repeats{$val} = $repeat;
1213		}
1214	    }
1215	} elsif (/^\s*([A-Z_\[\]\d]+)\s*:=\s*(.*?)\s*$/) {
1216	    next if ($skip);
1217
1218	    my $lvalue = $1;
1219	    my $rvalue = $2;
1220
1221	    # process config variables.
1222	    # Config variables are only active while reading the
1223	    # config and can be defined anywhere. They also ignore
1224	    # TEST_START and DEFAULTS, but are skipped if they are in
1225	    # on of these sections that have SKIP defined.
1226	    # The save variable can be
1227	    # defined multiple times and the new one simply overrides
1228	    # the previous one.
1229	    set_variable($lvalue, $rvalue);
1230
1231	} else {
1232	    die "$name: $.: Garbage found in config\n$_";
1233	}
1234    }
1235
1236    if ($test_num) {
1237	$test_num += $repeat - 1;
1238	$opt{"NUM_TESTS"} = $test_num;
1239    }
1240
1241    close($in);
1242
1243    $$current_test_num = $test_num;
1244
1245    return $test_case;
1246}
1247
1248sub get_test_case {
1249    print "What test case would you like to run?\n";
1250    print " (build, install or boot)\n";
1251    print " Other tests are available but require editing ktest.conf\n";
1252    print " (see tools/testing/ktest/sample.conf)\n";
1253    my $ans = <STDIN>;
1254    chomp $ans;
1255    $default{"TEST_TYPE"} = $ans;
1256}
1257
1258sub read_config {
1259    my ($config) = @_;
1260
1261    my $test_case;
1262    my $test_num = 0;
1263
1264    $test_case = __read_config $config, \$test_num;
1265
1266    # make sure we have all mandatory configs
1267    get_mandatory_configs;
1268
1269    # was a test specified?
1270    if (!$test_case) {
1271	print "No test case specified.\n";
1272	get_test_case;
1273    }
1274
1275    # set any defaults
1276
1277    foreach my $default (keys %default) {
1278	if (!defined($opt{$default})) {
1279	    $opt{$default} = $default{$default};
1280	}
1281    }
1282
1283    if ($opt{"IGNORE_UNUSED"} == 1) {
1284	return;
1285    }
1286
1287    my %not_used;
1288
1289    # check if there are any stragglers (typos?)
1290    foreach my $option (keys %opt) {
1291	my $op = $option;
1292	# remove per test labels.
1293	$op =~ s/\[.*\]//;
1294	if (!exists($option_map{$op}) &&
1295	    !exists($default{$op}) &&
1296	    !exists($used_options{$op})) {
1297	    $not_used{$op} = 1;
1298	}
1299    }
1300
1301    if (%not_used) {
1302	my $s = "s are";
1303	$s = " is" if (keys %not_used == 1);
1304	print "The following option$s not used; could be a typo:\n";
1305	foreach my $option (keys %not_used) {
1306	    print "$option\n";
1307	}
1308	print "Set IGNORE_UNUSED = 1 to have ktest ignore unused variables\n";
1309	if (!read_yn "Do you want to continue?") {
1310	    exit -1;
1311	}
1312    }
1313}
1314
1315sub __eval_option {
1316    my ($name, $option, $i) = @_;
1317
1318    # Add space to evaluate the character before $
1319    $option = " $option";
1320    my $retval = "";
1321    my $repeated = 0;
1322    my $parent = 0;
1323
1324    foreach my $test (keys %repeat_tests) {
1325	if ($i >= $test &&
1326	    $i < $test + $repeat_tests{$test}) {
1327
1328	    $repeated = 1;
1329	    $parent = $test;
1330	    last;
1331	}
1332    }
1333
1334    while ($option =~ /(.*?[^\\])\$\{(.*?)\}(.*)/) {
1335	my $start = $1;
1336	my $var = $2;
1337	my $end = $3;
1338
1339	# Append beginning of line
1340	$retval = "$retval$start";
1341
1342	# If the iteration option OPT[$i] exists, then use that.
1343	# otherwise see if the default OPT (without [$i]) exists.
1344
1345	my $o = "$var\[$i\]";
1346	my $parento = "$var\[$parent\]";
1347
1348	# If a variable contains itself, use the default var
1349	if (($var eq $name) && defined($opt{$var})) {
1350	    $o = $opt{$var};
1351	    $retval = "$retval$o";
1352	} elsif (defined($opt{$o})) {
1353	    $o = $opt{$o};
1354	    $retval = "$retval$o";
1355	} elsif ($repeated && defined($opt{$parento})) {
1356	    $o = $opt{$parento};
1357	    $retval = "$retval$o";
1358	} elsif (defined($opt{$var})) {
1359	    $o = $opt{$var};
1360	    $retval = "$retval$o";
1361	} elsif ($var eq "KERNEL_VERSION" && defined($make)) {
1362	    # special option KERNEL_VERSION uses kernel version
1363	    get_version();
1364	    $retval = "$retval$version";
1365	} else {
1366	    $retval = "$retval\$\{$var\}";
1367	}
1368
1369	$option = $end;
1370    }
1371
1372    $retval = "$retval$option";
1373
1374    $retval =~ s/^ //;
1375
1376    return $retval;
1377}
1378
1379sub process_evals {
1380    my ($name, $option, $i) = @_;
1381
1382    my $option_name = "$name\[$i\]";
1383    my $ev;
1384
1385    my $old_option = $option;
1386
1387    if (defined($evals{$option_name})) {
1388	$ev = $evals{$option_name};
1389    } elsif (defined($evals{$name})) {
1390	$ev = $evals{$name};
1391    } else {
1392	return $option;
1393    }
1394
1395    for my $e (@{$ev}) {
1396	eval "\$option =~ $e";
1397    }
1398
1399    if ($option ne $old_option) {
1400	doprint("$name changed from '$old_option' to '$option'\n");
1401    }
1402
1403    return $option;
1404}
1405
1406sub eval_option {
1407    my ($name, $option, $i) = @_;
1408
1409    my $prev = "";
1410
1411    # Since an option can evaluate to another option,
1412    # keep iterating until we do not evaluate any more
1413    # options.
1414    my $r = 0;
1415    while ($prev ne $option) {
1416	# Check for recursive evaluations.
1417	# 100 deep should be more than enough.
1418	if ($r++ > 100) {
1419	    die "Over 100 evaluations occurred with $option\n" .
1420		"Check for recursive variables\n";
1421	}
1422	$prev = $option;
1423	$option = __eval_option($name, $option, $i);
1424    }
1425
1426    $option = process_evals($name, $option, $i);
1427
1428    return $option;
1429}
1430
1431sub reboot {
1432    my ($time) = @_;
1433    my $powercycle = 0;
1434
1435    # test if the machine can be connected to within a few seconds
1436    my $stat = run_ssh("echo check machine status", $connect_timeout);
1437    if (!$stat) {
1438	doprint("power cycle\n");
1439	$powercycle = 1;
1440    }
1441
1442    if ($powercycle) {
1443	run_command "$power_cycle";
1444
1445	start_monitor;
1446	# flush out current monitor
1447	# May contain the reboot success line
1448	wait_for_monitor 1;
1449
1450    } else {
1451	# Make sure everything has been written to disk
1452	run_ssh("sync", 10);
1453
1454	if (defined($time)) {
1455	    start_monitor;
1456	    # flush out current monitor
1457	    # May contain the reboot success line
1458	    wait_for_monitor 1;
1459	}
1460
1461	# try to reboot normally
1462	if (run_command $reboot) {
1463	    if (defined($powercycle_after_reboot)) {
1464		sleep $powercycle_after_reboot;
1465		run_command "$power_cycle";
1466	    }
1467	} else {
1468	    # nope? power cycle it.
1469	    run_command "$power_cycle";
1470	}
1471    }
1472
1473    if (defined($time)) {
1474
1475	# We only want to get to the new kernel, don't fail
1476	# if we stumble over a call trace.
1477	my $save_ignore_errors = $ignore_errors;
1478	$ignore_errors = 1;
1479
1480	# Look for the good kernel to boot
1481	if (wait_for_monitor($time, "Linux version")) {
1482	    # reboot got stuck?
1483	    doprint "Reboot did not finish. Forcing power cycle\n";
1484	    run_command "$power_cycle";
1485	}
1486
1487	$ignore_errors = $save_ignore_errors;
1488
1489	# Still need to wait for the reboot to finish
1490	wait_for_monitor($time, $reboot_success_line);
1491
1492	end_monitor;
1493    }
1494}
1495
1496sub reboot_to_good {
1497    my ($time) = @_;
1498
1499    if (defined($switch_to_good)) {
1500	run_command $switch_to_good;
1501    }
1502
1503    reboot $time;
1504}
1505
1506sub do_not_reboot {
1507    my $i = $iteration;
1508
1509    return $test_type eq "build" || $no_reboot ||
1510	($test_type eq "patchcheck" && $opt{"PATCHCHECK_TYPE[$i]"} eq "build") ||
1511	($test_type eq "bisect" && $opt{"BISECT_TYPE[$i]"} eq "build") ||
1512	($test_type eq "config_bisect" && $opt{"CONFIG_BISECT_TYPE[$i]"} eq "build");
1513}
1514
1515sub get_test_name() {
1516    my $name;
1517
1518    if (defined($test_name)) {
1519	$name = "$test_name:$test_type";
1520    } else {
1521	$name = $test_type;
1522    }
1523    return $name;
1524}
1525
1526sub dodie {
1527    # avoid recursion
1528    return if ($in_die);
1529    $in_die = 1;
1530
1531    my $i = $iteration;
1532
1533    doprint "CRITICAL FAILURE... [TEST $i] ", @_, "\n";
1534
1535    if ($reboot_on_error && !do_not_reboot) {
1536	doprint "REBOOTING\n";
1537	reboot_to_good;
1538    } elsif ($poweroff_on_error && defined($power_off)) {
1539	doprint "POWERING OFF\n";
1540	`$power_off`;
1541    }
1542
1543    if (defined($opt{"LOG_FILE"})) {
1544	print " See $opt{LOG_FILE} for more info.\n";
1545    }
1546
1547    if ($email_on_error) {
1548	my $name = get_test_name;
1549	my $log_file;
1550
1551	if (defined($opt{"LOG_FILE"})) {
1552	    my $whence = 2; # End of file
1553	    my $log_size = tell LOG;
1554	    my $size = $log_size - $test_log_start;
1555
1556	    if (defined($mail_max_size)) {
1557		if ($size > $mail_max_size) {
1558		    $size = $mail_max_size;
1559		}
1560	    }
1561	    my $pos = - $size;
1562	    $log_file = "$tmpdir/log";
1563	    open (L, "$opt{LOG_FILE}") or die "Can't open $opt{LOG_FILE} to read)";
1564	    open (O, "> $tmpdir/log") or die "Can't open $tmpdir/log\n";
1565	    seek(L, $pos, $whence);
1566	    while (<L>) {
1567		print O;
1568	    }
1569	    close O;
1570	    close L;
1571	}
1572
1573	send_email("KTEST: critical failure for test $i [$name]",
1574		"Your test started at $script_start_time has failed with:\n@_\n", $log_file);
1575    }
1576
1577    if ($monitor_cnt) {
1578	# restore terminal settings
1579	system("stty $stty_orig");
1580    }
1581
1582    if (defined($post_test)) {
1583	run_command $post_test;
1584    }
1585
1586    die @_, "\n";
1587}
1588
1589sub create_pty {
1590    my ($ptm, $pts) = @_;
1591    my $tmp;
1592    my $TIOCSPTLCK = 0x40045431;
1593    my $TIOCGPTN = 0x80045430;
1594
1595    sysopen($ptm, "/dev/ptmx", O_RDWR | O_NONBLOCK) or
1596	dodie "Can't open /dev/ptmx";
1597
1598    # unlockpt()
1599    $tmp = pack("i", 0);
1600    ioctl($ptm, $TIOCSPTLCK, $tmp) or
1601	dodie "ioctl TIOCSPTLCK for /dev/ptmx failed";
1602
1603    # ptsname()
1604    ioctl($ptm, $TIOCGPTN, $tmp) or
1605	dodie "ioctl TIOCGPTN for /dev/ptmx failed";
1606    $tmp = unpack("i", $tmp);
1607
1608    sysopen($pts, "/dev/pts/$tmp", O_RDWR | O_NONBLOCK) or
1609	dodie "Can't open /dev/pts/$tmp";
1610}
1611
1612sub exec_console {
1613    my ($ptm, $pts) = @_;
1614
1615    close($ptm);
1616
1617    close(\*STDIN);
1618    close(\*STDOUT);
1619    close(\*STDERR);
1620
1621    open(\*STDIN, '<&', $pts);
1622    open(\*STDOUT, '>&', $pts);
1623    open(\*STDERR, '>&', $pts);
1624
1625    close($pts);
1626
1627    exec $console or
1628	dodie "Can't open console $console";
1629}
1630
1631sub open_console {
1632    my ($ptm) = @_;
1633    my $pts = \*PTSFD;
1634    my $pid;
1635
1636    # save terminal settings
1637    $stty_orig = `stty -g`;
1638
1639    # place terminal in cbreak mode so that stdin can be read one character at
1640    # a time without having to wait for a newline
1641    system("stty -icanon -echo -icrnl");
1642
1643    create_pty($ptm, $pts);
1644
1645    $pid = fork;
1646
1647    if (!$pid) {
1648	# child
1649	exec_console($ptm, $pts)
1650    }
1651
1652    # parent
1653    close($pts);
1654
1655    return $pid;
1656
1657    open(PTSFD, "Stop perl from warning about single use of PTSFD");
1658}
1659
1660sub close_console {
1661    my ($fp, $pid) = @_;
1662
1663    doprint "kill child process $pid\n";
1664    kill $close_console_signal, $pid;
1665
1666    doprint "wait for child process $pid to exit\n";
1667    waitpid($pid, 0);
1668
1669    print "closing!\n";
1670    close($fp);
1671
1672    # restore terminal settings
1673    system("stty $stty_orig");
1674}
1675
1676sub start_monitor {
1677    if ($monitor_cnt++) {
1678	return;
1679    }
1680    $monitor_fp = \*MONFD;
1681    $monitor_pid = open_console $monitor_fp;
1682
1683    return;
1684
1685    open(MONFD, "Stop perl from warning about single use of MONFD");
1686}
1687
1688sub end_monitor {
1689    return if (!defined $console);
1690    if (--$monitor_cnt) {
1691	return;
1692    }
1693    close_console($monitor_fp, $monitor_pid);
1694}
1695
1696sub wait_for_monitor {
1697    my ($time, $stop) = @_;
1698    my $full_line = "";
1699    my $line;
1700    my $booted = 0;
1701    my $start_time = time;
1702    my $skip_call_trace = 0;
1703    my $bug = 0;
1704    my $bug_ignored = 0;
1705    my $now;
1706
1707    doprint "** Wait for monitor to settle down **\n";
1708
1709    # read the monitor and wait for the system to calm down
1710    while (!$booted) {
1711	$line = wait_for_input($monitor_fp, $time);
1712	last if (!defined($line));
1713	print "$line";
1714	$full_line .= $line;
1715
1716	if (defined($stop) && $full_line =~ /$stop/) {
1717	    doprint "wait for monitor detected $stop\n";
1718	    $booted = 1;
1719	}
1720
1721	if ($full_line =~ /\[ backtrace testing \]/) {
1722	    $skip_call_trace = 1;
1723	}
1724
1725	if ($full_line =~ /call trace:/i) {
1726	    if (!$bug && !$skip_call_trace) {
1727		if ($ignore_errors) {
1728		    $bug_ignored = 1;
1729		} else {
1730		    $bug = 1;
1731		}
1732	    }
1733	}
1734
1735	if ($full_line =~ /\[ end of backtrace testing \]/) {
1736	    $skip_call_trace = 0;
1737	}
1738
1739	if ($full_line =~ /Kernel panic -/) {
1740	    $bug = 1;
1741	}
1742
1743	if ($line =~ /\n/) {
1744	    $full_line = "";
1745	}
1746	$now = time;
1747	if ($now - $start_time >= $max_monitor_wait) {
1748	    doprint "Exiting monitor flush due to hitting MAX_MONITOR_WAIT\n";
1749	    return 1;
1750	}
1751    }
1752    print "** Monitor flushed **\n";
1753
1754    # if stop is defined but wasn't hit, return error
1755    # used by reboot (which wants to see a reboot)
1756    if (defined($stop) && !$booted) {
1757	$bug = 1;
1758    }
1759    return $bug;
1760}
1761
1762sub save_logs {
1763    my ($result, $basedir) = @_;
1764    my @t = localtime;
1765    my $date = sprintf "%04d%02d%02d%02d%02d%02d",
1766	1900+$t[5],$t[4],$t[3],$t[2],$t[1],$t[0];
1767
1768    my $type = $build_type;
1769    if ($type =~ /useconfig/) {
1770	$type = "useconfig";
1771    }
1772
1773    my $dir = "$machine-$test_type-$type-$result-$date";
1774
1775    $dir = "$basedir/$dir";
1776
1777    if (!-d $dir) {
1778	mkpath($dir) or
1779	    dodie "can't create $dir";
1780    }
1781
1782    my %files = (
1783	"config" => $output_config,
1784	"buildlog" => $buildlog,
1785	"dmesg" => $dmesg,
1786	"testlog" => $testlog,
1787    );
1788
1789    while (my ($name, $source) = each(%files)) {
1790	if (-f "$source") {
1791	    cp "$source", "$dir/$name" or
1792		dodie "failed to copy $source";
1793	}
1794    }
1795
1796    doprint "*** Saved info to $dir ***\n";
1797}
1798
1799sub fail {
1800
1801    if ($die_on_failure) {
1802	dodie @_;
1803    }
1804
1805    doprint "FAILED\n";
1806
1807    my $i = $iteration;
1808
1809    # no need to reboot for just building.
1810    if (!do_not_reboot) {
1811	doprint "REBOOTING\n";
1812	reboot_to_good $sleep_time;
1813    }
1814
1815    my $name = "";
1816
1817    if (defined($test_name)) {
1818	$name = " ($test_name)";
1819    }
1820
1821    print_times;
1822
1823    doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
1824    doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
1825    doprint "KTEST RESULT: TEST $i$name Failed: ", @_, "\n";
1826    doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
1827    doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
1828
1829    if (defined($store_failures)) {
1830	save_logs "fail", $store_failures;
1831    }
1832
1833    if (defined($post_test)) {
1834	run_command $post_test;
1835    }
1836
1837    return 1;
1838}
1839
1840sub run_command {
1841    my ($command, $redirect, $timeout) = @_;
1842    my $start_time;
1843    my $end_time;
1844    my $dolog = 0;
1845    my $dord = 0;
1846    my $dostdout = 0;
1847    my $pid;
1848    my $command_orig = $command;
1849
1850    $command =~ s/\$SSH_USER/$ssh_user/g;
1851    $command =~ s/\$MACHINE/$machine/g;
1852
1853    doprint("$command ... ");
1854    $start_time = time;
1855
1856    $pid = open(CMD, "$command 2>&1 |") or
1857	(fail "unable to exec $command" and return 0);
1858
1859    if (defined($opt{"LOG_FILE"})) {
1860	$dolog = 1;
1861    }
1862
1863    if (defined($redirect)) {
1864	if ($redirect eq 1) {
1865	    $dostdout = 1;
1866	    # Have the output of the command on its own line
1867	    doprint "\n";
1868	} else {
1869	    open (RD, ">$redirect") or
1870		dodie "failed to write to redirect $redirect";
1871	    $dord = 1;
1872	}
1873    }
1874
1875    my $hit_timeout = 0;
1876
1877    while (1) {
1878	my $fp = \*CMD;
1879	if (defined($timeout)) {
1880	    doprint "timeout = $timeout\n";
1881	}
1882	my $line = wait_for_input($fp, $timeout);
1883	if (!defined($line)) {
1884	    my $now = time;
1885	    if (defined($timeout) && (($now - $start_time) >= $timeout)) {
1886		doprint "Hit timeout of $timeout, killing process\n";
1887		$hit_timeout = 1;
1888		kill 9, $pid;
1889	    }
1890	    last;
1891	}
1892	print LOG $line if ($dolog);
1893	print RD $line if ($dord);
1894	print $line if ($dostdout);
1895    }
1896
1897    waitpid($pid, 0);
1898    # shift 8 for real exit status
1899    $run_command_status = $? >> 8;
1900
1901    if ($command_orig eq $default{REBOOT} &&
1902	$run_command_status == $reboot_return_code) {
1903	$run_command_status = 0;
1904    }
1905
1906    close(CMD);
1907    close(RD)  if ($dord);
1908
1909    $end_time = time;
1910    my $delta = $end_time - $start_time;
1911
1912    if ($delta == 1) {
1913	doprint "[1 second] ";
1914    } else {
1915	doprint "[$delta seconds] ";
1916    }
1917
1918    if ($hit_timeout) {
1919	$run_command_status = 1;
1920    }
1921
1922    if ($run_command_status) {
1923	doprint "FAILED!\n";
1924    } else {
1925	doprint "SUCCESS\n";
1926    }
1927
1928    return !$run_command_status;
1929}
1930
1931sub run_ssh {
1932    my ($cmd, $timeout) = @_;
1933    my $cp_exec = $ssh_exec;
1934
1935    $cp_exec =~ s/\$SSH_COMMAND/$cmd/g;
1936    return run_command "$cp_exec", undef , $timeout;
1937}
1938
1939sub run_scp {
1940    my ($src, $dst, $cp_scp) = @_;
1941
1942    $cp_scp =~ s/\$SRC_FILE/$src/g;
1943    $cp_scp =~ s/\$DST_FILE/$dst/g;
1944
1945    return run_command "$cp_scp";
1946}
1947
1948sub run_scp_install {
1949    my ($src, $dst) = @_;
1950
1951    my $cp_scp = $scp_to_target_install;
1952
1953    return run_scp($src, $dst, $cp_scp);
1954}
1955
1956sub run_scp_mod {
1957    my ($src, $dst) = @_;
1958
1959    my $cp_scp = $scp_to_target;
1960
1961    return run_scp($src, $dst, $cp_scp);
1962}
1963
1964sub _get_grub_index {
1965
1966    my ($command, $target, $skip, $submenu) = @_;
1967
1968    return if (defined($grub_number) && defined($last_grub_menu) &&
1969	$last_grub_menu eq $grub_menu && defined($last_machine) &&
1970	$last_machine eq $machine);
1971
1972    doprint "Find $reboot_type menu ... ";
1973    $grub_number = -1;
1974
1975    my $ssh_grub = $ssh_exec;
1976    $ssh_grub =~ s,\$SSH_COMMAND,$command,g;
1977
1978    open(IN, "$ssh_grub |") or
1979	dodie "unable to execute $command";
1980
1981    my $found = 0;
1982
1983    my $submenu_number = 0;
1984
1985    while (<IN>) {
1986	if (/$target/) {
1987	    $grub_number++;
1988	    $found = 1;
1989	    last;
1990	} elsif (defined($submenu) && /$submenu/) {
1991		$submenu_number++;
1992		$grub_number = -1;
1993	} elsif (/$skip/) {
1994	    $grub_number++;
1995	}
1996    }
1997    close(IN);
1998
1999    dodie "Could not find '$grub_menu' through $command on $machine"
2000	if (!$found);
2001    if ($submenu_number > 0) {
2002	$grub_number = "$submenu_number>$grub_number";
2003    }
2004    doprint "$grub_number\n";
2005    $last_grub_menu = $grub_menu;
2006    $last_machine = $machine;
2007}
2008
2009sub get_grub_index {
2010
2011    my $command;
2012    my $target;
2013    my $skip;
2014    my $submenu;
2015    my $grub_menu_qt;
2016
2017    if ($reboot_type !~ /^grub/) {
2018	return;
2019    }
2020
2021    $grub_menu_qt = quotemeta($grub_menu);
2022
2023    if ($reboot_type eq "grub") {
2024	$command = "cat /boot/grub/menu.lst";
2025	$target = '^\s*title\s+' . $grub_menu_qt . '\s*$';
2026	$skip = '^\s*title\s';
2027    } elsif ($reboot_type eq "grub2") {
2028	$command = "cat $grub_file";
2029	$target = '^\s*menuentry.*' . $grub_menu_qt;
2030	$skip = '^\s*menuentry';
2031	$submenu = '^\s*submenu\s';
2032    } elsif ($reboot_type eq "grub2bls") {
2033	$command = $grub_bls_get;
2034	$target = '^title=.*' . $grub_menu_qt;
2035	$skip = '^title=';
2036    } else {
2037	return;
2038    }
2039
2040    _get_grub_index($command, $target, $skip, $submenu);
2041}
2042
2043sub wait_for_input {
2044    my ($fp, $time) = @_;
2045    my $start_time;
2046    my $rin;
2047    my $rout;
2048    my $nr;
2049    my $buf;
2050    my $line;
2051    my $ch;
2052
2053    if (!defined($time)) {
2054	$time = $timeout;
2055    }
2056
2057    $rin = '';
2058    vec($rin, fileno($fp), 1) = 1;
2059    vec($rin, fileno(\*STDIN), 1) = 1;
2060
2061    $start_time = time;
2062
2063    while (1) {
2064	$nr = select($rout=$rin, undef, undef, $time);
2065
2066	last if ($nr <= 0);
2067
2068	# copy data from stdin to the console
2069	if (vec($rout, fileno(\*STDIN), 1) == 1) {
2070	    $nr = sysread(\*STDIN, $buf, 1000);
2071	    syswrite($fp, $buf, $nr) if ($nr > 0);
2072	}
2073
2074	# The timeout is based on time waiting for the fp data
2075	if (vec($rout, fileno($fp), 1) != 1) {
2076	    last if (defined($time) && (time - $start_time > $time));
2077	    next;
2078	}
2079
2080	$line = "";
2081
2082	# try to read one char at a time
2083	while (sysread $fp, $ch, 1) {
2084	    $line .= $ch;
2085	    last if ($ch eq "\n");
2086	}
2087
2088	last if (!length($line));
2089
2090	return $line;
2091    }
2092    return undef;
2093}
2094
2095sub reboot_to {
2096    if (defined($switch_to_test)) {
2097	run_command $switch_to_test;
2098    }
2099
2100    if ($reboot_type eq "grub") {
2101	run_ssh "'(echo \"savedefault --default=$grub_number --once\" | grub --batch)'";
2102    } elsif (($reboot_type eq "grub2") or ($reboot_type eq "grub2bls")) {
2103	run_ssh "$grub_reboot \"'$grub_number'\"";
2104    } elsif ($reboot_type eq "syslinux") {
2105	run_ssh "$syslinux --once \\\"$syslinux_label\\\" $syslinux_path";
2106    } elsif (defined $reboot_script) {
2107	run_command "$reboot_script";
2108    }
2109    reboot;
2110}
2111
2112sub get_sha1 {
2113    my ($commit) = @_;
2114
2115    doprint "git rev-list --max-count=1 $commit ... ";
2116    my $sha1 = `git rev-list --max-count=1 $commit`;
2117    my $ret = $?;
2118
2119    logit $sha1;
2120
2121    if ($ret) {
2122	doprint "FAILED\n";
2123	dodie "Failed to get git $commit";
2124    }
2125
2126    print "SUCCESS\n";
2127
2128    chomp $sha1;
2129
2130    return $sha1;
2131}
2132
2133sub monitor {
2134    my $booted = 0;
2135    my $bug = 0;
2136    my $bug_ignored = 0;
2137    my $skip_call_trace = 0;
2138    my $loops;
2139
2140    my $start_time = time;
2141
2142    wait_for_monitor 5;
2143
2144    my $line;
2145    my $full_line = "";
2146
2147    open(DMESG, "> $dmesg") or
2148	dodie "unable to write to $dmesg";
2149
2150    reboot_to;
2151
2152    my $success_start;
2153    my $failure_start;
2154    my $monitor_start = time;
2155    my $done = 0;
2156    my $version_found = 0;
2157
2158    while (!$done) {
2159	if ($bug && defined($stop_after_failure) &&
2160	    $stop_after_failure >= 0) {
2161	    my $time = $stop_after_failure - (time - $failure_start);
2162	    $line = wait_for_input($monitor_fp, $time);
2163	    if (!defined($line)) {
2164		doprint "bug timed out after $booted_timeout seconds\n";
2165		doprint "Test forced to stop after $stop_after_failure seconds after failure\n";
2166		last;
2167	    }
2168	} elsif ($booted) {
2169	    $line = wait_for_input($monitor_fp, $booted_timeout);
2170	    if (!defined($line)) {
2171		my $s = $booted_timeout == 1 ? "" : "s";
2172		doprint "Successful boot found: break after $booted_timeout second$s\n";
2173		last;
2174	    }
2175	} else {
2176	    $line = wait_for_input($monitor_fp);
2177	    if (!defined($line)) {
2178		my $s = $timeout == 1 ? "" : "s";
2179		doprint "Timed out after $timeout second$s\n";
2180		last;
2181	    }
2182	}
2183
2184	doprint $line;
2185	print DMESG $line;
2186
2187	# we are not guaranteed to get a full line
2188	$full_line .= $line;
2189
2190	if ($full_line =~ /$success_line/) {
2191	    $booted = 1;
2192	    $success_start = time;
2193	}
2194
2195	if ($booted && defined($stop_after_success) &&
2196	    $stop_after_success >= 0) {
2197	    my $now = time;
2198	    if ($now - $success_start >= $stop_after_success) {
2199		doprint "Test forced to stop after $stop_after_success seconds after success\n";
2200		last;
2201	    }
2202	}
2203
2204	if ($full_line =~ /\[ backtrace testing \]/) {
2205	    $skip_call_trace = 1;
2206	}
2207
2208	if ($full_line =~ /call trace:/i) {
2209	    if (!$bug && !$skip_call_trace) {
2210		if ($ignore_errors) {
2211		    $bug_ignored = 1;
2212		} else {
2213		    $bug = 1;
2214		    $failure_start = time;
2215		}
2216	    }
2217	}
2218
2219	if ($bug && defined($stop_after_failure) &&
2220	    $stop_after_failure >= 0) {
2221	    my $now = time;
2222	    if ($now - $failure_start >= $stop_after_failure) {
2223		doprint "Test forced to stop after $stop_after_failure seconds after failure\n";
2224		last;
2225	    }
2226	}
2227
2228	if ($full_line =~ /\[ end of backtrace testing \]/) {
2229	    $skip_call_trace = 0;
2230	}
2231
2232	if ($full_line =~ /Kernel panic -/) {
2233	    $failure_start = time;
2234	    $bug = 1;
2235	}
2236
2237	# Detect triple faults by testing the banner
2238	if ($full_line =~ /\bLinux version (\S+).*\n/) {
2239	    if ($1 eq $version) {
2240		$version_found = 1;
2241	    } elsif ($version_found && $detect_triplefault) {
2242		# We already booted into the kernel we are testing,
2243		# but now we booted into another kernel?
2244		# Consider this a triple fault.
2245		doprint "Already booted in Linux kernel $version, but now\n";
2246		doprint "we booted into Linux kernel $1.\n";
2247		doprint "Assuming that this is a triple fault.\n";
2248		doprint "To disable this: set DETECT_TRIPLE_FAULT to 0\n";
2249		last;
2250	    }
2251	}
2252
2253	if ($line =~ /\n/) {
2254	    $full_line = "";
2255	}
2256
2257	if ($stop_test_after > 0 && !$booted && !$bug) {
2258	    if (time - $monitor_start > $stop_test_after) {
2259		doprint "STOP_TEST_AFTER ($stop_test_after seconds) timed out\n";
2260		$done = 1;
2261	    }
2262	}
2263    }
2264
2265    my $end_time = time;
2266    $reboot_time = $end_time - $start_time;
2267
2268    close(DMESG);
2269
2270    if ($bug) {
2271	return 0 if ($in_bisect);
2272	fail "failed - got a bug report" and return 0;
2273    }
2274
2275    if (!$booted) {
2276	return 0 if ($in_bisect);
2277	fail "failed - never got a boot prompt." and return 0;
2278    }
2279
2280    if ($bug_ignored) {
2281	doprint "WARNING: Call Trace detected but ignored due to IGNORE_ERRORS=1\n";
2282    }
2283
2284    return 1;
2285}
2286
2287sub eval_kernel_version {
2288    my ($option) = @_;
2289
2290    $option =~ s/\$KERNEL_VERSION/$version/g;
2291
2292    return $option;
2293}
2294
2295sub do_post_install {
2296
2297    return if (!defined($post_install));
2298
2299    my $cp_post_install = eval_kernel_version $post_install;
2300    run_command "$cp_post_install" or
2301	dodie "Failed to run post install";
2302}
2303
2304# Sometimes the reboot fails, and will hang. We try to ssh to the box
2305# and if we fail, we force another reboot, that should powercycle it.
2306sub test_booted {
2307    if (!run_ssh "echo testing connection") {
2308	reboot $sleep_time;
2309    }
2310}
2311
2312sub install {
2313
2314    return if ($no_install);
2315
2316    my $start_time = time;
2317
2318    if (defined($pre_install)) {
2319	my $cp_pre_install = eval_kernel_version $pre_install;
2320	run_command "$cp_pre_install" or
2321	    dodie "Failed to run pre install";
2322    }
2323
2324    my $cp_target = eval_kernel_version $target_image;
2325
2326    test_booted;
2327
2328    run_scp_install "$outputdir/$build_target", "$cp_target" or
2329	dodie "failed to copy image";
2330
2331    my $install_mods = 0;
2332
2333    # should we process modules?
2334    $install_mods = 0;
2335    open(IN, "$output_config") or dodie("Can't read config file");
2336    while (<IN>) {
2337	if (/CONFIG_MODULES(=y)?/) {
2338	    if (defined($1)) {
2339		$install_mods = 1;
2340		last;
2341	    }
2342	}
2343    }
2344    close(IN);
2345
2346    if (!$install_mods) {
2347	do_post_install;
2348	doprint "No modules needed\n";
2349	my $end_time = time;
2350	$install_time = $end_time - $start_time;
2351	return;
2352    }
2353
2354    run_command "$make INSTALL_MOD_STRIP=1 INSTALL_MOD_PATH=$tmpdir modules_install" or
2355	dodie "Failed to install modules";
2356
2357    my $modlib = "/lib/modules/$version";
2358    my $modtar = "ktest-mods.tar.bz2";
2359
2360    run_ssh "rm -rf $modlib" or
2361	dodie "failed to remove old mods: $modlib";
2362
2363    # would be nice if scp -r did not follow symbolic links
2364    run_command "cd $tmpdir && tar -cjf $modtar lib/modules/$version" or
2365	dodie "making tarball";
2366
2367    run_scp_mod "$tmpdir/$modtar", "/tmp" or
2368	dodie "failed to copy modules";
2369
2370    unlink "$tmpdir/$modtar";
2371
2372    run_ssh "'(cd / && tar xjf /tmp/$modtar)'" or
2373	dodie "failed to tar modules";
2374
2375    run_ssh "rm -f /tmp/$modtar";
2376
2377    do_post_install;
2378
2379    my $end_time = time;
2380    $install_time = $end_time - $start_time;
2381}
2382
2383sub get_version {
2384    # get the release name
2385    return if ($have_version);
2386    doprint "$make kernelrelease ... ";
2387    $version = `$make -s kernelrelease | tail -1`;
2388    chomp($version);
2389    doprint "$version\n";
2390    $have_version = 1;
2391}
2392
2393sub start_monitor_and_install {
2394    # Make sure the stable kernel has finished booting
2395
2396    # Install bisects, don't need console
2397    if (defined $console) {
2398	start_monitor;
2399	wait_for_monitor 5;
2400	end_monitor;
2401    }
2402
2403    get_grub_index;
2404    get_version;
2405    install;
2406
2407    start_monitor if (defined $console);
2408    return monitor;
2409}
2410
2411sub process_warning_line {
2412    my ($line) = @_;
2413
2414    chomp $line;
2415
2416    # for distcc heterogeneous systems, some compilers
2417    # do things differently causing warning lines
2418    # to be slightly different. This makes an attempt
2419    # to fixe those issues.
2420
2421    # chop off the index into the line
2422    # using distcc, some compilers give different indexes
2423    # depending on white space
2424    $line =~ s/^(\s*\S+:\d+:)\d+/$1/;
2425
2426    # Some compilers use UTF-8 extended for quotes and some don't.
2427    $line =~ s/$utf8_quote/'/g;
2428
2429    return $line;
2430}
2431
2432# Read buildlog and check against warnings file for any
2433# new warnings.
2434#
2435# Returns 1 if OK
2436#         0 otherwise
2437sub check_buildlog {
2438    return 1 if (!defined $warnings_file);
2439
2440    my %warnings_list;
2441
2442    # Failed builds should not reboot the target
2443    my $save_no_reboot = $no_reboot;
2444    $no_reboot = 1;
2445
2446    if (-f $warnings_file) {
2447	open(IN, $warnings_file) or
2448	    dodie "Error opening $warnings_file";
2449
2450	while (<IN>) {
2451	    if (/$check_build_re/) {
2452		my $warning = process_warning_line $_;
2453
2454		$warnings_list{$warning} = 1;
2455	    }
2456	}
2457	close(IN);
2458    }
2459
2460    # If warnings file didn't exist, and WARNINGS_FILE exist,
2461    # then we fail on any warning!
2462
2463    open(IN, $buildlog) or dodie "Can't open $buildlog";
2464    while (<IN>) {
2465	if (/$check_build_re/) {
2466	    my $warning = process_warning_line $_;
2467
2468	    if (!defined $warnings_list{$warning}) {
2469		fail "New warning found (not in $warnings_file)\n$_\n";
2470		$no_reboot = $save_no_reboot;
2471		return 0;
2472	    }
2473	}
2474    }
2475    $no_reboot = $save_no_reboot;
2476    close(IN);
2477}
2478
2479sub check_patch_buildlog {
2480    my ($patch) = @_;
2481
2482    my @files = `git show $patch | diffstat -l`;
2483
2484    foreach my $file (@files) {
2485	chomp $file;
2486    }
2487
2488    open(IN, "git show $patch |") or
2489	dodie "failed to show $patch";
2490    while (<IN>) {
2491	if (m,^--- a/(.*),) {
2492	    chomp $1;
2493	    $files[$#files] = $1;
2494	}
2495    }
2496    close(IN);
2497
2498    open(IN, $buildlog) or dodie "Can't open $buildlog";
2499    while (<IN>) {
2500	if (/^\s*(.*?):.*(warning|error)/) {
2501	    my $err = $1;
2502	    foreach my $file (@files) {
2503		my $fullpath = "$builddir/$file";
2504		if ($file eq $err || $fullpath eq $err) {
2505		    fail "$file built with warnings" and return 0;
2506		}
2507	    }
2508	}
2509    }
2510    close(IN);
2511
2512    return 1;
2513}
2514
2515sub apply_min_config {
2516    my $outconfig = "$output_config.new";
2517
2518    # Read the config file and remove anything that
2519    # is in the force_config hash (from minconfig and others)
2520    # then add the force config back.
2521
2522    doprint "Applying minimum configurations into $output_config.new\n";
2523
2524    open (OUT, ">$outconfig") or
2525	dodie "Can't create $outconfig";
2526
2527    if (-f $output_config) {
2528	open (IN, $output_config) or
2529	    dodie "Failed to open $output_config";
2530	while (<IN>) {
2531	    if (/^(# )?(CONFIG_[^\s=]*)/) {
2532		next if (defined($force_config{$2}));
2533	    }
2534	    print OUT;
2535	}
2536	close IN;
2537    }
2538    foreach my $config (keys %force_config) {
2539	print OUT "$force_config{$config}\n";
2540    }
2541    close OUT;
2542
2543    run_command "mv $outconfig $output_config";
2544}
2545
2546sub make_oldconfig {
2547
2548    my @force_list = keys %force_config;
2549
2550    if ($#force_list >= 0) {
2551	apply_min_config;
2552    }
2553
2554    if (!run_command "$make olddefconfig") {
2555	# Perhaps olddefconfig doesn't exist in this version of the kernel
2556	# try oldnoconfig
2557	doprint "olddefconfig failed, trying make oldnoconfig\n";
2558	if (!run_command "$make oldnoconfig") {
2559	    doprint "oldnoconfig failed, trying yes '' | make oldconfig\n";
2560	    # try a yes '' | oldconfig
2561	    run_command "yes '' | $make oldconfig" or
2562		dodie "failed make config oldconfig";
2563	}
2564    }
2565}
2566
2567# read a config file and use this to force new configs.
2568sub load_force_config {
2569    my ($config) = @_;
2570
2571    doprint "Loading force configs from $config\n";
2572    open(IN, $config) or
2573	dodie "failed to read $config";
2574    while (<IN>) {
2575	chomp;
2576	if (/^(CONFIG[^\s=]*)(\s*=.*)/) {
2577	    $force_config{$1} = $_;
2578	} elsif (/^# (CONFIG_\S*) is not set/) {
2579	    $force_config{$1} = $_;
2580	}
2581    }
2582    close IN;
2583}
2584
2585sub build {
2586    my ($type) = @_;
2587
2588    unlink $buildlog;
2589
2590    my $start_time = time;
2591
2592    # Failed builds should not reboot the target
2593    my $save_no_reboot = $no_reboot;
2594    $no_reboot = 1;
2595
2596    # Calculate a new version from here.
2597    $have_version = 0;
2598
2599    if (defined($pre_build)) {
2600	my $ret = run_command $pre_build;
2601	if (!$ret && defined($pre_build_die) &&
2602	    $pre_build_die) {
2603	    dodie "failed to pre_build\n";
2604	}
2605    }
2606
2607    if ($type =~ /^useconfig:(.*)/) {
2608	run_command "cp $1 $output_config" or
2609	    dodie "could not copy $1 to .config";
2610
2611	$type = "oldconfig";
2612    }
2613
2614    # old config can ask questions
2615    if ($type eq "oldconfig") {
2616	$type = "olddefconfig";
2617
2618	# allow for empty configs
2619	run_command "touch $output_config";
2620
2621	if (!$noclean) {
2622	    run_command "mv $output_config $outputdir/config_temp" or
2623		dodie "moving .config";
2624
2625	    run_command "$make mrproper" or dodie "make mrproper";
2626
2627	    run_command "mv $outputdir/config_temp $output_config" or
2628		dodie "moving config_temp";
2629	}
2630    } elsif (!$noclean) {
2631	unlink "$output_config";
2632	run_command "$make mrproper" or
2633	    dodie "make mrproper";
2634    }
2635
2636    # add something to distinguish this build
2637    open(OUT, "> $outputdir/localversion") or dodie("Can't make localversion file");
2638    print OUT "$localversion\n";
2639    close(OUT);
2640
2641    if (defined($minconfig)) {
2642	load_force_config($minconfig);
2643    }
2644
2645    if ($type ne "olddefconfig") {
2646	run_command "$make $type" or
2647	    dodie "failed make config";
2648    }
2649    # Run old config regardless, to enforce min configurations
2650    make_oldconfig;
2651
2652    if (not defined($build_options)){
2653	$build_options = "";
2654    }
2655    my $build_ret = run_command "$make $build_options", $buildlog;
2656
2657    if (defined($post_build)) {
2658	# Because a post build may change the kernel version
2659	# do it now.
2660	get_version;
2661	my $ret = run_command $post_build;
2662	if (!$ret && defined($post_build_die) &&
2663	    $post_build_die) {
2664	    dodie "failed to post_build\n";
2665	}
2666    }
2667
2668    if (!$build_ret) {
2669	# bisect may need this to pass
2670	if ($in_bisect) {
2671	    $no_reboot = $save_no_reboot;
2672	    return 0;
2673	}
2674	fail "failed build" and return 0;
2675    }
2676
2677    $no_reboot = $save_no_reboot;
2678
2679    my $end_time = time;
2680    $build_time = $end_time - $start_time;
2681
2682    return 1;
2683}
2684
2685sub halt {
2686    if (!run_ssh "halt" or defined($power_off)) {
2687	if (defined($poweroff_after_halt)) {
2688	    sleep $poweroff_after_halt;
2689	    run_command "$power_off";
2690	}
2691    } else {
2692	# nope? the zap it!
2693	run_command "$power_off";
2694    }
2695}
2696
2697sub success {
2698    my ($i) = @_;
2699
2700    $successes++;
2701
2702    my $name = "";
2703
2704    if (defined($test_name)) {
2705	$name = " ($test_name)";
2706    }
2707
2708    print_times;
2709
2710    doprint "\n\n";
2711    doprint "*******************************************\n";
2712    doprint "*******************************************\n";
2713    doprint "KTEST RESULT: TEST $i$name SUCCESS!!!!   **\n";
2714    doprint "*******************************************\n";
2715    doprint "*******************************************\n";
2716
2717    if (defined($store_successes)) {
2718	save_logs "success", $store_successes;
2719    }
2720
2721    if ($i != $opt{"NUM_TESTS"} && !do_not_reboot) {
2722	doprint "Reboot and wait $sleep_time seconds\n";
2723	reboot_to_good $sleep_time;
2724    }
2725
2726    if (defined($post_test)) {
2727	run_command $post_test;
2728    }
2729}
2730
2731sub answer_bisect {
2732    for (;;) {
2733	doprint "Pass, fail, or skip? [p/f/s]";
2734	my $ans = <STDIN>;
2735	chomp $ans;
2736	if ($ans eq "p" || $ans eq "P") {
2737	    return 1;
2738	} elsif ($ans eq "f" || $ans eq "F") {
2739	    return 0;
2740	} elsif ($ans eq "s" || $ans eq "S") {
2741	    return -1;
2742	} else {
2743	    print "Please answer 'p', 'f', or 's'\n";
2744	}
2745    }
2746}
2747
2748sub child_run_test {
2749
2750    # child should have no power
2751    $reboot_on_error = 0;
2752    $poweroff_on_error = 0;
2753    $die_on_failure = 1;
2754
2755    run_command $run_test, $testlog;
2756
2757    exit $run_command_status;
2758}
2759
2760sub child_finished {
2761    $child_done = 1;
2762}
2763
2764sub do_run_test {
2765    my $child_pid;
2766    my $child_exit;
2767    my $line;
2768    my $full_line;
2769    my $bug = 0;
2770    my $bug_ignored = 0;
2771
2772    my $start_time = time;
2773
2774    wait_for_monitor 1;
2775
2776    doprint "run test $run_test\n";
2777
2778    $child_done = 0;
2779
2780    $SIG{CHLD} = qw(child_finished);
2781
2782    $child_pid = fork;
2783
2784    child_run_test if (!$child_pid);
2785
2786    $full_line = "";
2787
2788    do {
2789	$line = wait_for_input($monitor_fp, 1);
2790	if (defined($line)) {
2791
2792	    # we are not guaranteed to get a full line
2793	    $full_line .= $line;
2794	    doprint $line;
2795
2796	    if ($full_line =~ /call trace:/i) {
2797		if ($ignore_errors) {
2798		    $bug_ignored = 1;
2799		} else {
2800		    $bug = 1;
2801		}
2802	    }
2803
2804	    if ($full_line =~ /Kernel panic -/) {
2805		$bug = 1;
2806	    }
2807
2808	    if ($line =~ /\n/) {
2809		$full_line = "";
2810	    }
2811	}
2812    } while (!$child_done && !$bug);
2813
2814    if (!$bug && $bug_ignored) {
2815	doprint "WARNING: Call Trace detected but ignored due to IGNORE_ERRORS=1\n";
2816    }
2817
2818    if ($bug) {
2819	my $failure_start = time;
2820	my $now;
2821	do {
2822	    $line = wait_for_input($monitor_fp, 1);
2823	    if (defined($line)) {
2824		doprint $line;
2825	    }
2826	    $now = time;
2827	    if ($now - $failure_start >= $stop_after_failure) {
2828		last;
2829	    }
2830	} while (defined($line));
2831
2832	doprint "Detected kernel crash!\n";
2833	# kill the child with extreme prejudice
2834	kill 9, $child_pid;
2835    }
2836
2837    waitpid $child_pid, 0;
2838    $child_exit = $? >> 8;
2839
2840    my $end_time = time;
2841    $test_time = $end_time - $start_time;
2842
2843    if (!$bug && $in_bisect) {
2844	if (defined($bisect_ret_good)) {
2845	    if ($child_exit == $bisect_ret_good) {
2846		return 1;
2847	    }
2848	}
2849	if (defined($bisect_ret_skip)) {
2850	    if ($child_exit == $bisect_ret_skip) {
2851		return -1;
2852	    }
2853	}
2854	if (defined($bisect_ret_abort)) {
2855	    if ($child_exit == $bisect_ret_abort) {
2856		fail "test abort" and return -2;
2857	    }
2858	}
2859	if (defined($bisect_ret_bad)) {
2860	    if ($child_exit == $bisect_ret_skip) {
2861		return 0;
2862	    }
2863	}
2864	if (defined($bisect_ret_default)) {
2865	    if ($bisect_ret_default eq "good") {
2866		return 1;
2867	    } elsif ($bisect_ret_default eq "bad") {
2868		return 0;
2869	    } elsif ($bisect_ret_default eq "skip") {
2870		return -1;
2871	    } elsif ($bisect_ret_default eq "abort") {
2872		return -2;
2873	    } else {
2874		fail "unknown default action: $bisect_ret_default"
2875		    and return -2;
2876	    }
2877	}
2878    }
2879
2880    if ($bug || $child_exit) {
2881	return 0 if $in_bisect;
2882	fail "test failed" and return 0;
2883    }
2884    return 1;
2885}
2886
2887sub run_git_bisect {
2888    my ($command) = @_;
2889
2890    doprint "$command ... ";
2891
2892    my $output = `$command 2>&1`;
2893    my $ret = $?;
2894
2895    logit $output;
2896
2897    if ($ret) {
2898	doprint "FAILED\n";
2899	dodie "Failed to git bisect";
2900    }
2901
2902    doprint "SUCCESS\n";
2903    if ($output =~ m/^(Bisecting: .*\(roughly \d+ steps?\))\s+\[([[:xdigit:]]+)\]/) {
2904	doprint "$1 [$2]\n";
2905    } elsif ($output =~ m/^([[:xdigit:]]+) is the first bad commit/) {
2906	$bisect_bad_commit = $1;
2907	doprint "Found bad commit... $1\n";
2908	return 0;
2909    } else {
2910	# we already logged it, just print it now.
2911	print $output;
2912    }
2913
2914    return 1;
2915}
2916
2917sub bisect_reboot {
2918    doprint "Reboot and sleep $bisect_sleep_time seconds\n";
2919    reboot_to_good $bisect_sleep_time;
2920}
2921
2922# returns 1 on success, 0 on failure, -1 on skip
2923sub run_bisect_test {
2924    my ($type, $buildtype) = @_;
2925
2926    my $failed = 0;
2927    my $result;
2928    my $output;
2929    my $ret;
2930
2931    $in_bisect = 1;
2932
2933    build $buildtype or $failed = 1;
2934
2935    if ($type ne "build") {
2936	if ($failed && $bisect_skip) {
2937	    $in_bisect = 0;
2938	    return -1;
2939	}
2940	dodie "Failed on build" if $failed;
2941
2942	# Now boot the box
2943	start_monitor_and_install or $failed = 1;
2944
2945	if ($type ne "boot") {
2946	    if ($failed && $bisect_skip) {
2947		end_monitor;
2948		bisect_reboot;
2949		$in_bisect = 0;
2950		return -1;
2951	    }
2952	    dodie "Failed on boot" if $failed;
2953
2954	    do_run_test or $failed = 1;
2955	}
2956	end_monitor;
2957    }
2958
2959    if ($failed) {
2960	$result = 0;
2961    } else {
2962	$result = 1;
2963    }
2964
2965    # reboot the box to a kernel we can ssh to
2966    if ($type ne "build") {
2967	bisect_reboot;
2968    }
2969    $in_bisect = 0;
2970
2971    return $result;
2972}
2973
2974sub run_bisect {
2975    my ($type) = @_;
2976    my $buildtype = "oldconfig";
2977
2978    # We should have a minconfig to use?
2979    if (defined($minconfig)) {
2980	$buildtype = "useconfig:$minconfig";
2981    }
2982
2983    # If the user sets bisect_tries to less than 1, then no tries
2984    # is a success.
2985    my $ret = 1;
2986
2987    # Still let the user manually decide that though.
2988    if ($bisect_tries < 1 && $bisect_manual) {
2989	$ret = answer_bisect;
2990    }
2991
2992    for (my $i = 0; $i < $bisect_tries; $i++) {
2993	if ($bisect_tries > 1) {
2994	    my $t = $i + 1;
2995	    doprint("Running bisect trial $t of $bisect_tries:\n");
2996	}
2997	$ret = run_bisect_test $type, $buildtype;
2998
2999	if ($bisect_manual) {
3000	    $ret = answer_bisect;
3001	}
3002
3003	last if (!$ret);
3004    }
3005
3006    # Are we looking for where it worked, not failed?
3007    if ($reverse_bisect && $ret >= 0) {
3008	$ret = !$ret;
3009    }
3010
3011    if ($ret > 0) {
3012	return "good";
3013    } elsif ($ret == 0) {
3014	return  "bad";
3015    } elsif ($bisect_skip) {
3016	doprint "HIT A BAD COMMIT ... SKIPPING\n";
3017	return "skip";
3018    }
3019}
3020
3021sub update_bisect_replay {
3022    my $tmp_log = "$tmpdir/ktest_bisect_log";
3023    run_command "git bisect log > $tmp_log" or
3024	dodie "can't create bisect log";
3025    return $tmp_log;
3026}
3027
3028sub bisect {
3029    my ($i) = @_;
3030
3031    my $result;
3032
3033    dodie "BISECT_GOOD[$i] not defined\n"	if (!defined($bisect_good));
3034    dodie "BISECT_BAD[$i] not defined\n"	if (!defined($bisect_bad));
3035    dodie "BISECT_TYPE[$i] not defined\n"	if (!defined($bisect_type));
3036
3037    my $good = $bisect_good;
3038    my $bad = $bisect_bad;
3039    my $type = $bisect_type;
3040    my $start = $bisect_start;
3041    my $replay = $bisect_replay;
3042    my $start_files = $bisect_files;
3043
3044    if (defined($start_files)) {
3045	$start_files = " -- " . $start_files;
3046    } else {
3047	$start_files = "";
3048    }
3049
3050    # convert to true sha1's
3051    $good = get_sha1($good);
3052    $bad = get_sha1($bad);
3053
3054    if (defined($bisect_reverse) && $bisect_reverse == 1) {
3055	doprint "Performing a reverse bisect (bad is good, good is bad!)\n";
3056	$reverse_bisect = 1;
3057    } else {
3058	$reverse_bisect = 0;
3059    }
3060
3061    # Can't have a test without having a test to run
3062    if ($type eq "test" && !defined($run_test)) {
3063	$type = "boot";
3064    }
3065
3066    # Check if a bisect was running
3067    my $bisect_start_file = "$builddir/.git/BISECT_START";
3068
3069    my $check = $bisect_check;
3070    my $do_check = defined($check) && $check ne "0";
3071
3072    if ( -f $bisect_start_file ) {
3073	print "Bisect in progress found\n";
3074	if ($do_check) {
3075	    print " If you say yes, then no checks of good or bad will be done\n";
3076	}
3077	if (defined($replay)) {
3078	    print "** BISECT_REPLAY is defined in config file **";
3079	    print " Ignore config option and perform new git bisect log?\n";
3080	    if (read_ync " (yes, no, or cancel) ") {
3081		$replay = update_bisect_replay;
3082		$do_check = 0;
3083	    }
3084	} elsif (read_yn "read git log and continue?") {
3085	    $replay = update_bisect_replay;
3086	    $do_check = 0;
3087	}
3088    }
3089
3090    if ($do_check) {
3091	# get current HEAD
3092	my $head = get_sha1("HEAD");
3093
3094	if ($check ne "good") {
3095	    doprint "TESTING BISECT BAD [$bad]\n";
3096	    run_command "git checkout $bad" or
3097		dodie "Failed to checkout $bad";
3098
3099	    $result = run_bisect $type;
3100
3101	    if ($result ne "bad") {
3102		fail "Tested BISECT_BAD [$bad] and it succeeded" and return 0;
3103	    }
3104	}
3105
3106	if ($check ne "bad") {
3107	    doprint "TESTING BISECT GOOD [$good]\n";
3108	    run_command "git checkout $good" or
3109		dodie "Failed to checkout $good";
3110
3111	    $result = run_bisect $type;
3112
3113	    if ($result ne "good") {
3114		fail "Tested BISECT_GOOD [$good] and it failed" and return 0;
3115	    }
3116	}
3117
3118	# checkout where we started
3119	run_command "git checkout $head" or
3120	    dodie "Failed to checkout $head";
3121    }
3122
3123    run_command "git bisect start$start_files" or
3124	dodie "could not start bisect";
3125
3126    if (defined($replay)) {
3127	run_command "git bisect replay $replay" or
3128	    dodie "failed to run replay";
3129    } else {
3130	run_command "git bisect good $good" or
3131	    dodie "could not set bisect good to $good";
3132
3133	run_git_bisect "git bisect bad $bad" or
3134	    dodie "could not set bisect bad to $bad";
3135    }
3136
3137    if (defined($start)) {
3138	run_command "git checkout $start" or
3139	    dodie "failed to checkout $start";
3140    }
3141
3142    my $test;
3143    do {
3144	$result = run_bisect $type;
3145	$test = run_git_bisect "git bisect $result";
3146	print_times;
3147    } while ($test);
3148
3149    run_command "git bisect log" or
3150	dodie "could not capture git bisect log";
3151
3152    run_command "git bisect reset" or
3153	dodie "could not reset git bisect";
3154
3155    doprint "Bad commit was [$bisect_bad_commit]\n";
3156
3157    success $i;
3158}
3159
3160sub assign_configs {
3161    my ($hash, $config) = @_;
3162
3163    doprint "Reading configs from $config\n";
3164
3165    open (IN, $config) or
3166	dodie "Failed to read $config";
3167
3168    while (<IN>) {
3169	chomp;
3170	if (/^((CONFIG\S*)=.*)/) {
3171	    ${$hash}{$2} = $1;
3172	} elsif (/^(# (CONFIG\S*) is not set)/) {
3173	    ${$hash}{$2} = $1;
3174	}
3175    }
3176
3177    close(IN);
3178}
3179
3180sub process_config_ignore {
3181    my ($config) = @_;
3182
3183    assign_configs \%config_ignore, $config;
3184}
3185
3186sub get_dependencies {
3187    my ($config) = @_;
3188
3189    my $arr = $dependency{$config};
3190    if (!defined($arr)) {
3191	return ();
3192    }
3193
3194    my @deps = @{$arr};
3195
3196    foreach my $dep (@{$arr}) {
3197	print "ADD DEP $dep\n";
3198	@deps = (@deps, get_dependencies $dep);
3199    }
3200
3201    return @deps;
3202}
3203
3204sub save_config {
3205    my ($pc, $file) = @_;
3206
3207    my %configs = %{$pc};
3208
3209    doprint "Saving configs into $file\n";
3210
3211    open(OUT, ">$file") or dodie "Can not write to $file";
3212
3213    foreach my $config (keys %configs) {
3214	print OUT "$configs{$config}\n";
3215    }
3216    close(OUT);
3217}
3218
3219sub create_config {
3220    my ($name, $pc) = @_;
3221
3222    doprint "Creating old config from $name configs\n";
3223
3224    save_config $pc, $output_config;
3225
3226    make_oldconfig;
3227}
3228
3229sub run_config_bisect_test {
3230    my ($type) = @_;
3231
3232    my $ret = run_bisect_test $type, "oldconfig";
3233
3234    if ($bisect_manual) {
3235	$ret = answer_bisect;
3236    }
3237
3238    return $ret;
3239}
3240
3241sub config_bisect_end {
3242    my ($good, $bad) = @_;
3243    my $diffexec = "diff -u";
3244
3245    if (-f "$builddir/scripts/diffconfig") {
3246	$diffexec = "$builddir/scripts/diffconfig";
3247    }
3248    doprint "\n\n***************************************\n";
3249    doprint "No more config bisecting possible.\n";
3250    run_command "$diffexec $good $bad", 1;
3251    doprint "***************************************\n\n";
3252}
3253
3254sub run_config_bisect {
3255    my ($good, $bad, $last_result) = @_;
3256    my $reset = "";
3257    my $cmd;
3258    my $ret;
3259
3260    if (!length($last_result)) {
3261	$reset = "-r";
3262    }
3263    run_command "$config_bisect_exec $reset -b $outputdir $good $bad $last_result", 1;
3264
3265    # config-bisect returns:
3266    #   0 if there is more to bisect
3267    #   1 for finding a good config
3268    #   2 if it can not find any more configs
3269    #  -1 (255) on error
3270    if ($run_command_status) {
3271	return $run_command_status;
3272    }
3273
3274    $ret = run_config_bisect_test $config_bisect_type;
3275    if ($ret) {
3276	doprint "NEW GOOD CONFIG ($pass)\n";
3277	system("cp $output_config $tmpdir/good_config.tmp.$pass");
3278	$pass++;
3279	# Return 3 for good config
3280	return 3;
3281    } else {
3282	doprint "NEW BAD CONFIG ($pass)\n";
3283	system("cp $output_config $tmpdir/bad_config.tmp.$pass");
3284	$pass++;
3285	# Return 4 for bad config
3286	return 4;
3287    }
3288}
3289
3290sub config_bisect {
3291    my ($i) = @_;
3292
3293    my $good_config;
3294    my $bad_config;
3295
3296    my $type = $config_bisect_type;
3297    my $ret;
3298
3299    $bad_config = $config_bisect;
3300
3301    if (defined($config_bisect_good)) {
3302	$good_config = $config_bisect_good;
3303    } elsif (defined($minconfig)) {
3304	$good_config = $minconfig;
3305    } else {
3306	doprint "No config specified, checking if defconfig works";
3307	$ret = run_bisect_test $type, "defconfig";
3308	if (!$ret) {
3309	    fail "Have no good config to compare with, please set CONFIG_BISECT_GOOD";
3310	    return 1;
3311	}
3312	$good_config = $output_config;
3313    }
3314
3315    if (!defined($config_bisect_exec)) {
3316	# First check the location that ktest.pl ran
3317	my @locations = (
3318		"$pwd/config-bisect.pl",
3319		"$dirname/config-bisect.pl",
3320		"$builddir/tools/testing/ktest/config-bisect.pl",
3321		undef );
3322	foreach my $loc (@locations) {
3323	    doprint "loc = $loc\n";
3324	    $config_bisect_exec = $loc;
3325	    last if (defined($config_bisect_exec && -x $config_bisect_exec));
3326	}
3327	if (!defined($config_bisect_exec)) {
3328	    fail "Could not find an executable config-bisect.pl\n",
3329		"  Set CONFIG_BISECT_EXEC to point to config-bisect.pl";
3330	    return 1;
3331	}
3332    }
3333
3334    # we don't want min configs to cause issues here.
3335    doprint "Disabling 'MIN_CONFIG' for this test\n";
3336    undef $minconfig;
3337
3338    my %good_configs;
3339    my %bad_configs;
3340    my %tmp_configs;
3341
3342    if (-f "$tmpdir/good_config.tmp" || -f "$tmpdir/bad_config.tmp") {
3343	if (read_yn "Interrupted config-bisect. Continue (n - will start new)?") {
3344	    if (-f "$tmpdir/good_config.tmp") {
3345		$good_config = "$tmpdir/good_config.tmp";
3346	    } else {
3347		$good_config = "$tmpdir/good_config";
3348	    }
3349	    if (-f "$tmpdir/bad_config.tmp") {
3350		$bad_config = "$tmpdir/bad_config.tmp";
3351	    } else {
3352		$bad_config = "$tmpdir/bad_config";
3353	    }
3354	}
3355    }
3356    doprint "Run good configs through make oldconfig\n";
3357    assign_configs \%tmp_configs, $good_config;
3358    create_config "$good_config", \%tmp_configs;
3359    $good_config = "$tmpdir/good_config";
3360    system("cp $output_config $good_config") == 0 or dodie "cp good config";
3361
3362    doprint "Run bad configs through make oldconfig\n";
3363    assign_configs \%tmp_configs, $bad_config;
3364    create_config "$bad_config", \%tmp_configs;
3365    $bad_config = "$tmpdir/bad_config";
3366    system("cp $output_config $bad_config") == 0 or dodie "cp bad config";
3367
3368    if (defined($config_bisect_check) && $config_bisect_check ne "0") {
3369	if ($config_bisect_check ne "good") {
3370	    doprint "Testing bad config\n";
3371
3372	    $ret = run_bisect_test $type, "useconfig:$bad_config";
3373	    if ($ret) {
3374		fail "Bad config succeeded when expected to fail!";
3375		return 0;
3376	    }
3377	}
3378	if ($config_bisect_check ne "bad") {
3379	    doprint "Testing good config\n";
3380
3381	    $ret = run_bisect_test $type, "useconfig:$good_config";
3382	    if (!$ret) {
3383		fail "Good config failed when expected to succeed!";
3384		return 0;
3385	    }
3386	}
3387    }
3388
3389    my $last_run = "";
3390
3391    do {
3392	$ret = run_config_bisect $good_config, $bad_config, $last_run;
3393	if ($ret == 3) {
3394	    $last_run = "good";
3395	} elsif ($ret == 4) {
3396	    $last_run = "bad";
3397	}
3398	print_times;
3399    } while ($ret == 3 || $ret == 4);
3400
3401    if ($ret == 2) {
3402	config_bisect_end "$good_config.tmp", "$bad_config.tmp";
3403    }
3404
3405    return $ret if ($ret < 0);
3406
3407    success $i;
3408}
3409
3410sub patchcheck_reboot {
3411    doprint "Reboot and sleep $patchcheck_sleep_time seconds\n";
3412    reboot_to_good $patchcheck_sleep_time;
3413}
3414
3415sub patchcheck {
3416    my ($i) = @_;
3417
3418    dodie "PATCHCHECK_START[$i] not defined\n"
3419	if (!defined($patchcheck_start));
3420    dodie "PATCHCHECK_TYPE[$i] not defined\n"
3421	if (!defined($patchcheck_type));
3422
3423    my $start = $patchcheck_start;
3424
3425    my $cherry = $patchcheck_cherry;
3426    if (!defined($cherry)) {
3427	$cherry = 0;
3428    }
3429
3430    my $end = "HEAD";
3431    if (defined($patchcheck_end)) {
3432	$end = $patchcheck_end;
3433    } elsif ($cherry) {
3434	dodie "PATCHCHECK_END must be defined with PATCHCHECK_CHERRY\n";
3435    }
3436
3437    # Get the true sha1's since we can use things like HEAD~3
3438    $start = get_sha1($start);
3439    $end = get_sha1($end);
3440
3441    my $type = $patchcheck_type;
3442
3443    # Can't have a test without having a test to run
3444    if ($type eq "test" && !defined($run_test)) {
3445	$type = "boot";
3446    }
3447
3448    if ($cherry) {
3449	open (IN, "git cherry -v $start $end|") or
3450	    dodie "could not get git list";
3451    } else {
3452	open (IN, "git log --pretty=oneline $end|") or
3453	    dodie "could not get git list";
3454    }
3455
3456    my @list;
3457
3458    while (<IN>) {
3459	chomp;
3460	# git cherry adds a '+' we want to remove
3461	s/^\+ //;
3462	$list[$#list+1] = $_;
3463	last if (/^$start/);
3464    }
3465    close(IN);
3466
3467    if (!$cherry) {
3468	if ($list[$#list] !~ /^$start/) {
3469	    fail "SHA1 $start not found";
3470	}
3471
3472	# go backwards in the list
3473	@list = reverse @list;
3474    }
3475
3476    doprint("Going to test the following commits:\n");
3477    foreach my $l (@list) {
3478	doprint "$l\n";
3479    }
3480
3481    my $save_clean = $noclean;
3482    my %ignored_warnings;
3483
3484    if (defined($ignore_warnings)) {
3485	foreach my $sha1 (split /\s+/, $ignore_warnings) {
3486	    $ignored_warnings{$sha1} = 1;
3487	}
3488    }
3489
3490    $in_patchcheck = 1;
3491    foreach my $item (@list) {
3492	my $sha1 = $item;
3493	$sha1 =~ s/^([[:xdigit:]]+).*/$1/;
3494
3495	doprint "\nProcessing commit \"$item\"\n\n";
3496
3497	run_command "git checkout $sha1" or
3498	    dodie "Failed to checkout $sha1";
3499
3500	# only clean on the first and last patch
3501	if ($item eq $list[0] ||
3502	    $item eq $list[$#list]) {
3503	    $noclean = $save_clean;
3504	} else {
3505	    $noclean = 1;
3506	}
3507
3508	if (defined($minconfig)) {
3509	    build "useconfig:$minconfig" or return 0;
3510	} else {
3511	    # ?? no config to use?
3512	    build "oldconfig" or return 0;
3513	}
3514
3515	# No need to do per patch checking if warnings file exists
3516	if (!defined($warnings_file) && !defined($ignored_warnings{$sha1})) {
3517	    check_patch_buildlog $sha1 or return 0;
3518	}
3519
3520	check_buildlog or return 0;
3521
3522	next if ($type eq "build");
3523
3524	my $failed = 0;
3525
3526	start_monitor_and_install or $failed = 1;
3527
3528	if (!$failed && $type ne "boot"){
3529	    do_run_test or $failed = 1;
3530	}
3531	end_monitor;
3532	if ($failed) {
3533	    print_times;
3534	    return 0;
3535	}
3536	patchcheck_reboot;
3537	print_times;
3538    }
3539    $in_patchcheck = 0;
3540    success $i;
3541
3542    return 1;
3543}
3544
3545sub add_dep {
3546    # $config depends on $dep
3547    my ($config, $dep) = @_;
3548
3549    if (defined($depends{$config})) {
3550	$depends{$config} .= " " . $dep;
3551    } else {
3552	$depends{$config} = $dep;
3553    }
3554
3555    # record the number of configs depending on $dep
3556    if (defined $depcount{$dep}) {
3557	$depcount{$dep}++;
3558    } else {
3559	$depcount{$dep} = 1;
3560    }
3561}
3562
3563# taken from streamline_config.pl
3564sub read_kconfig {
3565    my ($kconfig) = @_;
3566
3567    my $state = "NONE";
3568    my $config;
3569    my @kconfigs;
3570
3571    my $cont = 0;
3572    my $line;
3573
3574    if (! -f $kconfig) {
3575	doprint "file $kconfig does not exist, skipping\n";
3576	return;
3577    }
3578
3579    open(KIN, "$kconfig")
3580	or dodie "Can't open $kconfig";
3581    while (<KIN>) {
3582	chomp;
3583
3584	# Make sure that lines ending with \ continue
3585	if ($cont) {
3586	    $_ = $line . " " . $_;
3587	}
3588
3589	if (s/\\$//) {
3590	    $cont = 1;
3591	    $line = $_;
3592	    next;
3593	}
3594
3595	$cont = 0;
3596
3597	# collect any Kconfig sources
3598	if (/^source\s*"(.*)"/) {
3599	    $kconfigs[$#kconfigs+1] = $1;
3600	}
3601
3602	# configs found
3603	if (/^\s*(menu)?config\s+(\S+)\s*$/) {
3604	    $state = "NEW";
3605	    $config = $2;
3606
3607	    for (my $i = 0; $i < $iflevel; $i++) {
3608		add_dep $config, $ifdeps[$i];
3609	    }
3610
3611	# collect the depends for the config
3612	} elsif ($state eq "NEW" && /^\s*depends\s+on\s+(.*)$/) {
3613
3614	    add_dep $config, $1;
3615
3616	# Get the configs that select this config
3617	} elsif ($state eq "NEW" && /^\s*select\s+(\S+)/) {
3618
3619	    # selected by depends on config
3620	    add_dep $1, $config;
3621
3622	# Check for if statements
3623	} elsif (/^if\s+(.*\S)\s*$/) {
3624	    my $deps = $1;
3625	    # remove beginning and ending non text
3626	    $deps =~ s/^[^a-zA-Z0-9_]*//;
3627	    $deps =~ s/[^a-zA-Z0-9_]*$//;
3628
3629	    my @deps = split /[^a-zA-Z0-9_]+/, $deps;
3630
3631	    $ifdeps[$iflevel++] = join ':', @deps;
3632
3633	} elsif (/^endif/) {
3634
3635	    $iflevel-- if ($iflevel);
3636
3637	# stop on "help"
3638	} elsif (/^\s*help\s*$/) {
3639	    $state = "NONE";
3640	}
3641    }
3642    close(KIN);
3643
3644    # read in any configs that were found.
3645    foreach $kconfig (@kconfigs) {
3646	if (!defined($read_kconfigs{$kconfig})) {
3647	    $read_kconfigs{$kconfig} = 1;
3648	    read_kconfig("$builddir/$kconfig");
3649	}
3650    }
3651}
3652
3653sub read_depends {
3654    # find out which arch this is by the kconfig file
3655    open (IN, $output_config) or
3656	dodie "Failed to read $output_config";
3657    my $arch;
3658    while (<IN>) {
3659	if (m,Linux/(\S+)\s+\S+\s+Kernel Configuration,) {
3660	    $arch = $1;
3661	    last;
3662	}
3663    }
3664    close IN;
3665
3666    if (!defined($arch)) {
3667	doprint "Could not find arch from config file\n";
3668	doprint "no dependencies used\n";
3669	return;
3670    }
3671
3672    # arch is really the subarch, we need to know
3673    # what directory to look at.
3674    if ($arch eq "i386" || $arch eq "x86_64") {
3675	$arch = "x86";
3676    }
3677
3678    my $kconfig = "$builddir/arch/$arch/Kconfig";
3679
3680    if (! -f $kconfig && $arch =~ /\d$/) {
3681	my $orig = $arch;
3682	# some subarchs have numbers, truncate them
3683	$arch =~ s/\d*$//;
3684	$kconfig = "$builddir/arch/$arch/Kconfig";
3685	if (! -f $kconfig) {
3686	    doprint "No idea what arch dir $orig is for\n";
3687	    doprint "no dependencies used\n";
3688	    return;
3689	}
3690    }
3691
3692    read_kconfig($kconfig);
3693}
3694
3695sub make_new_config {
3696    my @configs = @_;
3697
3698    open (OUT, ">$output_config")
3699	or dodie "Failed to write $output_config";
3700
3701    foreach my $config (@configs) {
3702	print OUT "$config\n";
3703    }
3704    close OUT;
3705}
3706
3707sub chomp_config {
3708    my ($config) = @_;
3709
3710    $config =~ s/CONFIG_//;
3711
3712    return $config;
3713}
3714
3715sub get_depends {
3716    my ($dep) = @_;
3717
3718    my $kconfig = chomp_config $dep;
3719
3720    $dep = $depends{"$kconfig"};
3721
3722    # the dep string we have saves the dependencies as they
3723    # were found, including expressions like ! && ||. We
3724    # want to split this out into just an array of configs.
3725
3726    my $valid = "A-Za-z_0-9";
3727
3728    my @configs;
3729
3730    while ($dep =~ /[$valid]/) {
3731	if ($dep =~ /^[^$valid]*([$valid]+)/) {
3732	    my $conf = "CONFIG_" . $1;
3733
3734	    $configs[$#configs + 1] = $conf;
3735
3736	    $dep =~ s/^[^$valid]*[$valid]+//;
3737	} else {
3738	    dodie "this should never happen";
3739	}
3740    }
3741
3742    return @configs;
3743}
3744
3745sub test_this_config {
3746    my ($config) = @_;
3747
3748    my $found;
3749
3750    # if we already processed this config, skip it
3751    if (defined($processed_configs{$config})) {
3752	return undef;
3753    }
3754    $processed_configs{$config} = 1;
3755
3756    # if this config failed during this round, skip it
3757    if (defined($nochange_config{$config})) {
3758	return undef;
3759    }
3760
3761    my $kconfig = chomp_config $config;
3762
3763    # Test dependencies first
3764    if (defined($depends{"$kconfig"})) {
3765	my @parents = get_depends $config;
3766	foreach my $parent (@parents) {
3767	    # if the parent is in the min config, check it first
3768	    next if (!defined($min_configs{$parent}));
3769	    $found = test_this_config($parent);
3770	    if (defined($found)) {
3771		return $found;
3772	    }
3773	}
3774    }
3775
3776    # Remove this config from the list of configs
3777    # do a make olddefconfig and then read the resulting
3778    # .config to make sure it is missing the config that
3779    # we had before
3780    my %configs = %min_configs;
3781    $configs{$config} = "# $config is not set";
3782    make_new_config ((values %configs), (values %keep_configs));
3783    make_oldconfig;
3784    delete $configs{$config};
3785    undef %configs;
3786    assign_configs \%configs, $output_config;
3787
3788    if (!defined($configs{$config}) || $configs{$config} =~ /^#/) {
3789	return $config;
3790    }
3791
3792    doprint "disabling config $config did not change .config\n";
3793
3794    $nochange_config{$config} = 1;
3795
3796    return undef;
3797}
3798
3799sub make_min_config {
3800    my ($i) = @_;
3801
3802    my $type = $minconfig_type;
3803    if ($type ne "boot" && $type ne "test") {
3804	fail "Invalid MIN_CONFIG_TYPE '$minconfig_type'\n" .
3805	    " make_min_config works only with 'boot' and 'test'\n" and return;
3806    }
3807
3808    if (!defined($output_minconfig)) {
3809	fail "OUTPUT_MIN_CONFIG not defined" and return;
3810    }
3811
3812    # If output_minconfig exists, and the start_minconfig
3813    # came from min_config, than ask if we should use
3814    # that instead.
3815    if (-f $output_minconfig && !$start_minconfig_defined) {
3816	print "$output_minconfig exists\n";
3817	if (!defined($use_output_minconfig)) {
3818	    if (read_yn " Use it as minconfig?") {
3819		$start_minconfig = $output_minconfig;
3820	    }
3821	} elsif ($use_output_minconfig > 0) {
3822	    doprint "Using $output_minconfig as MIN_CONFIG\n";
3823	    $start_minconfig = $output_minconfig;
3824	} else {
3825	    doprint "Set to still use MIN_CONFIG as starting point\n";
3826	}
3827    }
3828
3829    if (!defined($start_minconfig)) {
3830	fail "START_MIN_CONFIG or MIN_CONFIG not defined" and return;
3831    }
3832
3833    my $temp_config = "$tmpdir/temp_config";
3834
3835    # First things first. We build an allnoconfig to find
3836    # out what the defaults are that we can't touch.
3837    # Some are selections, but we really can't handle selections.
3838
3839    my $save_minconfig = $minconfig;
3840    undef $minconfig;
3841
3842    run_command "$make allnoconfig" or return 0;
3843
3844    read_depends;
3845
3846    process_config_ignore $output_config;
3847
3848    undef %save_configs;
3849    undef %min_configs;
3850
3851    if (defined($ignore_config)) {
3852	# make sure the file exists
3853	`touch $ignore_config`;
3854	assign_configs \%save_configs, $ignore_config;
3855    }
3856
3857    %keep_configs = %save_configs;
3858
3859    doprint "Load initial configs from $start_minconfig\n";
3860
3861    # Look at the current min configs, and save off all the
3862    # ones that were set via the allnoconfig
3863    assign_configs \%min_configs, $start_minconfig;
3864
3865    my @config_keys = keys %min_configs;
3866
3867    # All configs need a depcount
3868    foreach my $config (@config_keys) {
3869	my $kconfig = chomp_config $config;
3870	if (!defined $depcount{$kconfig}) {
3871	    $depcount{$kconfig} = 0;
3872	}
3873    }
3874
3875    # Remove anything that was set by the make allnoconfig
3876    # we shouldn't need them as they get set for us anyway.
3877    foreach my $config (@config_keys) {
3878	# Remove anything in the ignore_config
3879	if (defined($keep_configs{$config})) {
3880	    my $file = $ignore_config;
3881	    $file =~ s,.*/(.*?)$,$1,;
3882	    doprint "$config set by $file ... ignored\n";
3883	    delete $min_configs{$config};
3884	    next;
3885	}
3886	# But make sure the settings are the same. If a min config
3887	# sets a selection, we do not want to get rid of it if
3888	# it is not the same as what we have. Just move it into
3889	# the keep configs.
3890	if (defined($config_ignore{$config})) {
3891	    if ($config_ignore{$config} ne $min_configs{$config}) {
3892		doprint "$config is in allnoconfig as '$config_ignore{$config}'";
3893		doprint " but it is '$min_configs{$config}' in minconfig .. keeping\n";
3894		$keep_configs{$config} = $min_configs{$config};
3895	    } else {
3896		doprint "$config set by allnoconfig ... ignored\n";
3897	    }
3898	    delete $min_configs{$config};
3899	}
3900    }
3901
3902    my $done = 0;
3903    my $take_two = 0;
3904
3905    while (!$done) {
3906	my $config;
3907	my $found;
3908
3909	# Now disable each config one by one and do a make oldconfig
3910	# till we find a config that changes our list.
3911
3912	my @test_configs = keys %min_configs;
3913
3914	# Sort keys by who is most dependent on
3915	@test_configs = sort  { $depcount{chomp_config($b)} <=> $depcount{chomp_config($a)} }
3916	    @test_configs ;
3917
3918	# Put configs that did not modify the config at the end.
3919	my $reset = 1;
3920	for (my $i = 0; $i < $#test_configs; $i++) {
3921	    if (!defined($nochange_config{$test_configs[0]})) {
3922		$reset = 0;
3923		last;
3924	    }
3925	    # This config didn't change the .config last time.
3926	    # Place it at the end
3927	    my $config = shift @test_configs;
3928	    push @test_configs, $config;
3929	}
3930
3931	# if every test config has failed to modify the .config file
3932	# in the past, then reset and start over.
3933	if ($reset) {
3934	    undef %nochange_config;
3935	}
3936
3937	undef %processed_configs;
3938
3939	foreach my $config (@test_configs) {
3940
3941	    $found = test_this_config $config;
3942
3943	    last if (defined($found));
3944
3945	    # oh well, try another config
3946	}
3947
3948	if (!defined($found)) {
3949	    # we could have failed due to the nochange_config hash
3950	    # reset and try again
3951	    if (!$take_two) {
3952		undef %nochange_config;
3953		$take_two = 1;
3954		next;
3955	    }
3956	    doprint "No more configs found that we can disable\n";
3957	    $done = 1;
3958	    last;
3959	}
3960	$take_two = 0;
3961
3962	$config = $found;
3963
3964	doprint "Test with $config disabled\n";
3965
3966	# set in_bisect to keep build and monitor from dieing
3967	$in_bisect = 1;
3968
3969	my $failed = 0;
3970	build "oldconfig" or $failed = 1;
3971	if (!$failed) {
3972	    start_monitor_and_install or $failed = 1;
3973
3974	    if ($type eq "test" && !$failed) {
3975		do_run_test or $failed = 1;
3976	    }
3977
3978	    end_monitor;
3979	}
3980
3981	$in_bisect = 0;
3982
3983	if ($failed) {
3984	    doprint "$min_configs{$config} is needed to boot the box... keeping\n";
3985	    # this config is needed, add it to the ignore list.
3986	    $keep_configs{$config} = $min_configs{$config};
3987	    $save_configs{$config} = $min_configs{$config};
3988	    delete $min_configs{$config};
3989
3990	    # update new ignore configs
3991	    if (defined($ignore_config)) {
3992		open (OUT, ">$temp_config") or
3993		    dodie "Can't write to $temp_config";
3994		foreach my $config (keys %save_configs) {
3995		    print OUT "$save_configs{$config}\n";
3996		}
3997		close OUT;
3998		run_command "mv $temp_config $ignore_config" or
3999		    dodie "failed to copy update to $ignore_config";
4000	    }
4001
4002	} else {
4003	    # We booted without this config, remove it from the minconfigs.
4004	    doprint "$config is not needed, disabling\n";
4005
4006	    delete $min_configs{$config};
4007
4008	    # Also disable anything that is not enabled in this config
4009	    my %configs;
4010	    assign_configs \%configs, $output_config;
4011	    my @config_keys = keys %min_configs;
4012	    foreach my $config (@config_keys) {
4013		if (!defined($configs{$config})) {
4014		    doprint "$config is not set, disabling\n";
4015		    delete $min_configs{$config};
4016		}
4017	    }
4018
4019	    # Save off all the current mandatory configs
4020	    open (OUT, ">$temp_config") or
4021		dodie "Can't write to $temp_config";
4022	    foreach my $config (keys %keep_configs) {
4023		print OUT "$keep_configs{$config}\n";
4024	    }
4025	    foreach my $config (keys %min_configs) {
4026		print OUT "$min_configs{$config}\n";
4027	    }
4028	    close OUT;
4029
4030	    run_command "mv $temp_config $output_minconfig" or
4031		dodie "failed to copy update to $output_minconfig";
4032	}
4033
4034	doprint "Reboot and wait $sleep_time seconds\n";
4035	reboot_to_good $sleep_time;
4036    }
4037
4038    success $i;
4039    return 1;
4040}
4041
4042sub make_warnings_file {
4043    my ($i) = @_;
4044
4045    if (!defined($warnings_file)) {
4046	dodie "Must define WARNINGS_FILE for make_warnings_file test";
4047    }
4048
4049    if ($build_type eq "nobuild") {
4050	dodie "BUILD_TYPE can not be 'nobuild' for make_warnings_file test";
4051    }
4052
4053    build $build_type or dodie "Failed to build";
4054
4055    open(OUT, ">$warnings_file") or dodie "Can't create $warnings_file";
4056
4057    open(IN, $buildlog) or dodie "Can't open $buildlog";
4058    while (<IN>) {
4059	# Some compilers use UTF-8 extended for quotes
4060	# for distcc heterogeneous systems, this causes issues
4061	s/$utf8_quote/'/g;
4062
4063	if (/$check_build_re/) {
4064	    print OUT;
4065	}
4066    }
4067    close(IN);
4068
4069    close(OUT);
4070
4071    success $i;
4072}
4073
4074sub option_defined {
4075    my ($option) = @_;
4076
4077    if (defined($opt{$option}) && $opt{$option} !~ /^\s*$/) {
4078	return 1;
4079    }
4080
4081    return 0;
4082}
4083
4084sub __set_test_option {
4085    my ($name, $i) = @_;
4086
4087    my $option = "$name\[$i\]";
4088
4089    if (option_defined($option)) {
4090	return $opt{$option};
4091    }
4092
4093    foreach my $test (keys %repeat_tests) {
4094	if ($i >= $test &&
4095	    $i < $test + $repeat_tests{$test}) {
4096	    $option = "$name\[$test\]";
4097	    if (option_defined($option)) {
4098		return $opt{$option};
4099	    }
4100	}
4101    }
4102
4103    if (option_defined($name)) {
4104	return $opt{$name};
4105    }
4106
4107    return undef;
4108}
4109
4110sub set_test_option {
4111    my ($name, $i) = @_;
4112
4113    my $option = __set_test_option($name, $i);
4114    return $option if (!defined($option));
4115
4116    return eval_option($name, $option, $i);
4117}
4118
4119sub find_mailer {
4120    my ($mailer) = @_;
4121
4122    my @paths = split /:/, $ENV{PATH};
4123
4124    # sendmail is usually in /usr/sbin
4125    $paths[$#paths + 1] = "/usr/sbin";
4126
4127    foreach my $path (@paths) {
4128	if (-x "$path/$mailer") {
4129	    return $path;
4130	}
4131    }
4132
4133    return undef;
4134}
4135
4136sub do_send_mail {
4137    my ($subject, $message, $file) = @_;
4138
4139    if (!defined($mail_path)) {
4140	# find the mailer
4141	$mail_path = find_mailer $mailer;
4142	if (!defined($mail_path)) {
4143	    die "\nCan not find $mailer in PATH\n";
4144	}
4145    }
4146
4147    my $header_file = "$tmpdir/header";
4148    open (HEAD, ">$header_file") or die "Can not create $header_file\n";
4149    print HEAD "To: $mailto\n";
4150    print HEAD "Subject: $subject\n\n";
4151    print HEAD "$message\n";
4152    close HEAD;
4153
4154    if (!defined($mail_command)) {
4155	if ($mailer eq "mail" || $mailer eq "mailx") {
4156	    $mail_command = "cat \$HEADER_FILE \$BODY_FILE | \$MAIL_PATH/\$MAILER -s \'\$SUBJECT\' \$MAILTO";
4157	} elsif ($mailer eq "sendmail" ) {
4158	    $mail_command =  "cat \$HEADER_FILE \$BODY_FILE | \$MAIL_PATH/\$MAILER -t \$MAILTO";
4159	} else {
4160	    die "\nYour mailer: $mailer is not supported.\n";
4161	}
4162    }
4163
4164    if (defined($file)) {
4165	$mail_command =~ s/\$BODY_FILE/$file/g;
4166    } else {
4167	$mail_command =~ s/\$BODY_FILE//g;
4168    }
4169
4170    $mail_command =~ s/\$HEADER_FILE/$header_file/g;
4171    $mail_command =~ s/\$MAILER/$mailer/g;
4172    $mail_command =~ s/\$MAIL_PATH/$mail_path/g;
4173    $mail_command =~ s/\$MAILTO/$mailto/g;
4174    $mail_command =~ s/\$SUBJECT/$subject/g;
4175    $mail_command =~ s/\$MESSAGE/$message/g;
4176
4177    my $ret = run_command $mail_command;
4178    if (!$ret && defined($file)) {
4179	# try again without the file
4180	$message .= "\n\n*** FAILED TO SEND LOG ***\n\n";
4181	do_send_email($subject, $message);
4182    }
4183}
4184
4185sub send_email {
4186    if (defined($mailto)) {
4187	if (!defined($mailer)) {
4188	    doprint "No email sent: email or mailer not specified in config.\n";
4189	    return;
4190	}
4191	do_send_mail @_;
4192    }
4193}
4194
4195sub cancel_test {
4196    if ($email_when_canceled) {
4197	my $name = get_test_name;
4198	send_email("KTEST: Your [$name] test was cancelled",
4199	    "Your test started at $script_start_time was cancelled: sig int");
4200    }
4201    die "\nCaught Sig Int, test interrupted: $!\n"
4202}
4203
4204$#ARGV < 1 or die "ktest.pl version: $VERSION\n   usage: ktest.pl [config-file]\n";
4205
4206if ($#ARGV == 0) {
4207    $ktest_config = $ARGV[0];
4208    if (! -f $ktest_config) {
4209	print "$ktest_config does not exist.\n";
4210	if (!read_yn "Create it?") {
4211	    exit 0;
4212	}
4213    }
4214}
4215
4216if (! -f $ktest_config) {
4217    $newconfig = 1;
4218    get_test_case;
4219    open(OUT, ">$ktest_config") or die "Can not create $ktest_config";
4220    print OUT << "EOF"
4221# Generated by ktest.pl
4222#
4223
4224# PWD is a ktest.pl variable that will result in the process working
4225# directory that ktest.pl is executed in.
4226
4227# THIS_DIR is automatically assigned the PWD of the path that generated
4228# the config file. It is best to use this variable when assigning other
4229# directory paths within this directory. This allows you to easily
4230# move the test cases to other locations or to other machines.
4231#
4232THIS_DIR := $variable{"PWD"}
4233
4234# Define each test with TEST_START
4235# The config options below it will override the defaults
4236TEST_START
4237TEST_TYPE = $default{"TEST_TYPE"}
4238
4239DEFAULTS
4240EOF
4241;
4242    close(OUT);
4243}
4244read_config $ktest_config;
4245
4246if (defined($opt{"LOG_FILE"})) {
4247    $opt{"LOG_FILE"} = eval_option("LOG_FILE", $opt{"LOG_FILE"}, -1);
4248}
4249
4250# Append any configs entered in manually to the config file.
4251my @new_configs = keys %entered_configs;
4252if ($#new_configs >= 0) {
4253    print "\nAppending entered in configs to $ktest_config\n";
4254    open(OUT, ">>$ktest_config") or die "Can not append to $ktest_config";
4255    foreach my $config (@new_configs) {
4256	print OUT "$config = $entered_configs{$config}\n";
4257	$opt{$config} = process_variables($entered_configs{$config});
4258    }
4259}
4260
4261if (defined($opt{"LOG_FILE"})) {
4262    if ($opt{"CLEAR_LOG"}) {
4263	unlink $opt{"LOG_FILE"};
4264    }
4265    open(LOG, ">> $opt{LOG_FILE}") or die "Can't write to $opt{LOG_FILE}";
4266    LOG->autoflush(1);
4267}
4268
4269doprint "\n\nSTARTING AUTOMATED TESTS\n\n";
4270
4271for (my $i = 0, my $repeat = 1; $i <= $opt{"NUM_TESTS"}; $i += $repeat) {
4272
4273    if (!$i) {
4274	doprint "DEFAULT OPTIONS:\n";
4275    } else {
4276	doprint "\nTEST $i OPTIONS";
4277	if (defined($repeat_tests{$i})) {
4278	    $repeat = $repeat_tests{$i};
4279	    doprint " ITERATE $repeat";
4280	}
4281	doprint "\n";
4282    }
4283
4284    foreach my $option (sort keys %opt) {
4285	if ($option =~ /\[(\d+)\]$/) {
4286	    next if ($i != $1);
4287	} else {
4288	    next if ($i);
4289	}
4290
4291	doprint "$option = $opt{$option}\n";
4292    }
4293}
4294
4295$SIG{INT} = qw(cancel_test);
4296
4297# First we need to do is the builds
4298for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
4299
4300    # Do not reboot on failing test options
4301    $no_reboot = 1;
4302    $reboot_success = 0;
4303
4304    $have_version = 0;
4305
4306    $iteration = $i;
4307
4308    $build_time = 0;
4309    $install_time = 0;
4310    $reboot_time = 0;
4311    $test_time = 0;
4312
4313    undef %force_config;
4314
4315    my $makecmd = set_test_option("MAKE_CMD", $i);
4316
4317    $outputdir = set_test_option("OUTPUT_DIR", $i);
4318    $builddir = set_test_option("BUILD_DIR", $i);
4319
4320    chdir $builddir || dodie "can't change directory to $builddir";
4321
4322    if (!-d $outputdir) {
4323	mkpath($outputdir) or
4324	    dodie "can't create $outputdir";
4325    }
4326
4327    $make = "$makecmd O=$outputdir";
4328
4329    # Load all the options into their mapped variable names
4330    foreach my $opt (keys %option_map) {
4331	${$option_map{$opt}} = set_test_option($opt, $i);
4332    }
4333
4334    $start_minconfig_defined = 1;
4335
4336    # The first test may override the PRE_KTEST option
4337    if ($i == 1) {
4338	if (defined($pre_ktest)) {
4339	    doprint "\n";
4340	    run_command $pre_ktest;
4341	}
4342	if ($email_when_started) {
4343	    my $name = get_test_name;
4344	    send_email("KTEST: Your [$name] test was started",
4345		"Your test was started on $script_start_time");
4346	}
4347    }
4348
4349    # Any test can override the POST_KTEST option
4350    # The last test takes precedence.
4351    if (defined($post_ktest)) {
4352	$final_post_ktest = $post_ktest;
4353    }
4354
4355    if (!defined($start_minconfig)) {
4356	$start_minconfig_defined = 0;
4357	$start_minconfig = $minconfig;
4358    }
4359
4360    if (!-d $tmpdir) {
4361	mkpath($tmpdir) or
4362	    dodie "can't create $tmpdir";
4363    }
4364
4365    $ENV{"SSH_USER"} = $ssh_user;
4366    $ENV{"MACHINE"} = $machine;
4367
4368    $buildlog = "$tmpdir/buildlog-$machine";
4369    $testlog = "$tmpdir/testlog-$machine";
4370    $dmesg = "$tmpdir/dmesg-$machine";
4371    $output_config = "$outputdir/.config";
4372
4373    if (!$buildonly) {
4374	$target = "$ssh_user\@$machine";
4375	if (($reboot_type eq "grub") or ($reboot_type eq "grub2bls")) {
4376	    dodie "GRUB_MENU not defined" if (!defined($grub_menu));
4377	} elsif ($reboot_type eq "grub2") {
4378	    dodie "GRUB_MENU not defined" if (!defined($grub_menu));
4379	    dodie "GRUB_FILE not defined" if (!defined($grub_file));
4380	} elsif ($reboot_type eq "syslinux") {
4381	    dodie "SYSLINUX_LABEL not defined" if (!defined($syslinux_label));
4382	}
4383    }
4384
4385    my $run_type = $build_type;
4386    if ($test_type eq "patchcheck") {
4387	$run_type = $patchcheck_type;
4388    } elsif ($test_type eq "bisect") {
4389	$run_type = $bisect_type;
4390    } elsif ($test_type eq "config_bisect") {
4391	$run_type = $config_bisect_type;
4392    } elsif ($test_type eq "make_min_config") {
4393	$run_type = "";
4394    } elsif ($test_type eq "make_warnings_file") {
4395	$run_type = "";
4396    }
4397
4398    # mistake in config file?
4399    if (!defined($run_type)) {
4400	$run_type = "ERROR";
4401    }
4402
4403    my $installme = "";
4404    $installme = " no_install" if ($no_install);
4405
4406    my $name = "";
4407
4408    if (defined($test_name)) {
4409	$name = " ($test_name)";
4410    }
4411
4412    doprint "\n\n";
4413
4414    if (defined($opt{"LOG_FILE"})) {
4415	$test_log_start = tell(LOG);
4416    }
4417
4418    doprint "RUNNING TEST $i of $opt{NUM_TESTS}$name with option $test_type $run_type$installme\n\n";
4419
4420    if (defined($pre_test)) {
4421	my $ret = run_command $pre_test;
4422	if (!$ret && defined($pre_test_die) &&
4423	    $pre_test_die) {
4424		dodie "failed to pre_test\n";
4425	}
4426    }
4427
4428    unlink $dmesg;
4429    unlink $buildlog;
4430    unlink $testlog;
4431
4432    if (defined($addconfig)) {
4433	my $min = $minconfig;
4434	if (!defined($minconfig)) {
4435	    $min = "";
4436	}
4437	run_command "cat $addconfig $min > $tmpdir/add_config" or
4438	    dodie "Failed to create temp config";
4439	$minconfig = "$tmpdir/add_config";
4440    }
4441
4442    if (defined($checkout)) {
4443	run_command "git checkout $checkout" or
4444	    dodie "failed to checkout $checkout";
4445    }
4446
4447    $no_reboot = 0;
4448
4449    # A test may opt to not reboot the box
4450    if ($reboot_on_success) {
4451	$reboot_success = 1;
4452    }
4453
4454    if ($test_type eq "bisect") {
4455	bisect $i;
4456	next;
4457    } elsif ($test_type eq "config_bisect") {
4458	config_bisect $i;
4459	next;
4460    } elsif ($test_type eq "patchcheck") {
4461	patchcheck $i;
4462	next;
4463    } elsif ($test_type eq "make_min_config") {
4464	make_min_config $i;
4465	next;
4466    } elsif ($test_type eq "make_warnings_file") {
4467	$no_reboot = 1;
4468	make_warnings_file $i;
4469	next;
4470    }
4471
4472    if ($build_type ne "nobuild") {
4473	build $build_type or next;
4474	check_buildlog or next;
4475    }
4476
4477    if ($test_type eq "install") {
4478	get_version;
4479	install;
4480	success $i;
4481	next;
4482    }
4483
4484    if ($test_type ne "build") {
4485	my $failed = 0;
4486	start_monitor_and_install or $failed = 1;
4487
4488	if (!$failed && $test_type ne "boot" && defined($run_test)) {
4489	    do_run_test or $failed = 1;
4490	}
4491	end_monitor;
4492	if ($failed) {
4493	    print_times;
4494	    next;
4495	}
4496    }
4497
4498    print_times;
4499
4500    success $i;
4501}
4502
4503if (defined($final_post_ktest)) {
4504
4505    my $cp_final_post_ktest = eval_kernel_version $final_post_ktest;
4506    run_command $cp_final_post_ktest;
4507}
4508
4509if ($opt{"POWEROFF_ON_SUCCESS"}) {
4510    halt;
4511} elsif ($opt{"REBOOT_ON_SUCCESS"} && !do_not_reboot && $reboot_success) {
4512    reboot_to_good;
4513} elsif (defined($switch_to_good)) {
4514    # still need to get to the good kernel
4515    run_command $switch_to_good;
4516}
4517
4518doprint "\n    $successes of $opt{NUM_TESTS} tests were successful\n\n";
4519
4520if ($email_when_finished) {
4521    send_email("KTEST: Your test has finished!",
4522	"$successes of $opt{NUM_TESTS} tests started at $script_start_time were successful!");
4523}
4524
4525if (defined($opt{"LOG_FILE"})) {
4526    print "\n See $opt{LOG_FILE} for the record of results.\n\n";
4527    close LOG;
4528}
4529
4530exit 0;
4531
4532##
4533# The following are here to standardize tabs/spaces/etc across the most likely editors
4534###
4535
4536# Local Variables:
4537# mode: perl
4538# End:
4539# vim: softtabstop=4
4540