Support running multiple expected-to-pass unit tests (#4582)

This allows batching all of the tests that are expected to return status zero,
which is currently 161 tests, and running the batch with a single instance of
Wesnoth.  It doesn't include the changes to the run_wml_tests script to use
this new feature.

Timing on my PC:
* 4 seconds to run a single test on a debug build
* 90 seconds to run the whole batch of 161 on a debug build
* 1.2 seconds to run a single test on a release build
* 31.2 seconds to run the whole batch of 161 on a release build

src/commandline_options.cpp

@@ -276,7 +276,7 @@ commandline_options::commandline_options (const std::vector<std::string>& args)
 	po::options_description testing_opts("Testing options");
 	testing_opts.add_options()
 		("test,t", po::value<std::string>()->implicit_value(std::string()), "runs the game in a small test scenario. If specified, scenario <arg> will be used instead.")
-		("unit,u", po::value<std::string>()->implicit_value(std::string()), "runs a unit test scenario. Works like test, except that the exit code of the program reflects the victory / defeat conditions of the scenario.\n\t0 - PASS\n\t1 - FAIL\n\t3 - FAIL (INVALID REPLAY)\n\t4 - FAIL (ERRORED REPLAY)")
+		("unit,u", po::value<std::vector<std::string>>(), "runs a unit test scenario. The GUI is not shown and the exit code of the program reflects the victory / defeat conditions of the scenario.\n\t0 - PASS\n\t1 - FAIL\n\t3 - FAIL (INVALID REPLAY)\n\t4 - FAIL (ERRORED REPLAY)\n\tMultiple tests can be run by giving this option multiple times, in this case the test run will stop immediately after any test which doesn't PASS and the return code will be the status of the test that caused the stop.")
 		("showgui", "don't run headlessly (for debugging a failing test)")
 		("log-strict", po::value<std::string>(), "sets the strict level of the logger. any messages sent to log domains of this level or more severe will cause the unit test to fail regardless of the victory result.")
 		("noreplaycheck", "don't try to validate replay of unit test.")
@@ -495,7 +495,7 @@ commandline_options::commandline_options (const std::vector<std::string>& args)
 		test = vm["test"].as<std::string>();
 	if (vm.count("unit"))
 	{
-		unit_test = vm["unit"].as<std::string>();
+		unit_test = vm["unit"].as<std::vector<std::string>>();
 		headless_unit_test = true;
 	}
 	if (vm.count("showgui"))

src/commandline_options.hpp

@@ -186,7 +186,7 @@ public:
 	/// Non-empty if --test was given on the command line. Goes directly into test mode, into a scenario, if specified.
 	boost::optional<std::string> test;
 	/// Non-empty if --unit was given on the command line. Goes directly into unit test mode, into a scenario, if specified.
-	boost::optional<std::string> unit_test;
+	std::vector<std::string> unit_test;
 	/// True if --unit is used and --showgui is not present.
 	bool headless_unit_test;
 	/// True if --noreplaycheck was given on the command line. Dependent on --unit.

src/game_launcher.cpp

@@ -112,7 +112,7 @@ game_launcher::game_launcher(const commandline_options& cmdline_opts, const char
 	hotkey_manager_(),
 	music_thinker_(),
 	music_muter_(),
-	test_scenario_("test"),
+	test_scenarios_{"test"},
 	screenshot_map_(),
 	screenshot_filename_(),
 	state_(),
@@ -254,14 +254,11 @@ game_launcher::game_launcher(const commandline_options& cmdline_opts, const char
 	if (cmdline_opts_.test)
 	{
 		if (!cmdline_opts_.test->empty())
-			test_scenario_ = *cmdline_opts_.test;
+			test_scenarios_ = {*cmdline_opts_.test};
 	}
-	if (cmdline_opts_.unit_test)
+	if (!cmdline_opts_.unit_test.empty())
 	{
-		if (!cmdline_opts_.unit_test->empty()) {
-			test_scenario_ = *cmdline_opts_.unit_test;
-		}
-
+		test_scenarios_ = cmdline_opts_.unit_test;
 	}
 	if (cmdline_opts_.windowed)
 		video().set_fullscreen(false);
@@ -454,6 +451,9 @@ void game_launcher::set_test(const std::string& id)
 
 bool game_launcher::play_test()
 {
+	// This first_time variable was added in 70f3c80a3e2 so that using the GUI
+	// menu to load a game works. That seems to have edge-cases, for example if
+	// you try to load a game a second time then Wesnoth exits.
 	static bool first_time = true;
 
 	if(!cmdline_opts_.test) {
@@ -464,8 +464,15 @@ bool game_launcher::play_test()
 
 	first_time = false;
 
-	set_test(test_scenario_);
-	state_.classification().campaign_define = "TEST";
+	if(test_scenarios_.size() == 0) {
+		// shouldn't happen, as test_scenarios_ is initialised to {"test"}
+		std::cerr << "Error in the test handling code" << std::endl;
+		return false;
+	}
+	if(test_scenarios_.size() > 1) {
+		std::cerr << "You can't run more than one unit test in interactive mode" << std::endl;
+	}
+	set_test(test_scenarios_.at(0));
 
 	game_config_manager::get()->
 		load_game_config_for_game(state_.classification());
@@ -481,26 +488,51 @@ bool game_launcher::play_test()
 	return false;
 }
 
+/**
+ * Runs unit tests specified on the command line.
+ *
+ * If multiple unit tests were specified, then this will stop at the first test
+ * which returns a non-zero status.
+ */
 // Same as play_test except that we return the results of play_game.
-int game_launcher::unit_test()
+// \todo "same ... except" ... and many other changes, such as testing the replay
+game_launcher::unit_test_result game_launcher::unit_test()
 {
-	static bool first_time_unit = true;
-
-	if(!cmdline_opts_.unit_test) {
-		return 0;
+	// There's no copy of play_test's first_time variable. That seems to be for handling
+	// the player loading a game via the GUI, which makes no sense in a non-interactive test.
+	if(cmdline_opts_.unit_test.empty()) {
+		return unit_test_result::TEST_FAIL;
 	}
-	if(!first_time_unit)
-		return 0;
-
-	first_time_unit = false;
-
-	state_.classification().campaign_type = game_classification::CAMPAIGN_TYPE::TEST;
-	state_.classification().campaign_define = "TEST";
-	state_.set_carryover_sides_start(
-		config {"next_scenario", test_scenario_}
-	);
 
+	auto ret = unit_test_result::TEST_FAIL; // will only be returned if no test is run
+	for(const auto& scenario : test_scenarios_) {
+		set_test(scenario);
+		ret = single_unit_test();
+		const char* describe_result;
+		switch(ret) {
+			case unit_test_result::TEST_PASS:
+				describe_result = "PASS TEST";
+				break;
+			case unit_test_result::TEST_FAIL_LOADING_REPLAY:
+				describe_result = "FAIL TEST (INVALID REPLAY)";
+				break;
+			case unit_test_result::TEST_FAIL_PLAYING_REPLAY:
+				describe_result = "FAIL TEST (ERRORED REPLAY)";
+				break;
+			default:
+				describe_result = "FAIL TEST";
+				break;
+		}
+		std::cerr << describe_result << ": " << scenario << std::endl;
+		if (ret != unit_test_result::TEST_PASS) {
+			break;
+		}
+	}
+	return ret;
+}
 
+game_launcher::unit_test_result game_launcher::single_unit_test()
+{
 	game_config_manager::get()->
 		load_game_config_for_game(state_.classification());
 
@@ -508,17 +540,17 @@ int game_launcher::unit_test()
 		campaign_controller ccontroller(state_, game_config_manager::get()->terrain_types(), true);
 		LEVEL_RESULT res = ccontroller.play_game();
 		if (!(res == LEVEL_RESULT::VICTORY) || lg::broke_strict()) {
-			return 1;
+			return unit_test_result::TEST_FAIL;
 		}
 	} catch(const wml_exception& e) {
 		std::cerr << "Caught WML Exception:" << e.dev_message << std::endl;
-		return 1;
+		return unit_test_result::TEST_FAIL;
 	}
 
 	savegame::clean_saves(state_.classification().label);
 
 	if (cmdline_opts_.noreplaycheck)
-		return 0; //we passed, huzzah!
+		return unit_test_result::TEST_PASS; //we passed, huzzah!
 
 	savegame::replay_savegame save(state_, compression::NONE);
 	save.save_game_automatic(false, "unit_test_replay"); //false means don't check for overwrite
@@ -527,7 +559,7 @@ int game_launcher::unit_test()
 
 	if (!load_game()) {
 		std::cerr << "Failed to load the replay!" << std::endl;
-		return 3; //failed to load replay
+		return unit_test_result::TEST_FAIL_LOADING_REPLAY; //failed to load replay
 	}
 
 	try {
@@ -535,14 +567,14 @@ int game_launcher::unit_test()
 		LEVEL_RESULT res = ccontroller.play_replay();
 		if (!(res == LEVEL_RESULT::VICTORY) || lg::broke_strict()) {
 			std::cerr << "Observed failure on replay" << std::endl;
-			return 4;
+			return unit_test_result::TEST_FAIL_PLAYING_REPLAY;
 		}
 	} catch(const wml_exception& e) {
 		std::cerr << "WML Exception while playing replay: " << e.dev_message << std::endl;
-		return 4; //failed with an error during the replay
+		return unit_test_result::TEST_FAIL_PLAYING_REPLAY;
 	}
 
-	return 0; //we passed, huzzah!
+	return unit_test_result::TEST_PASS; //we passed, huzzah!
 }
 
 bool game_launcher::play_screenshot_mode()

src/game_launcher.hpp

@@ -57,6 +57,17 @@ public:
 
 	enum mp_selection {MP_CONNECT, MP_HOST, MP_LOCAL};
 
+	/**
+	 * Status code after running a unit test, should match the run_wml_tests
+	 * script and the documentation for the --unit_test command-line option.
+	 */
+	enum class unit_test_result : int {
+		TEST_PASS = 0,
+		TEST_FAIL = 1,
+		TEST_FAIL_LOADING_REPLAY = 3,
+		TEST_FAIL_PLAYING_REPLAY = 4,
+	};
+
 	bool init_video();
 	bool init_language();
 	bool init_lua_script();
@@ -64,7 +75,8 @@ public:
 	bool play_test();
 	bool play_screenshot_mode();
 	bool play_render_image_mode();
-	int unit_test();
+	/// Runs unit tests specified on the command line
+	unit_test_result unit_test();
 
 	bool is_loading() const;
 	void clear_loaded_game();
@@ -105,6 +117,10 @@ private:
 
 	editor::EXIT_STATUS start_editor(const std::string& filename);
 
+	/// Internal to the implementation of unit_test(). If a single instance of
+	/// Wesnoth is running multiple unit tests, this gets called once per test.
+	unit_test_result single_unit_test();
+
 	const commandline_options& cmdline_opts_;
 	//Never null.
 	const std::unique_ptr<CVideo> video_;
@@ -117,7 +133,7 @@ private:
 	sound::music_thinker music_thinker_;
 	sound::music_muter music_muter_;
 
-	std::string test_scenario_;
+	std::vector<std::string> test_scenarios_;
 
 	std::string screenshot_map_, screenshot_filename_;
 

src/wesnoth.cpp

@@ -827,13 +827,8 @@ static int do_gameloop(const std::vector<std::string>& args)
 		plugins.play_slice();
 		plugins.play_slice();
 
-		if(cmdline_opts.unit_test) {
-			int worker_result = game->unit_test();
-			std::cerr << ((worker_result == 0) ? "PASS TEST " : "FAIL TEST ")
-					  << ((worker_result == 3) ? "(INVALID REPLAY)" : "")
-					  << ((worker_result == 4) ? "(ERRORED REPLAY)" : "") << ": " << *cmdline_opts.unit_test
-					  << std::endl;
-			return worker_result;
+		if(!cmdline_opts.unit_test.empty()) {
+			return static_cast<int>(game->unit_test());
 		}
 
 		if(game->play_test() == false) {