Line data    Source code

       1             : // SPDX-FileCopyrightText: 2024 PairInteraction Developers
       2             : // SPDX-License-Identifier: LGPL-3.0-or-later
       3             : 
       4             : #define DOCTEST_CONFIG_IMPLEMENT
       5             : 
       6             : #include "pairinteraction/tools/run_unit_tests.hpp"
       7             : 
       8             : #include "pairinteraction/database/Database.hpp"
       9             : #include "pairinteraction/utils/paths.hpp"
      10             : #include "pairinteraction/utils/streamed.hpp"
      11             : #include "pairinteraction/version.hpp"
      12             : 
      13             : #include <cstdlib>
      14             : #include <doctest/doctest.h>
      15             : #include <filesystem>
      16             : #include <httplib.h>
      17             : #include <mutex>
      18             : #include <spdlog/spdlog.h>
      19             : 
      20             : // Create a reporter for doctest that logs to spdlog
      21             : namespace doctest {
      22             : 
      23             : // The code of the LoggingReporter is based on the ConsoleReporter from doctest,
      24             : // https://github.com/doctest/doctest/blob/ae7a13539fb71f270b87eb2e874fbac80bc8dda2/doctest/parts/doctest.cpp#L2868.
      25             : //
      26             : // SPDX-SnippetBegin
      27             : // SPDX-FileCopyrightText: (c) 2016-2025 Viktor Kirilov, Sebastian Weber
      28             : // SPDX-License-Identifier: MIT
      29             : 
      30             : struct LoggingReporter : public ConsoleReporter {
      31           1 :     LoggingReporter(const ContextOptions &co) : ConsoleReporter(co) {}
      32             : 
      33         121 :     void log_contexts() {}
      34             : 
      35         157 :     void logTestStart() {}
      36             : 
      37           1 :     void test_run_end(const TestRunStats &p) override {
      38           1 :         if (opt.minimal && p.numTestCasesFailed == 0) {
      39           0 :             return;
      40             :         }
      41             : 
      42           1 :         std::stringstream ss;
      43           1 :         ss << Color::Yellow
      44           1 :            << "==============================================================================="
      45           1 :            << Color::None << "\n";
      46           1 :         ss << std::dec;
      47             : 
      48             :         auto totwidth =
      49           2 :             int(std::ceil(log10(static_cast<double>(std::max(p.numTestCasesPassingFilters,
      50           1 :                                                              static_cast<unsigned>(p.numAsserts))) +
      51           1 :                                 1)));
      52             :         auto passwidth =
      53           2 :             int(std::ceil(log10(static_cast<double>(std::max(
      54           2 :                                     p.numTestCasesPassingFilters - p.numTestCasesFailed,
      55           1 :                                     static_cast<unsigned>(p.numAsserts - p.numAssertsFailed))) +
      56           1 :                                 1)));
      57             :         auto failwidth = int(
      58           2 :             std::ceil(log10(static_cast<double>(std::max(
      59           1 :                                 p.numTestCasesFailed, static_cast<unsigned>(p.numAssertsFailed))) +
      60           1 :                             1)));
      61           1 :         const bool anythingFailed = p.numTestCasesFailed > 0 || p.numAssertsFailed > 0;
      62           1 :         ss << "test cases: " << std::setw(totwidth) << p.numTestCasesPassingFilters << " | "
      63           1 :            << ((p.numTestCasesPassingFilters == 0 || anythingFailed) ? Color::None : Color::Green)
      64           1 :            << std::setw(passwidth) << p.numTestCasesPassingFilters - p.numTestCasesFailed
      65           1 :            << " passed" << Color::None << " | "
      66           1 :            << (p.numTestCasesFailed > 0 ? Color::Red : Color::None) << std::setw(failwidth)
      67           1 :            << p.numTestCasesFailed << " failed" << Color::None << " |";
      68           1 :         if (!opt.no_skipped_summary) {
      69           1 :             const unsigned int numSkipped = p.numTestCases - p.numTestCasesPassingFilters;
      70           1 :             ss << " " << (numSkipped == 0 ? Color::None : Color::Yellow) << numSkipped << " skipped"
      71           1 :                << Color::None;
      72             :         }
      73           1 :         ss << "\n";
      74           1 :         ss << "assertions: " << std::setw(totwidth) << p.numAsserts << " | "
      75           1 :            << ((p.numAsserts == 0 || anythingFailed) ? Color::None : Color::Green)
      76           1 :            << std::setw(passwidth) << (p.numAsserts - p.numAssertsFailed) << " passed"
      77           1 :            << Color::None << " | " << (p.numAssertsFailed > 0 ? Color::Red : Color::None)
      78           1 :            << std::setw(failwidth) << p.numAssertsFailed << " failed" << Color::None << " |\n";
      79           1 :         ss << "Status: " << (p.numTestCasesFailed > 0 ? Color::Red : Color::Green)
      80           1 :            << ((p.numTestCasesFailed > 0) ? "FAILURE!" : "SUCCESS!") << Color::None << std::endl;
      81             : 
      82           1 :         if (p.numTestCasesFailed > 0) {
      83           0 :             for (std::string line; std::getline(ss, line);) {
      84           0 :                 SPDLOG_ERROR(line);
      85           0 :             }
      86             :         } else {
      87           5 :             for (std::string line; std::getline(ss, line);) {
      88           4 :                 SPDLOG_INFO(line);
      89           1 :             }
      90             :         }
      91           1 :     }
      92             : 
      93          36 :     void test_case_end(const CurrentTestCaseStats &st) override {
      94          36 :         if (tc->m_no_output) {
      95           0 :             return;
      96             :         }
      97             : 
      98          36 :         if (opt.duration ||
      99           0 :             (st.failure_flags != 0 &&
     100           0 :              st.failure_flags != static_cast<int>(TestCaseFailureReason::AssertFailure))) {
     101          36 :             logTestStart();
     102             :         }
     103             : 
     104          36 :         if (opt.duration) {
     105          36 :             std::stringstream ss;
     106          36 :             ss << std::setprecision(6) << std::fixed << st.seconds << " s: " << tc->m_name;
     107          36 :             SPDLOG_INFO(ss.str());
     108          36 :         }
     109             : 
     110          36 :         if ((st.failure_flags & TestCaseFailureReason::Timeout) != 0) {
     111           0 :             std::stringstream ss;
     112           0 :             ss << Color::Red << "Test case exceeded time limit of " << std::setprecision(6)
     113           0 :                << std::fixed << tc->m_timeout << "!" << Color::None;
     114           0 :             SPDLOG_ERROR(ss.str());
     115           0 :         }
     116             : 
     117          36 :         if ((st.failure_flags & TestCaseFailureReason::ShouldHaveFailedButDidnt) != 0) {
     118           0 :             std::stringstream ss;
     119           0 :             ss << Color::Red << "Should have failed but didn't! Marking it as failed!"
     120           0 :                << Color::None;
     121           0 :             SPDLOG_ERROR(ss.str());
     122          36 :         } else if ((st.failure_flags & TestCaseFailureReason::ShouldHaveFailedAndDid) != 0) {
     123           0 :             std::stringstream ss;
     124           0 :             ss << Color::Yellow << "Failed as expected so marking it as not failed" << Color::None;
     125           0 :             SPDLOG_WARN(ss.str());
     126          36 :         } else if ((st.failure_flags & TestCaseFailureReason::CouldHaveFailedAndDid) != 0) {
     127           0 :             std::stringstream ss;
     128           0 :             ss << Color::Yellow << "Allowed to fail so marking it as not failed" << Color::None;
     129           0 :             SPDLOG_WARN(ss.str());
     130          36 :         } else if ((st.failure_flags & TestCaseFailureReason::DidntFailExactlyNumTimes) != 0) {
     131           0 :             std::stringstream ss;
     132           0 :             ss << Color::Red << "Didn't fail exactly " << tc->m_expected_failures
     133           0 :                << " times so marking it as failed!" << Color::None;
     134           0 :             SPDLOG_ERROR(ss.str());
     135          36 :         } else if ((st.failure_flags & TestCaseFailureReason::FailedExactlyNumTimes) != 0) {
     136           0 :             std::stringstream ss;
     137           0 :             ss << Color::Yellow << "Failed exactly " << tc->m_expected_failures
     138           0 :                << " times as expected so marking it as not failed!" << Color::None;
     139           0 :             SPDLOG_WARN(ss.str());
     140           0 :         }
     141             : 
     142          36 :         if ((st.failure_flags & TestCaseFailureReason::TooManyFailedAsserts) != 0) {
     143           0 :             std::stringstream ss;
     144           0 :             ss << Color::Red << "Aborting - too many failed asserts!" << Color::None;
     145           0 :             SPDLOG_ERROR(ss.str());
     146           0 :         }
     147             :     }
     148             : 
     149           0 :     void test_case_exception(const TestCaseException &e) override {
     150           0 :         if (tc->m_no_output) {
     151           0 :             return;
     152             :         }
     153             : 
     154           0 :         DOCTEST_LOCK_MUTEX(mutex)
     155             : 
     156           0 :         logTestStart();
     157             : 
     158           0 :         std::stringstream ss;
     159           0 :         ss << "[" << skipPathFromFilename(tc->m_file.c_str()) << (opt.gnu_file_line ? ":" : "(")
     160           0 :            << (opt.no_line_numbers ? 0 : tc->m_line) << (opt.gnu_file_line ? "" : ")") << "] ";
     161           0 :         std::string loc = ss.str();
     162           0 :         ss.str("");
     163           0 :         ss << Color::Red << (e.is_crash ? "test case CRASHED: " : "test case THREW exception: ")
     164           0 :            << Color::None << e.error_string;
     165           0 :         for (std::string line; std::getline(ss, line);) {
     166           0 :             SPDLOG_ERROR(loc + line);
     167           0 :         }
     168           0 :     }
     169             : 
     170        1686 :     void log_assert(const AssertData &rb) override {
     171        1686 :         if ((!rb.m_failed && !opt.success) || tc->m_no_output) {
     172        1686 :             return;
     173             :         }
     174             : 
     175           0 :         DOCTEST_LOCK_MUTEX(mutex)
     176             : 
     177           0 :         logTestStart();
     178             : 
     179           0 :         std::stringstream ss;
     180           0 :         ss << "[" << skipPathFromFilename(rb.m_file) << (opt.gnu_file_line ? ":" : "(")
     181           0 :            << (opt.no_line_numbers ? 0 : rb.m_line) << (opt.gnu_file_line ? "" : ")") << "] ";
     182           0 :         std::string loc = ss.str();
     183           0 :         ss.str("");
     184           0 :         fulltext_log_assert_to_stream(ss, rb);
     185           0 :         if (rb.m_failed) {
     186           0 :             for (std::string line; std::getline(ss, line);) {
     187           0 :                 SPDLOG_ERROR(loc + line);
     188           0 :             }
     189             :         } else {
     190           0 :             for (std::string line; std::getline(ss, line);) {
     191           0 :                 SPDLOG_INFO(loc + line);
     192           0 :             }
     193             :         }
     194             : 
     195           0 :         log_contexts();
     196           0 :     }
     197             : 
     198         121 :     void log_message(const MessageData &mb) override {
     199         121 :         if (tc->m_no_output) {
     200           0 :             return;
     201             :         }
     202             : 
     203         121 :         DOCTEST_LOCK_MUTEX(mutex)
     204             : 
     205         121 :         logTestStart();
     206             : 
     207         121 :         std::stringstream ss;
     208         121 :         ss << "[" << skipPathFromFilename(mb.m_file) << (opt.gnu_file_line ? ":" : "(")
     209         121 :            << (opt.no_line_numbers ? 0 : mb.m_line) << (opt.gnu_file_line ? "" : ")") << "] ";
     210         121 :         std::string loc = ss.str();
     211         121 :         ss.str("");
     212         121 :         ss << getSuccessOrFailColor(false, mb.m_severity)
     213         121 :            << getSuccessOrFailString((mb.m_severity & assertType::is_warn) != 0, mb.m_severity,
     214             :                                      "MESSAGE")
     215         121 :            << ": " << Color::None << mb.m_string;
     216         242 :         for (std::string line; std::getline(ss, line);) {
     217         121 :             SPDLOG_INFO(loc + line);
     218         121 :         }
     219             : 
     220         121 :         log_contexts();
     221         121 :     }
     222             : };
     223             : 
     224             : // SPDX-SnippetEnd
     225             : 
     226             : REGISTER_REPORTER("logging", 1, doctest::LoggingReporter);
     227             : } // namespace doctest
     228             : 
     229             : constexpr std::string_view OS_NAME =
     230             : #if defined(_WIN32)
     231             :     "Windows";
     232             : #elif defined(__APPLE__)
     233             :     "macOS";
     234             : #elif defined(__linux__)
     235             :     "Linux";
     236             : #else
     237             :     "Unknown";
     238             : #endif
     239             : 
     240             : namespace pairinteraction {
     241           1 : int run_unit_tests(int argc, char **argv, bool download_missing, bool use_cache,
     242             :                    std::filesystem::path database_dir) {
     243             : 
     244             :     // Setup the tests
     245           1 :     doctest::Context ctx;
     246           1 :     ctx.setOption("abort-after", 5);
     247           1 :     ctx.setOption("no-run", 0);
     248           1 :     ctx.setOption("duration", true);
     249           1 :     ctx.setOption("no-path-filenames", true);
     250           1 :     ctx.applyCommandLine(argc, argv);
     251           1 :     ctx.setOption("no-colors", true);
     252           1 :     ctx.setOption("no-breaks", true);
     253           1 :     ctx.setOption("reporters", "logging");
     254           1 :     ctx.setOption("no-intro", true);
     255             : 
     256             :     // Log the version and system information
     257           2 :     SPDLOG_INFO("Version of pairinteraction: {}.{}.{}", VERSION_MAJOR, VERSION_MINOR,
     258             :                 VERSION_PATCH);
     259           2 :     SPDLOG_INFO("Operating system: {}", OS_NAME);
     260             : 
     261             :     // Create a global database instance and run the tests
     262           1 :     Database::get_global_instance(download_missing, use_cache, std::move(database_dir));
     263           1 :     int exitcode = ctx.run();
     264             : 
     265           1 :     std::filesystem::path logdir = paths::get_cache_directory() / "logs";
     266           2 :     SPDLOG_INFO("The log was stored to {}", logdir.string());
     267             : 
     268           1 :     if (exitcode != 0) {
     269           0 :         if (download_missing) {
     270           0 :             httplib::Client client("https://www.github.com");
     271           0 :             auto res = client.Head("/");
     272           0 :             if (!res) {
     273           0 :                 SPDLOG_ERROR(
     274             :                     "Test failed. Please check your internet connection. An internet "
     275             :                     "connection is required to download databases of atomic states and matrix "
     276             :                     "elements if they are not available locally. The log was stored to {}",
     277             :                     logdir.string());
     278             :             } else {
     279           0 :                 SPDLOG_ERROR(
     280             :                     "Tests failed. Consider creating an issue on "
     281             :                     "https://github.com/pairinteraction/pairinteraction/issues, attaching the "
     282             :                     "log. The log was stored to {}",
     283             :                     logdir.string());
     284             :             }
     285           0 :         } else {
     286           0 :             SPDLOG_ERROR(
     287             :                 "Tests failed. Consider creating an issue on "
     288             :                 "https://github.com/pairinteraction/pairinteraction/issues, attaching the "
     289             :                 "log. If the tests failed because of unavailable states or "
     290             :                 "matrix elements, consider downloading missing databases by calling "
     291             :                 "the test function with 'download_missing = true'. The log was stored to {}",
     292             :                 logdir.string());
     293             :         }
     294             :     }
     295             : 
     296           1 :     return exitcode;
     297           1 : };
     298             : } // namespace pairinteraction