Line data    Source code

       1             : // SPDX-FileCopyrightText: 2024 Pairinteraction Developers
       2             : // SPDX-License-Identifier: LGPL-3.0-or-later
       3             : 
       4             : #define DOCTEST_CONFIG_IMPLEMENT
       5             : 
       6             : #include "pairinteraction/tools/run_unit_tests.hpp"
       7             : 
       8             : #include "pairinteraction/database/Database.hpp"
       9             : #include "pairinteraction/utils/paths.hpp"
      10             : #include "pairinteraction/utils/streamed.hpp"
      11             : #include "pairinteraction/version.hpp"
      12             : 
      13             : #include <cstdlib>
      14             : #include <doctest/doctest.h>
      15             : #include <filesystem>
      16             : #include <httplib.h>
      17             : #include <mutex>
      18             : #include <spdlog/spdlog.h>
      19             : 
      20             : // Create a reporter for doctest that logs to spdlog
      21             : namespace doctest {
      22             : 
      23             : // The code of the LoggingReporter is based on the ConsoleReporter from doctest,
      24             : // https://github.com/doctest/doctest/blob/ae7a13539fb71f270b87eb2e874fbac80bc8dda2/doctest/parts/doctest.cpp#L2868.
      25             : //
      26             : // SPDX-SnippetBegin
      27             : // SPDX-FileCopyrightText: (c) 2016-2025 Viktor Kirilov, Sebastian Weber
      28             : // SPDX-License-Identifier: MIT
      29             : 
      30             : // NOLINTBEGIN(cppcoreguidelines-macro-usage)
      31             : #define DOCTEST_LOCK_MUTEX(name)                                                                   \
      32             :     std::lock_guard<std::mutex> DOCTEST_ANONYMOUS(DOCTEST_ANON_LOCK_)(name);
      33             : // NOLINTEND(cppcoreguidelines-macro-usage)
      34             : 
      35             : struct LoggingReporter : public ConsoleReporter {
      36           1 :     LoggingReporter(const ContextOptions &co) : ConsoleReporter(co) {}
      37             : 
      38         121 :     void log_contexts() {}
      39             : 
      40         156 :     void logTestStart() {}
      41             : 
      42           1 :     void test_run_end(const TestRunStats &p) override {
      43           1 :         if (opt.minimal && p.numTestCasesFailed == 0) {
      44           0 :             return;
      45             :         }
      46             : 
      47           1 :         std::stringstream ss;
      48           1 :         ss << Color::Yellow
      49           1 :            << "==============================================================================="
      50           1 :            << Color::None << "\n";
      51           1 :         ss << std::dec;
      52             : 
      53             :         auto totwidth =
      54           2 :             int(std::ceil(log10(static_cast<double>(std::max(p.numTestCasesPassingFilters,
      55           1 :                                                              static_cast<unsigned>(p.numAsserts))) +
      56           1 :                                 1)));
      57             :         auto passwidth =
      58           2 :             int(std::ceil(log10(static_cast<double>(std::max(
      59           2 :                                     p.numTestCasesPassingFilters - p.numTestCasesFailed,
      60           1 :                                     static_cast<unsigned>(p.numAsserts - p.numAssertsFailed))) +
      61           1 :                                 1)));
      62             :         auto failwidth = int(
      63           2 :             std::ceil(log10(static_cast<double>(std::max(
      64           1 :                                 p.numTestCasesFailed, static_cast<unsigned>(p.numAssertsFailed))) +
      65           1 :                             1)));
      66           1 :         const bool anythingFailed = p.numTestCasesFailed > 0 || p.numAssertsFailed > 0;
      67           1 :         ss << "test cases: " << std::setw(totwidth) << p.numTestCasesPassingFilters << " | "
      68           1 :            << ((p.numTestCasesPassingFilters == 0 || anythingFailed) ? Color::None : Color::Green)
      69           1 :            << std::setw(passwidth) << p.numTestCasesPassingFilters - p.numTestCasesFailed
      70           1 :            << " passed" << Color::None << " | "
      71           1 :            << (p.numTestCasesFailed > 0 ? Color::Red : Color::None) << std::setw(failwidth)
      72           1 :            << p.numTestCasesFailed << " failed" << Color::None << " |";
      73           1 :         if (!opt.no_skipped_summary) {
      74           1 :             const unsigned int numSkipped = p.numTestCases - p.numTestCasesPassingFilters;
      75           1 :             ss << " " << (numSkipped == 0 ? Color::None : Color::Yellow) << numSkipped << " skipped"
      76           1 :                << Color::None;
      77             :         }
      78           1 :         ss << "\n";
      79           1 :         ss << "assertions: " << std::setw(totwidth) << p.numAsserts << " | "
      80           1 :            << ((p.numAsserts == 0 || anythingFailed) ? Color::None : Color::Green)
      81           1 :            << std::setw(passwidth) << (p.numAsserts - p.numAssertsFailed) << " passed"
      82           1 :            << Color::None << " | " << (p.numAssertsFailed > 0 ? Color::Red : Color::None)
      83           1 :            << std::setw(failwidth) << p.numAssertsFailed << " failed" << Color::None << " |\n";
      84           1 :         ss << "Status: " << (p.numTestCasesFailed > 0 ? Color::Red : Color::Green)
      85           1 :            << ((p.numTestCasesFailed > 0) ? "FAILURE!" : "SUCCESS!") << Color::None << std::endl;
      86             : 
      87           1 :         if (p.numTestCasesFailed > 0) {
      88           0 :             for (std::string line; std::getline(ss, line);) {
      89           0 :                 SPDLOG_ERROR(line);
      90           0 :             }
      91             :         } else {
      92           5 :             for (std::string line; std::getline(ss, line);) {
      93           4 :                 SPDLOG_INFO(line);
      94           1 :             }
      95             :         }
      96           1 :     }
      97             : 
      98          35 :     void test_case_end(const CurrentTestCaseStats &st) override {
      99          35 :         if (tc->m_no_output) {
     100           0 :             return;
     101             :         }
     102             : 
     103          35 :         if (opt.duration ||
     104           0 :             (st.failure_flags != 0 &&
     105           0 :              st.failure_flags != static_cast<int>(TestCaseFailureReason::AssertFailure))) {
     106          35 :             logTestStart();
     107             :         }
     108             : 
     109          35 :         if (opt.duration) {
     110          35 :             std::stringstream ss;
     111          35 :             ss << std::setprecision(6) << std::fixed << st.seconds << " s: " << tc->m_name;
     112          35 :             SPDLOG_INFO(ss.str());
     113          35 :         }
     114             : 
     115          35 :         if ((st.failure_flags & TestCaseFailureReason::Timeout) != 0) {
     116           0 :             std::stringstream ss;
     117           0 :             ss << Color::Red << "Test case exceeded time limit of " << std::setprecision(6)
     118           0 :                << std::fixed << tc->m_timeout << "!" << Color::None;
     119           0 :             SPDLOG_ERROR(ss.str());
     120           0 :         }
     121             : 
     122          35 :         if ((st.failure_flags & TestCaseFailureReason::ShouldHaveFailedButDidnt) != 0) {
     123           0 :             std::stringstream ss;
     124           0 :             ss << Color::Red << "Should have failed but didn't! Marking it as failed!"
     125           0 :                << Color::None;
     126           0 :             SPDLOG_ERROR(ss.str());
     127          35 :         } else if ((st.failure_flags & TestCaseFailureReason::ShouldHaveFailedAndDid) != 0) {
     128           0 :             std::stringstream ss;
     129           0 :             ss << Color::Yellow << "Failed as expected so marking it as not failed" << Color::None;
     130           0 :             SPDLOG_WARN(ss.str());
     131          35 :         } else if ((st.failure_flags & TestCaseFailureReason::CouldHaveFailedAndDid) != 0) {
     132           0 :             std::stringstream ss;
     133           0 :             ss << Color::Yellow << "Allowed to fail so marking it as not failed" << Color::None;
     134           0 :             SPDLOG_WARN(ss.str());
     135          35 :         } else if ((st.failure_flags & TestCaseFailureReason::DidntFailExactlyNumTimes) != 0) {
     136           0 :             std::stringstream ss;
     137           0 :             ss << Color::Red << "Didn't fail exactly " << tc->m_expected_failures
     138           0 :                << " times so marking it as failed!" << Color::None;
     139           0 :             SPDLOG_ERROR(ss.str());
     140          35 :         } else if ((st.failure_flags & TestCaseFailureReason::FailedExactlyNumTimes) != 0) {
     141           0 :             std::stringstream ss;
     142           0 :             ss << Color::Yellow << "Failed exactly " << tc->m_expected_failures
     143           0 :                << " times as expected so marking it as not failed!" << Color::None;
     144           0 :             SPDLOG_WARN(ss.str());
     145           0 :         }
     146             : 
     147          35 :         if ((st.failure_flags & TestCaseFailureReason::TooManyFailedAsserts) != 0) {
     148           0 :             std::stringstream ss;
     149           0 :             ss << Color::Red << "Aborting - too many failed asserts!" << Color::None;
     150           0 :             SPDLOG_ERROR(ss.str());
     151           0 :         }
     152             :     }
     153             : 
     154           0 :     void test_case_exception(const TestCaseException &e) override {
     155           0 :         if (tc->m_no_output) {
     156           0 :             return;
     157             :         }
     158             : 
     159           0 :         DOCTEST_LOCK_MUTEX(mutex)
     160             : 
     161           0 :         logTestStart();
     162             : 
     163           0 :         std::stringstream ss;
     164           0 :         ss << "[" << skipPathFromFilename(tc->m_file.c_str()) << (opt.gnu_file_line ? ":" : "(")
     165           0 :            << (opt.no_line_numbers ? 0 : tc->m_line) << (opt.gnu_file_line ? "" : ")") << "] ";
     166           0 :         std::string loc = ss.str();
     167           0 :         ss.str("");
     168           0 :         ss << Color::Red << (e.is_crash ? "test case CRASHED: " : "test case THREW exception: ")
     169           0 :            << Color::None << e.error_string;
     170           0 :         for (std::string line; std::getline(ss, line);) {
     171           0 :             SPDLOG_ERROR(loc + line);
     172           0 :         }
     173           0 :     }
     174             : 
     175        1670 :     void log_assert(const AssertData &rb) override {
     176        1670 :         if ((!rb.m_failed && !opt.success) || tc->m_no_output) {
     177        1670 :             return;
     178             :         }
     179             : 
     180           0 :         DOCTEST_LOCK_MUTEX(mutex)
     181             : 
     182           0 :         logTestStart();
     183             : 
     184           0 :         std::stringstream ss;
     185           0 :         ss << "[" << skipPathFromFilename(rb.m_file) << (opt.gnu_file_line ? ":" : "(")
     186           0 :            << (opt.no_line_numbers ? 0 : rb.m_line) << (opt.gnu_file_line ? "" : ")") << "] ";
     187           0 :         std::string loc = ss.str();
     188           0 :         ss.str("");
     189           0 :         fulltext_log_assert_to_stream(ss, rb);
     190           0 :         if (rb.m_failed) {
     191           0 :             for (std::string line; std::getline(ss, line);) {
     192           0 :                 SPDLOG_ERROR(loc + line);
     193           0 :             }
     194             :         } else {
     195           0 :             for (std::string line; std::getline(ss, line);) {
     196           0 :                 SPDLOG_INFO(loc + line);
     197           0 :             }
     198             :         }
     199             : 
     200           0 :         log_contexts();
     201           0 :     }
     202             : 
     203         121 :     void log_message(const MessageData &mb) override {
     204         121 :         if (tc->m_no_output) {
     205           0 :             return;
     206             :         }
     207             : 
     208         121 :         DOCTEST_LOCK_MUTEX(mutex)
     209             : 
     210         121 :         logTestStart();
     211             : 
     212         121 :         std::stringstream ss;
     213         121 :         ss << "[" << skipPathFromFilename(mb.m_file) << (opt.gnu_file_line ? ":" : "(")
     214         121 :            << (opt.no_line_numbers ? 0 : mb.m_line) << (opt.gnu_file_line ? "" : ")") << "] ";
     215         121 :         std::string loc = ss.str();
     216         121 :         ss.str("");
     217         121 :         ss << getSuccessOrFailColor(false, mb.m_severity)
     218         121 :            << getSuccessOrFailString((mb.m_severity & assertType::is_warn) != 0, mb.m_severity,
     219             :                                      "MESSAGE")
     220         121 :            << ": " << Color::None << mb.m_string;
     221         242 :         for (std::string line; std::getline(ss, line);) {
     222         121 :             SPDLOG_INFO(loc + line);
     223         121 :         }
     224             : 
     225         121 :         log_contexts();
     226         121 :     }
     227             : };
     228             : 
     229             : // SPDX-SnippetEnd
     230             : 
     231             : REGISTER_REPORTER("logging", 1, doctest::LoggingReporter);
     232             : } // namespace doctest
     233             : 
     234             : constexpr std::string_view OS_NAME =
     235             : #if defined(_WIN32)
     236             :     "Windows";
     237             : #elif defined(__APPLE__)
     238             :     "macOS";
     239             : #elif defined(__linux__)
     240             :     "Linux";
     241             : #else
     242             :     "Unknown";
     243             : #endif
     244             : 
     245             : namespace pairinteraction {
     246           1 : int run_unit_tests(int argc, char **argv, bool download_missing, bool use_cache,
     247             :                    std::filesystem::path database_dir) {
     248             : 
     249             :     // Setup the tests
     250           1 :     doctest::Context ctx;
     251           1 :     ctx.setOption("abort-after", 5);
     252           1 :     ctx.setOption("no-run", 0);
     253           1 :     ctx.setOption("duration", true);
     254           1 :     ctx.setOption("no-path-filenames", true);
     255           1 :     ctx.applyCommandLine(argc, argv);
     256           1 :     ctx.setOption("no-colors", true);
     257           1 :     ctx.setOption("no-breaks", true);
     258           1 :     ctx.setOption("reporters", "logging");
     259           1 :     ctx.setOption("no-intro", true);
     260             : 
     261             :     // Log the version and system information
     262           2 :     SPDLOG_INFO("Version of pairinteraction: {}.{}.{}", VERSION_MAJOR, VERSION_MINOR,
     263             :                 VERSION_PATCH);
     264           2 :     SPDLOG_INFO("Operating system: {}", OS_NAME);
     265             : 
     266             :     // Create a global database instance and run the tests
     267           1 :     Database::get_global_instance(download_missing, use_cache, std::move(database_dir));
     268           1 :     int exitcode = ctx.run();
     269             : 
     270           1 :     std::filesystem::path logdir = paths::get_cache_directory() / "logs";
     271           2 :     SPDLOG_INFO("The log was stored to {}", logdir.string());
     272             : 
     273           1 :     if (exitcode != 0) {
     274           0 :         if (download_missing) {
     275           0 :             httplib::Client client("https://www.github.com");
     276           0 :             auto res = client.Head("/");
     277           0 :             if (!res) {
     278           0 :                 SPDLOG_ERROR(
     279             :                     "Test failed. Please check your internet connection. An internet "
     280             :                     "connection is required to download databases of atomic states and matrix "
     281             :                     "elements if they are not available locally. The log was stored to {}",
     282             :                     logdir.string());
     283             :             } else {
     284           0 :                 SPDLOG_ERROR(
     285             :                     "Tests failed. Consider creating an issue on "
     286             :                     "https://github.com/pairinteraction/pairinteraction/issues, attaching the "
     287             :                     "log. The log was stored to {}",
     288             :                     logdir.string());
     289             :             }
     290           0 :         } else {
     291           0 :             SPDLOG_ERROR(
     292             :                 "Tests failed. Consider creating an issue on "
     293             :                 "https://github.com/pairinteraction/pairinteraction/issues, attaching the "
     294             :                 "log. If the tests failed because of unavailable states or "
     295             :                 "matrix elements, consider downloading missing databases by calling "
     296             :                 "the test function with 'download_missing = true'. The log was stored to {}",
     297             :                 logdir.string());
     298             :         }
     299             :     }
     300             : 
     301           1 :     return exitcode;
     302           1 : };
     303             : } // namespace pairinteraction