diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8857347..5ff9500 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,7 +43,7 @@ repos: hooks: - id: clang-format name: clang-format (C/C++) - files: \.(c|cpp|cc|cxx|h|hpp|hh)$ + files: \.(c|cpp|cc|cxx|cu|h|hpp|hh)$ - repo: https://github.com/pre-commit/pre-commit-hooks rev: v6.0.0 hooks: diff --git a/CMakeLists.txt b/CMakeLists.txt index a78169d..d470998 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,6 +42,7 @@ option(LIBDIFFPY_TEST_INSTALLED "Build only the unit tests and link them with the installed libdiffpy" OFF) set(LIBDIFFPY_TESTS "" CACHE STRING "Comma or space separated patterns selecting the Test*.cpp unit test files") +option(LIBDIFFPY_CUDA "Build the CUDA calculation kernels" OFF) option(LIBDIFFPY_COVERAGE "Instrument the build for coverage analysis" OFF) option(LIBDIFFPY_PROFILE "Build with gprof profiling information" OFF) @@ -78,10 +79,13 @@ if(PROJECT_IS_TOP_LEVEL) if(MSVC) # /W3 rather than /Wall, which floods the log with warnings from the # standard library and boost headers. - add_compile_options(/W3) + add_compile_options($<$:/W3>) add_link_options(/OPT:NOREF /OPT:NOICF) else() - add_compile_options(-Wall $<$:-ffast-math>) + # for the C++ sources, nvcc does not take these options + add_compile_options( + $<$:-Wall> + $<$,$>:-ffast-math>) endif() endif() @@ -91,12 +95,12 @@ if(MSVC AND (LIBDIFFPY_COVERAGE OR LIBDIFFPY_PROFILE)) endif() if(LIBDIFFPY_COVERAGE) - add_compile_options(--coverage -O0 -g) + add_compile_options("$<$:--coverage;-fprofile-update=atomic;-O0;-g>") add_link_options(--coverage) endif() if(LIBDIFFPY_PROFILE) - add_compile_options(-pg) + add_compile_options($<$:-pg>) add_link_options(-pg) endif() @@ -247,6 +251,11 @@ if(LIBDIFFPY_HAS_OBJCRYST) else() set(DIFFPY_HAS_OBJCRYST 0) endif() +if(LIBDIFFPY_CUDA) + set(DIFFPY_HAS_CUDA 1) +else() + set(DIFFPY_HAS_CUDA 0) +endif() set(_features_header "${_generated_include_dir}/diffpy/features.hpp") configure_file("${PROJECT_SOURCE_DIR}/src/diffpy/features.tpl" "${_features_header}") @@ -343,11 +352,19 @@ if(LIBDIFFPY_HAS_OBJCRYST) target_link_libraries(diffpy PUBLIC ObjCryst::ObjCryst) endif() include(CalculationKernels) +if(LIBDIFFPY_CUDA) + include(Cuda) +endif() # Installation ---------------------------------------------------------------- if(LIBDIFFPY_INSTALL) set(LIBDIFFPY_CMAKE_INSTALL_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/libdiffpy") + if(LIBDIFFPY_CUDA AND NOT _libdiffpy_library_type STREQUAL "SHARED_LIBRARY") + # A static libdiffpy leaves the CUDA kernels to the program that links it. + install(TARGETS diffpy_cuda EXPORT libdiffpyTargets + ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}" COMPONENT Development) + endif() install(TARGETS diffpy EXPORT libdiffpyTargets RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}" COMPONENT Runtime diff --git a/CMakePresets.json b/CMakePresets.json index 3dd54d3..0873f53 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -17,6 +17,23 @@ "CMAKE_GENERATOR_TOOLSET": "" } }, + { + "name": "debug", + "inherits": "ninja", + "displayName": "Debug build with C assertions and unit tests", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug", + "LIBDIFFPY_BUILD_TESTS": "ON" + } + }, + { + "name": "debug-cuda", + "inherits": "debug", + "displayName": "Debug build with CUDA kernels and unit tests", + "cacheVariables": { + "LIBDIFFPY_CUDA": "ON" + } + }, { "name": "release", "inherits": "ninja", @@ -26,35 +43,42 @@ } }, { - "name": "debug", - "inherits": "ninja", - "displayName": "Debug build with C assertions and unit tests", + "name": "release-cuda", + "inherits": "release", + "displayName": "Optimized build with CUDA kernels", "cacheVariables": { - "CMAKE_BUILD_TYPE": "Debug", - "LIBDIFFPY_BUILD_TESTS": "ON" + "LIBDIFFPY_CUDA": "ON" } }, { "name": "coverage", - "inherits": "ninja", + "inherits": "debug", "displayName": "Debug build instrumented for coverage analysis", "cacheVariables": { - "CMAKE_BUILD_TYPE": "Debug", - "LIBDIFFPY_BUILD_TESTS": "ON", "LIBDIFFPY_COVERAGE": "ON" } } ], "buildPresets": [ + { + "name": "debug", + "configurePreset": "debug", + "configuration": "Debug" + }, + { + "name": "debug-cuda", + "configurePreset": "debug-cuda", + "configuration": "Debug" + }, { "name": "release", "configurePreset": "release", "configuration": "Release" }, { - "name": "debug", - "configurePreset": "debug", - "configuration": "Debug" + "name": "release-cuda", + "configurePreset": "release-cuda", + "configuration": "Release" }, { "name": "coverage", @@ -74,16 +98,26 @@ "outputOnFailure": true } }, + { + "name": "debug-cuda", + "inherits": "debug", + "configurePreset": "debug-cuda" + }, + { + "name": "release", + "inherits": "debug", + "configurePreset": "release", + "configuration": "Release" + }, + { + "name": "release-cuda", + "inherits": "release", + "configurePreset": "release-cuda" + }, { "name": "coverage", - "configurePreset": "coverage", - "configuration": "Debug", - "execution": { - "noTestsAction": "error" - }, - "output": { - "outputOnFailure": true - } + "inherits": "debug", + "configurePreset": "coverage" } ] } diff --git a/cmake/CalculationKernels.cmake b/cmake/CalculationKernels.cmake index 2987911..a7d5a40 100644 --- a/cmake/CalculationKernels.cmake +++ b/cmake/CalculationKernels.cmake @@ -40,9 +40,13 @@ endif() # The main source glob is not recursive, add the kernels explicitly. set(_kernel_dir "${PROJECT_SOURCE_DIR}/src/diffpy/srreal/kernels") +find_package(Threads REQUIRED) +target_link_libraries(diffpy PRIVATE Threads::Threads) target_sources(diffpy PRIVATE + "${_kernel_dir}/CalculationKernelCPU.cpp" "${_kernel_dir}/CalculationKernelScalar.cpp" - "${_kernel_dir}/CalculationKernelDispatch.cpp") + "${_kernel_dir}/CalculationKernelDispatch.cpp" + "${_kernel_dir}/PairTable.cpp") set(_kernel_names scalar) # All kernels need IEEE arithmetic to give consistent results. @@ -51,7 +55,10 @@ if(MSVC) else() set(_kernel_math_options -fno-fast-math -ffp-contract=off) endif() -set_source_files_properties("${_kernel_dir}/CalculationKernelScalar.cpp" +set_source_files_properties( + "${_kernel_dir}/CalculationKernelCPU.cpp" + "${_kernel_dir}/CalculationKernelScalar.cpp" + "${_kernel_dir}/PairTable.cpp" PROPERTIES COMPILE_OPTIONS "${_kernel_math_options}") # Inline functions compiled for the SIMD instruction sets must not be shared diff --git a/cmake/Cuda.cmake b/cmake/Cuda.cmake new file mode 100644 index 0000000..84e18b6 --- /dev/null +++ b/cmake/Cuda.cmake @@ -0,0 +1,37 @@ +# CUDA calculation kernels, see +# src/diffpy/srreal/kernels/CalculationKernelDispatch.cpp. +# +# The CUDA source makes the static library diffpy_cuda, which the diffpy +# library links. It is compiled by nvcc and statically linked with its +# own C++ standard and without the options of the C++ sources. + +include(CheckLanguage) +check_language(CUDA) +if(NOT CMAKE_CUDA_COMPILER) + message(FATAL_ERROR + "LIBDIFFPY_CUDA needs the CUDA compiler nvcc. Set CMAKE_CUDA_COMPILER " + "or the environment variable CUDACXX to it.") +endif() +enable_language(CUDA) + +add_library(diffpy_cuda STATIC + "${PROJECT_SOURCE_DIR}/src/diffpy/srreal/kernels/CalculationKernelCuda.cu") +set_target_properties(diffpy_cuda PROPERTIES + CUDA_STANDARD 17 + CUDA_STANDARD_REQUIRED ON + CUDA_EXTENSIONS OFF + CUDA_RUNTIME_LIBRARY Static + POSITION_INDEPENDENT_CODE ON + # The objects of nvcc have no data for link-time optimization. + INTERPROCEDURAL_OPTIMIZATION OFF) +target_include_directories(diffpy_cuda PRIVATE "${PROJECT_SOURCE_DIR}/src") +# Use the CPU kernels' rounding order for distances, widths and grid cutoffs. +target_compile_options(diffpy_cuda PRIVATE + $<$:--fmad=false> + $<$:-ffp-contract=off>) +# The runtime is named here for the programs that link diffpy_cuda through +# a static or OBJECT diffpy library, whose link step is not one of CUDA. +find_package(CUDAToolkit REQUIRED) +target_link_libraries(diffpy_cuda PRIVATE CUDA::cudart_static) +target_link_libraries(diffpy PRIVATE diffpy_cuda) +target_compile_definitions(diffpy PRIVATE LIBDIFFPY_KERNEL_CUDA=1) diff --git a/cmake/libdiffpyConfig.cmake.in b/cmake/libdiffpyConfig.cmake.in index 11700d6..16d4319 100644 --- a/cmake/libdiffpyConfig.cmake.in +++ b/cmake/libdiffpyConfig.cmake.in @@ -8,6 +8,7 @@ # Variables: # # libdiffpy_HAS_OBJCRYST true when libdiffpy was built with ObjCryst +# libdiffpy_HAS_CUDA true when libdiffpy has the CUDA kernels # libdiffpy_RUNTIME_DIR directory of the installed data files @PACKAGE_INIT@ @@ -15,11 +16,16 @@ include(CMakeFindDependencyMacro) set(libdiffpy_HAS_OBJCRYST @LIBDIFFPY_HAS_OBJCRYST@) +set(libdiffpy_HAS_CUDA @LIBDIFFPY_CUDA@) set_and_check(libdiffpy_RUNTIME_DIR "@PACKAGE_LIBDIFFPY_RUNTIME_INSTALL_DIR@") find_dependency(Boost CONFIG COMPONENTS serialization) if(NOT "@_libdiffpy_library_type@" STREQUAL "SHARED_LIBRARY") find_dependency(GSL) + find_dependency(Threads) + if(libdiffpy_HAS_CUDA) + find_dependency(CUDAToolkit) + endif() endif() if(libdiffpy_HAS_OBJCRYST) list(PREPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}") diff --git a/docs/source/development.rst b/docs/source/development.rst index 606b214..18781a8 100644 --- a/docs/source/development.rst +++ b/docs/source/development.rst @@ -32,8 +32,10 @@ The CMake presets require CMake 3.22 or later and use Ninja. On Windows, run them in a command prompt with the conda build environment activated. The build presets are -* ``release`` - optimized build with C assertion checks disabled * ``debug`` - unoptimized build with debug information and C assertion checks +* ``debug-cuda`` - debug build with CUDA calculation kernels +* ``release`` - optimized build with C assertion checks disabled +* ``release-cuda`` - optimized build with CUDA calculation kernels * ``coverage`` - unoptimized build instrumented for coverage analysis The library is built in the ``build/`` directory, for example @@ -48,8 +50,8 @@ ObjCryst adapter, pass ``-DLIBDIFFPY_ENABLE_OBJCRYST=OFF``. For gprof instrumentation, use ``-DLIBDIFFPY_PROFILE=ON``. To build the optimized shared library without link-time optimization, pass ``-DLIBDIFFPY_LTO=OFF``. -Calculation kernels -------------------- +SIMD calculation kernels +------------------------ ``PDFCalculator`` and ``DebyePDFCalculator`` run their inner loops in a calculation kernel. The scalar kernel is always built. With the default @@ -61,6 +63,26 @@ The SIMD kernels are ``avx512``, ``avx2`` and ``sse2`` on x86-64 and ``neon`` on arm64. On first use the library picks the best kernel that the CPU supports, for example ``avx2`` on a CPU without AVX-512. +CUDA calculation kernels +------------------------ + +Build and test with the ``debug-cuda`` preset :: + + cmake --preset debug-cuda + cmake --build --preset debug-cuda --parallel 4 + ctest --preset debug-cuda + +Use ``release-cuda`` for an optimized build. To run the tests with either +release preset, pass ``-DLIBDIFFPY_BUILD_TESTS=ON`` when configuring, then +build and run CTest with the same preset. +To select ``nvcc`` or its host C++ compiler, set ``CUDACXX`` or +``CUDAHOSTCXX``, respectively, before the first configure. + +On a compatible GPU, ``availableCalculationKernels`` lists ``cuda`` +(double precision) and ``cuda-mixed`` (single-precision sampling with +double-precision distances and sums). Without a compatible GPU, CPU +kernels remain available and the ``[cuda]`` tests skip. + Embed libdiffpy in another CMake project ---------------------------------------- diff --git a/news/cuda.rst b/news/cuda.rst new file mode 100644 index 0000000..de3358d --- /dev/null +++ b/news/cuda.rst @@ -0,0 +1,40 @@ +**Added:** + +* Build with ``LIBDIFFPY_CUDA=ON`` or either CUDA preset (``debug-cuda``, + ``release-cuda``) to enable the ``cuda`` and ``cuda-mixed`` kernels for full + PDF and Debye sums. Query build support with ``libdiffpy_HAS_CUDA`` (CMake) + or ``DIFFPY_HAS_CUDA`` (C++). + CUDA-enabled builds also run on CPU-only machines. +* Derived calculators can implement ``addAllPairContributions`` or reuse + ``addAllPairsOfPDF`` and ``addAllPairsOfDebyeSum``. The constant, Debye-Waller + and Jeong peak width models now provide ``calculateFromMSD``. + +**Changed:** + +* ``PDFCalculator`` and ``DebyePDFCalculator`` accelerate full sums with + batched pairs and multiple CPU threads, up to the hardware concurrency + reported by the system. Changing the summation order can affect rounding. +* ``OPTIMIZED`` can choose a full calculation when it costs less than updating + changed sites. ``getEvaluatorTypeUsed`` then returns ``BASIC``. +* ``PDFCalculator::getQmax`` and ``getQstep`` now return ``double`` by value. + +**Deprecated:** + +* + +**Removed:** + +* + +**Fixed:** + +* Separate calculators can run concurrently and share a read-only structure. +* ``DebyePDFCalculator`` uses each site's own occupancy when several sites + share an element. +* The ``OPTIMIZED`` evaluator recalculates when a change of adapter class + changes the structure, for example when a cluster becomes periodic. +* Copies of a calculator now have independent evaluators. + +**Security:** + +* diff --git a/src/diffpy/EventTicker.cpp b/src/diffpy/EventTicker.cpp index 68f3e70..cde56c8 100644 --- a/src/diffpy/EventTicker.cpp +++ b/src/diffpy/EventTicker.cpp @@ -20,6 +20,8 @@ * *****************************************************************************/ +#include + #include #include @@ -36,14 +38,7 @@ EventTicker::EventTicker() : mtick(0, 0) {} // Public Methods ------------------------------------------------------------ -void EventTicker::click() { - ++gtick.second; - if (0 >= gtick.second) { - ++gtick.first; - gtick.second = 0; - } - mtick = gtick; -} +void EventTicker::click() { mtick = advanceGlobalTick(); } void EventTicker::updateFrom(const EventTicker& other) { if (other > *this) mtick = other.mtick; @@ -75,9 +70,31 @@ bool EventTicker::operator!=(const EventTicker& other) const { return mtick != other.mtick; } -// Static Global Data -------------------------------------------------------- +// Private Static Methods ---------------------------------------------------- + +namespace { + +// Tickers click also while other files initialize their static objects, +// so the counter and its mutex must be initialized at compile time. +constinit std::mutex gtickmutex; +constinit EventTicker::value_type gtick(0, 0); -EventTicker::value_type EventTicker::gtick(0, 0); +} // namespace + +EventTicker::value_type EventTicker::advanceGlobalTick() { + const std::lock_guard lock(gtickmutex); + ++gtick.second; + if (0 >= gtick.second) { + ++gtick.first; + gtick.second = 0; + } + return gtick; +} + +EventTicker::value_type EventTicker::globalTick() { + const std::lock_guard lock(gtickmutex); + return gtick; +} } // namespace eventticker } // namespace diffpy diff --git a/src/diffpy/EventTicker.hpp b/src/diffpy/EventTicker.hpp index 67d7e15..82fadb3 100644 --- a/src/diffpy/EventTicker.hpp +++ b/src/diffpy/EventTicker.hpp @@ -41,6 +41,7 @@ class DLL_EXPORT EventTicker { // methods /// record an event and advance the global ticker. + /// Tickers of different threads may click at the same time. void click(); /// advance internal ticker if the other ticker is newer. void updateFrom(const EventTicker& other); @@ -56,8 +57,10 @@ class DLL_EXPORT EventTicker { bool operator!=(const EventTicker&) const; private: - // global counter - static value_type gtick; + /// Advance the global counter and return its new value. + static value_type advanceGlobalTick(); + /// Return the current value of the global counter. + static value_type globalTick(); // data value_type mtick; @@ -66,6 +69,7 @@ class DLL_EXPORT EventTicker { friend class boost::serialization::access; template void save(Archive& ar, const unsigned int version) const { + const value_type gtick = globalTick(); ar << mtick << gtick; } @@ -73,6 +77,7 @@ class DLL_EXPORT EventTicker { void load(Archive& ar, const unsigned int version) { value_type ga; ar >> mtick >> ga; + const value_type gtick = globalTick(); if (ga > gtick) { if (ga.first != gtick.first) mtick.first = mtick.second = 0; diff --git a/src/diffpy/HasClassRegistry.ipp b/src/diffpy/HasClassRegistry.ipp index f289873..47847f7 100644 --- a/src/diffpy/HasClassRegistry.ipp +++ b/src/diffpy/HasClassRegistry.ipp @@ -163,12 +163,8 @@ template typename HasClassRegistry::RegistryStorage& HasClassRegistry::getRegistry() { - static std::unique_ptr the_registry; - if (!the_registry.get()) - { - the_registry.reset(new RegistryStorage()); - } - return *the_registry; + static RegistryStorage the_registry; + return the_registry; } } // namespace diffpy diff --git a/src/diffpy/features.tpl b/src/diffpy/features.tpl index 526bdf5..73d4fcf 100644 --- a/src/diffpy/features.tpl +++ b/src/diffpy/features.tpl @@ -25,6 +25,10 @@ # define DIFFPY_HAS_OBJCRYST #endif +#if ${DIFFPY_HAS_CUDA} +# define DIFFPY_HAS_CUDA +#endif + #endif // FEATURES_HPP_INCLUDED // vim:ft=cpp: diff --git a/src/diffpy/srreal/AtomicStructureAdapter.cpp b/src/diffpy/srreal/AtomicStructureAdapter.cpp index dec63d4..37134fd 100644 --- a/src/diffpy/srreal/AtomicStructureAdapter.cpp +++ b/src/diffpy/srreal/AtomicStructureAdapter.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include @@ -135,6 +136,18 @@ StructureDifference AtomicStructureAdapter::diff( typedef std::shared_ptr APtr; APtr pother = std::dynamic_pointer_cast(other); if (!pother) return sd; + // The same atoms can be another structure in a derived adapter, such as a + // crystal. Only its diff can tell, so it must allow the fast update too. + if (typeid(*this) != typeid(*pother)) { + thread_local bool reversed = false; + if (!reversed) { + struct Reversal { + Reversal() { reversed = true; } + ~Reversal() { reversed = false; } + } reversal; + if (!other->diff(this->shared_from_this()).allowsfastupdate()) return sd; + } + } // try fast side-by-side comparison sd.diffmethod = StructureDifference::Method::SIDEBYSIDE; const AtomicStructureAdapter& astru0 = *this; diff --git a/src/diffpy/srreal/AtomicStructureAdapter.hpp b/src/diffpy/srreal/AtomicStructureAdapter.hpp index 9402792..46719b0 100644 --- a/src/diffpy/srreal/AtomicStructureAdapter.hpp +++ b/src/diffpy/srreal/AtomicStructureAdapter.hpp @@ -20,6 +20,8 @@ #ifndef ATOMICSTRUCTUREADAPTER_HPP_INCLUDED #define ATOMICSTRUCTUREADAPTER_HPP_INCLUDED +#include + #include #include @@ -115,6 +117,7 @@ class DLL_EXPORT AtomicStructureAdapter : public StructureAdapter { void assign(Iter first, Iter last) { matoms.assign(first, last); } + void assign(AtomVector&& atoms) noexcept { matoms = std::move(atoms); } void assign(size_t n, const Atom& a) { matoms.assign(n, a); } // iterator forwarding iterator begin() { return matoms.begin(); } diff --git a/src/diffpy/srreal/BaseBondGenerator.cpp b/src/diffpy/srreal/BaseBondGenerator.cpp index 919dc64..cedd5f1 100644 --- a/src/diffpy/srreal/BaseBondGenerator.cpp +++ b/src/diffpy/srreal/BaseBondGenerator.cpp @@ -185,7 +185,8 @@ void BaseBondGenerator::updateDistance() { // Private Methods ----------------------------------------------------------- void BaseBondGenerator::advanceWhileInvalid() { - while (!this->finished() && (this->bondOutOfRange() || this->atSelfPair())) { + while (!this->finished() && + (this->bondOutOfRange() || (!mkeepcoincident && this->atSelfPair()))) { this->getNextBond(); } } diff --git a/src/diffpy/srreal/BaseBondGenerator.hpp b/src/diffpy/srreal/BaseBondGenerator.hpp index 62080af..7d20433 100644 --- a/src/diffpy/srreal/BaseBondGenerator.hpp +++ b/src/diffpy/srreal/BaseBondGenerator.hpp @@ -28,6 +28,10 @@ namespace diffpy { namespace srreal { +namespace kernels { +class PairTable; +} + /// Use zero default for rmax so any misconfiguration of r-limits is obvious. const double DEFAULT_BONDGENERATOR_RMAX = 0.0; @@ -94,6 +98,10 @@ class DLL_EXPORT BaseBondGenerator { void updateDistance(); private: + friend class kernels::PairTable; + // A cached candidate may move away from a coincident site on the next call. + bool mkeepcoincident = false; + // methods void advanceWhileInvalid(); bool bondOutOfRange() const; diff --git a/src/diffpy/srreal/BaseDebyeSum.cpp b/src/diffpy/srreal/BaseDebyeSum.cpp index ea32084..7e027aa 100644 --- a/src/diffpy/srreal/BaseDebyeSum.cpp +++ b/src/diffpy/srreal/BaseDebyeSum.cpp @@ -16,14 +16,19 @@ * *****************************************************************************/ +#include +#include +#include #include +#include +#include +#include #include -#include -#include -#include #include -#include +#include +#include +#include #include #include #include @@ -43,6 +48,11 @@ namespace { /// Default cutoff for the Q-decreasing scale of the sine contributions. const double DEFAULT_DEBYE_PRECISION = 1e-6; +std::span factorRow(const QuantityType& factors, int row, + int nq) { + return std::span(factors).subspan(std::size_t(row) * nq, nq); +} + } // namespace // Constructor --------------------------------------------------------------- @@ -139,17 +149,12 @@ void BaseDebyeSum::addPairContribution(const BaseBondGenerator& bnds, int summationscale) { const double dist = bnds.distance(); if (eps_eq(0.0, dist)) return; - // calculate sigma parameter for the Debye-Waller dampign Gaussian const double fwhm = this->getPeakWidthModel()->calculate(bnds); - const double fwhmtosigma = 1.0 / (2 * sqrt(2 * M_LN2)); const int nqpts = pdfutils_qmaxSteps(this); - const auto& sftypeatkq = mstructure_cache.sftypeatkq; - const auto& typeofsite = mstructure_cache.typeofsite; - const auto& sf0 = sftypeatkq[typeofsite[bnds.site0()]]; - const auto& sf1 = sftypeatkq[typeofsite[bnds.site1()]]; - assert(nqpts <= int(mvalue.size())); - assert(nqpts <= int(sf0.size())); - assert(nqpts <= int(sf1.size())); + const auto& cache = mstructure_cache; + const int t0 = cache.types.typeofsite[bnds.site0()]; + const int t1 = cache.types.typeofsite[bnds.site1()]; + if (!nqpts) return; kernels::activeCalculationKernel().debyeSum({ .out = mvalue.data(), @@ -157,14 +162,48 @@ void BaseDebyeSum::addPairContribution(const BaseBondGenerator& bnds, .kqhi = nqpts, .qstep = this->getQstep(), .dist = dist, - .dwsigma = fwhmtosigma * fwhm, + .dwsigma = kernels::fwhmToSigma(fwhm), .scale = double(summationscale * bnds.multiplicity()), .prec = this->getDebyePrecision(), - .sf0 = sf0.data(), - .sf1 = sf1.data(), + .sf0 = factorRow(cache.factors, cache.types.curve[t0], nqpts).data(), + .sf1 = factorRow(cache.factors, cache.types.curve[t1], nqpts).data(), + .occ0 = cache.types.occupancy[t0], + .occ1 = cache.types.occupancy[t1], }); } +bool BaseDebyeSum::addAllPairContributions() { + if (typeid(*this) != typeid(BaseDebyeSum)) return false; + return this->addAllPairsOfDebyeSum(); +} + +bool BaseDebyeSum::addAllPairsOfDebyeSum() { + // Other peak width models and structures need per-bond evaluation. + kernels::DebyePairSumArgs args = {}; + if (!kernels::assignPairWidth(args.w, *this->getPeakWidthModel()) || + !kernels::PairTable::supports(*mstructure)) + return false; + BaseBondGenerator range(mstructure); + this->configureBondGenerator(range); + const auto& cache = mstructure_cache; + const int nq = cache.nq; + args.typeofsite = cache.types.typeofsite.data(); + args.curve = cache.types.curve.data(); + args.occupancy = cache.types.occupancy.data(); + args.ntypes = int(cache.types.curve.size()); + args.ncurves = nq ? int(cache.factors.size() / nq) : 0; + args.nq = nq; + args.sf = cache.factors.data(); + args.kqlo = pdfutils_qminSteps(this); + args.kqhi = nq; + args.qstep = this->getQstep(); + args.prec = this->getDebyePrecision(); + const auto& kernel = + this->preparePairSum(args, range.getRmin(), range.getRmax()); + mallpairs.record(kernel.debyePairSum(args), kernel.onDevice); + return true; +} + void BaseDebyeSum::stashPartialValue() { mdbsumstash = this->value(); } void BaseDebyeSum::restorePartialValue() { @@ -185,58 +224,135 @@ double BaseDebyeSum::sfAverageAtkQ(int kq) const { } void BaseDebyeSum::cacheStructureData() { - using std::placeholders::_1; - int cntsites = this->countSites(); - const int nqpts = pdfutils_qmaxSteps(this); - QuantityType zeros(nqpts, 0.0); - unordered_map atomtypeidx; - // sftypeatkq - mstructure_cache.typeofsite.clear(); - mstructure_cache.typeofsite.reserve(cntsites); - mstructure_cache.sftypeatkq.clear(); - for (int siteidx = 0; siteidx < cntsites; ++siteidx) { - const string& smbl = mstructure->siteAtomType(siteidx); - if (!atomtypeidx.count(smbl)) { - atomtypeidx.insert(make_pair(smbl, int(atomtypeidx.size()))); + auto& c = mstructure_cache; + const int nq = pdfutils_qmaxSteps(this); + SiteTypes sites; + sites.typeofsite.resize(this->countSites()); + if (typeid(*this) == typeid(DebyePDFCalculator) || + typeid(*this) == typeid(BaseDebyeSum)) { + this->cacheElementFactors(sites, nq); + } else { + this->cacheCustomFactors(sites, nq); + } + c.types = std::move(sites); + c.nq = nq; + c.totaloccupancy = mstructure->totalOccupancy(); + c.sfaverageatkq.assign(c.nq, 0.0); + std::vector multiplicity(c.types.curve.size(), 0); + for (int i = 0; i < this->countSites(); ++i) { + multiplicity[c.types.typeofsite[i]] += mstructure->siteMultiplicity(i); + } + const int lo = pdfutils_qminSteps(this); + for (size_t t = 0; t < multiplicity.size(); ++t) { + const auto row = factorRow(c.factors, c.types.curve[t], c.nq); + for (int k = lo; k < c.nq; ++k) { + c.sfaverageatkq[k] += (row[k] * c.types.occupancy[t]) * multiplicity[t]; } - int tpidx = atomtypeidx[smbl]; - mstructure_cache.typeofsite.push_back(tpidx); - // do nothing if the type has been already cached - if (tpidx < int(mstructure_cache.sftypeatkq.size())) continue; - assert(tpidx == int(mstructure_cache.sftypeatkq.size())); - // here we need to build a new array - mstructure_cache.sftypeatkq.push_back(zeros); - QuantityType& sfarray = mstructure_cache.sftypeatkq.back(); - for (int kq = pdfutils_qminSteps(this); kq < nqpts; ++kq) { - double q = this->getQstep() * kq; - sfarray[kq] = this->sfSiteAtQ(siteidx, q); + } + if (eps_gt(c.totaloccupancy, 0.0)) { + const double scale = 1.0 / c.totaloccupancy; + for (double& f : c.sfaverageatkq) f *= scale; + } +} + +void BaseDebyeSum::cacheElementFactors(SiteTypes& sites, int nq) { + auto& c = mstructure_cache; + std::map types; + std::map elements; + const auto* pdf = dynamic_cast(this); + for (int i = 0; i < this->countSites(); ++i) { + const auto& symbol = mstructure->siteAtomType(i); + const double occ = pdf ? mstructure->siteOccupancy(i) : 1.0; + auto [t, added] = + types.try_emplace({symbol, bit_cast(occ)}, int(types.size())); + sites.typeofsite[i] = t->second; + if (added) { + auto el = elements.try_emplace(symbol, int(elements.size())).first; + sites.curve.push_back(el->second); + sites.occupancy.push_back(occ); } } - assert(cntsites == int(mstructure_cache.typeofsite.size())); - assert(atomtypeidx.size() == mstructure_cache.sftypeatkq.size()); - // totaloccupancy - mstructure_cache.totaloccupancy = mstructure->totalOccupancy(); - // sfaverageatkq - QuantityType& sfak = mstructure_cache.sfaverageatkq; - sfak = zeros; - int ntps = mstructure_cache.sftypeatkq.size(); - vector tpmultipl(ntps, 0); - for (int siteidx = 0; siteidx < cntsites; ++siteidx) { - int tpidx = mstructure_cache.typeofsite[siteidx]; - assert(tpidx < ntps); - tpmultipl[tpidx] += mstructure->siteMultiplicity(siteidx); + const auto tick = this->ticker(); + if (elements == melements && tick == msfticker && + c.factors.size() == elements.size() * nq) + return; + QuantityType factors(elements.size() * nq); + const int lo = pdfutils_qminSteps(this); + for (const auto& [symbol, row] : elements) { + for (int k = lo; k < nq; ++k) { + factors[size_t(row) * nq + k] = + pdf ? pdf->getScatteringFactorTable()->lookup(symbol, + k * this->getQstep()) + : 1.0; + } } - for (int tpidx = 0; tpidx < ntps; ++tpidx) { - QuantityType& sfarray = mstructure_cache.sftypeatkq[tpidx]; - const int multipl = tpmultipl[tpidx]; - for (int kq = pdfutils_qminSteps(this); kq < nqpts; ++kq) { - sfak[kq] += sfarray[kq] * multipl; + c.factors.swap(factors); + melements.swap(elements); + msfticker = tick; +} + +void BaseDebyeSum::cacheCustomFactors(SiteTypes& sites, int nq) { + auto& c = mstructure_cache; + std::map types; + QuantityType factors, row(nq); + const bool comparePrevious = nq == c.nq && this->ticker() == msfticker; + const int lo = pdfutils_qminSteps(this); + bool changed = mcustomAmbiguous; + bool ambiguous = false; + for (int i = 0; i < this->countSites(); ++i) { + SiteKey key{mstructure->siteAtomType(i), + bit_cast(mstructure->siteOccupancy(i))}; + for (int k = lo; k < nq; ++k) + row[k] = this->sfSiteAtQ(i, k * this->getQstep()); + auto old = mcustomTypes.find(key); + if (comparePrevious && old != mcustomTypes.end() && + !std::ranges::equal(row, factorRow(c.factors, old->second, nq))) + changed = true; + const int next = int(sites.curve.size()); + auto [t, added] = types.try_emplace(std::move(key), next); + int id = t->second; + if (!added && !std::ranges::equal(row, factorRow(factors, id, nq))) { + id = next; + ambiguous = true; + } + if (id == next) { + factors.insert(factors.end(), row.begin(), row.end()); + sites.curve.push_back(id); + sites.occupancy.push_back(1.0); } + sites.typeofsite[i] = id; } - const double& totocc = mstructure_cache.totaloccupancy; - const double tosc = eps_gt(totocc, 0.0) ? (1.0 / totocc) : 1.0; - transform(sfak.begin(), sfak.end(), sfak.begin(), - bind(multiplies(), tosc, _1)); + if (changed || ambiguous) mticker.click(); + msfticker = this->ticker(); + mcustomAmbiguous = ambiguous; + mcustomTypes.swap(types); + c.factors.swap(factors); +} + +std::vector BaseDebyeSum::archiveFactors() const { + const auto& c = mstructure_cache; + std::vector rows(c.types.curve.size(), QuantityType(c.nq)); + for (size_t t = 0; t < rows.size(); ++t) { + const auto row = factorRow(c.factors, c.types.curve[t], c.nq); + std::ranges::transform(row, rows[t].begin(), + [&](double f) { return f * c.types.occupancy[t]; }); + } + return rows; +} + +void BaseDebyeSum::loadFactors(const std::vector& rows) { + auto& c = mstructure_cache; + c.nq = rows.empty() ? 0 : int(rows.front().size()); + c.factors.clear(); + c.types.curve.resize(rows.size()); + c.types.occupancy.assign(rows.size(), 1.0); + for (size_t t = 0; t < rows.size(); ++t) { + c.types.curve[t] = int(t); + c.factors.insert(c.factors.end(), rows[t].begin(), rows[t].end()); + } + melements.clear(); + mcustomTypes.clear(); + mcustomAmbiguous = false; } } // namespace srreal diff --git a/src/diffpy/srreal/BaseDebyeSum.hpp b/src/diffpy/srreal/BaseDebyeSum.hpp index b5cd11a..97a4204 100644 --- a/src/diffpy/srreal/BaseDebyeSum.hpp +++ b/src/diffpy/srreal/BaseDebyeSum.hpp @@ -19,6 +19,11 @@ #ifndef BASEDEBYESUM_HPP_INCLUDED #define BASEDEBYESUM_HPP_INCLUDED +#include +#include +#include +#include + #include #include #include @@ -62,6 +67,10 @@ class DLL_EXPORT BaseDebyeSum : public PairQuantity, // PairQuantity overloads virtual void resetValue(); virtual void addPairContribution(const BaseBondGenerator&, int); + virtual bool addAllPairContributions(); + /// addAllPairContributions for derived classes that keep the + /// addPairContribution of this class. + bool addAllPairsOfDebyeSum(); // support for PQEvaluatorOptimized virtual void stashPartialValue(); virtual void restorePartialValue(); @@ -81,13 +90,31 @@ class DLL_EXPORT BaseDebyeSum : public PairQuantity, double mqmax; double mqstep; double mdebyeprecision; - struct { + struct SiteTypes { std::vector typeofsite; - std::vector sftypeatkq; + // A scattering type names an elemental curve and a site occupancy. + std::vector curve; + std::vector occupancy; + }; + struct { + SiteTypes types; + QuantityType factors; // consecutive curves on the Q grid + int nq = 0; QuantityType sfaverageatkq; double totaloccupancy; } mstructure_cache; QuantityType mdbsumstash; + std::map melements; + eventticker::EventTicker msfticker; + + using SiteKey = std::pair; + // Representative curves identify changes to virtual, site-specific factors. + std::map mcustomTypes; + bool mcustomAmbiguous = false; + void cacheCustomFactors(SiteTypes&, int nq); + void cacheElementFactors(SiteTypes&, int nq); + std::vector archiveFactors() const; + void loadFactors(const std::vector& curves); // serialization friend class boost::serialization::access; @@ -100,8 +127,12 @@ class DLL_EXPORT BaseDebyeSum : public PairQuantity, ar & mqmax; ar & mqstep; ar & mdebyeprecision; - ar & mstructure_cache.typeofsite; - ar & mstructure_cache.sftypeatkq; + ar & mstructure_cache.types.typeofsite; + // Preserve the archive representation used before elemental curves. + std::vector curves; + if constexpr (Archive::is_saving::value) curves = this->archiveFactors(); + ar & curves; + if constexpr (Archive::is_loading::value) this->loadFactors(curves); ar & mstructure_cache.sfaverageatkq; ar & mstructure_cache.totaloccupancy; } diff --git a/src/diffpy/srreal/CalculationKernel.hpp b/src/diffpy/srreal/CalculationKernel.hpp index 35902a1..7f89610 100644 --- a/src/diffpy/srreal/CalculationKernel.hpp +++ b/src/diffpy/srreal/CalculationKernel.hpp @@ -24,12 +24,12 @@ namespace diffpy { namespace srreal { -/// Names of the kernels that this build and CPU can run, from the most to -/// the least preferred: "avx512", "avx2" and "sse2" on x86-64, "neon" on -/// arm64, and "scalar", which is last and always available. +/// Names of the kernels that this build and machine can run, CPU kernels +/// first in order of preference. A compatible CUDA device adds "cuda" and +/// "cuda-mixed", which only run full PDF and Debye sums. DLL_EXPORT std::vector availableCalculationKernels(); -/// Name of the kernel in use, by default the most preferred one. +/// Name of the kernel in use, by default the first available one. DLL_EXPORT std::string getCalculationKernel(); /// Use the named kernel for all calculators. diff --git a/src/diffpy/srreal/ConstantPeakWidth.cpp b/src/diffpy/srreal/ConstantPeakWidth.cpp index 2526da8..1659a63 100644 --- a/src/diffpy/srreal/ConstantPeakWidth.cpp +++ b/src/diffpy/srreal/ConstantPeakWidth.cpp @@ -82,6 +82,10 @@ const string& ConstantPeakWidth::type() const { return rv; } +double ConstantPeakWidth::calculateFromMSD(double r, double msd) const { + return this->getWidth(); +} + double ConstantPeakWidth::calculate(const BaseBondGenerator& bnds) const { return this->getWidth(); } diff --git a/src/diffpy/srreal/ConstantPeakWidth.hpp b/src/diffpy/srreal/ConstantPeakWidth.hpp index a991033..da78765 100644 --- a/src/diffpy/srreal/ConstantPeakWidth.hpp +++ b/src/diffpy/srreal/ConstantPeakWidth.hpp @@ -36,6 +36,9 @@ class DLL_EXPORT ConstantPeakWidth : public PeakWidthModel { // methods virtual const std::string& type() const; virtual double calculate(const BaseBondGenerator&) const; + /// Width of this model for a bond of the distance r with the mean square + /// displacement msd. This model uses neither. + double calculateFromMSD(double r, double msd) const; virtual double maxWidth(StructureAdapterPtr, double rmin, double rmax) const; // data access diff --git a/src/diffpy/srreal/CrystalStructureAdapter.cpp b/src/diffpy/srreal/CrystalStructureAdapter.cpp index 38a5a79..9d9cdf0 100644 --- a/src/diffpy/srreal/CrystalStructureAdapter.cpp +++ b/src/diffpy/srreal/CrystalStructureAdapter.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -65,6 +66,13 @@ bool sameLatticeBits(const Lattice& L0, const Lattice& L1) { return sameBits(L0.base().data(), L1.base().data()); } +/// Guards the symmetry positions of all adapters, which const methods +/// expand on demand, also in threads that share an adapter. +std::mutex& symmetryCacheMutex() { + static std::mutex mutex; + return mutex; +} + } // namespace ////////////////////////////////////////////////////////////////////////////// @@ -81,6 +89,7 @@ CrystalStructureAdapter::CrystalStructureAdapter() // Public Methods ------------------------------------------------------------ StructureAdapterPtr CrystalStructureAdapter::clone() const { + const std::lock_guard lock(symmetryCacheMutex()); StructureAdapterPtr rv(new CrystalStructureAdapter(*this)); return rv; } @@ -151,6 +160,7 @@ const SymOpRotTrans& CrystalStructureAdapter::getSymOp(int i) const { const CrystalStructureAdapter::AtomVector& CrystalStructureAdapter::getEquivalentAtoms(int idx) const { assert(0 <= idx && idx < this->countSites()); + const std::lock_guard lock(symmetryCacheMutex()); if (!this->isSiteSymmetryUpToDate(idx)) this->updateSymmetryPositions(); return msymatoms[idx]; } @@ -204,21 +214,24 @@ CrystalStructureAdapter::AtomVector CrystalStructureAdapter::expandLatticeAtom( } void CrystalStructureAdapter::updateSymmetryPositions() const { - // build asymmetric unit in lattice coordinates - AtomVector lcatoms(this->begin(), this->end()); - AtomVector::iterator lcai = lcatoms.begin(); - for (; lcai != lcatoms.end(); ++lcai) this->toFractional(*lcai); - // build symmetry positions for all atoms in the asymmetric unit - msymatoms.resize(this->countSites()); - assert(lcatoms.size() == msymatoms.size()); - lcai = lcatoms.begin(); - std::vector::iterator saii = msymatoms.begin(); - for (; lcai != lcatoms.end(); ++lcai, ++saii) { - *saii = this->expandLatticeAtom(*lcai); - iterator ai = saii->begin(); - for (; ai != saii->end(); ++ai) this->toCartesian(*ai); + // Expand only the sites whose atoms changed, or all of them after a change + // of the lattice or a reset of msymmetry_cached. + const int cntsites = this->countSites(); + const bool keepsame = this->isSymmetryCached() && + msymsource.size() == msymatoms.size() && + sameLatticeBits(msymlattice, this->getLattice()); + msymatoms.resize(cntsites); + if (!keepsame) msymsource.assign(cntsites, Atom()); + for (int i = 0; i < cntsites; ++i) { + const Atom& a = (*this)[i]; + if (keepsame && sameAtomBits(msymsource[i], a)) continue; + // expand the atom in lattice coordinates + Atom lca = a; + this->toFractional(lca); + msymatoms[i] = this->expandLatticeAtom(lca); + for (Atom& ai : msymatoms[i]) this->toCartesian(ai); + msymsource[i] = a; } - msymsource.assign(this->begin(), this->end()); msymlattice = this->getLattice(); msymmetry_cached = true; } @@ -272,6 +285,7 @@ bool CrystalStructureAdapter::isSymmetryUpToDate() const { const std::vector& CrystalStructureAdapter::symmetryAtoms() const { + const std::lock_guard lock(symmetryCacheMutex()); if (!this->isSymmetryUpToDate()) this->updateSymmetryPositions(); return msymatoms; } diff --git a/src/diffpy/srreal/DebyePDFCalculator.cpp b/src/diffpy/srreal/DebyePDFCalculator.cpp index b2f6300..364c04f 100644 --- a/src/diffpy/srreal/DebyePDFCalculator.cpp +++ b/src/diffpy/srreal/DebyePDFCalculator.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -231,6 +232,11 @@ void DebyePDFCalculator::configureBondGenerator(BaseBondGenerator& bnds) const { bnds.setRmax(this->rcalchi()); } +bool DebyePDFCalculator::addAllPairContributions() { + if (typeid(*this) != typeid(DebyePDFCalculator)) return false; + return this->addAllPairsOfDebyeSum(); +} + double DebyePDFCalculator::sfSiteAtQ(int siteidx, const double& Q) const { const ScatteringFactorTablePtr& sftable = this->getScatteringFactorTable(); const string& smbl = mstructure->siteAtomType(siteidx); diff --git a/src/diffpy/srreal/DebyePDFCalculator.hpp b/src/diffpy/srreal/DebyePDFCalculator.hpp index 0162fd6..2e3771d 100644 --- a/src/diffpy/srreal/DebyePDFCalculator.hpp +++ b/src/diffpy/srreal/DebyePDFCalculator.hpp @@ -104,6 +104,7 @@ class DLL_EXPORT DebyePDFCalculator : public BaseDebyeSum, virtual void resetValue(); virtual void configureBondGenerator(BaseBondGenerator&) const; virtual double sfSiteAtQ(int, const double& Q) const; + virtual bool addAllPairContributions(); private: // methods diff --git a/src/diffpy/srreal/DebyeWallerPeakWidth.cpp b/src/diffpy/srreal/DebyeWallerPeakWidth.cpp index e38db3a..b9d4fa2 100644 --- a/src/diffpy/srreal/DebyeWallerPeakWidth.cpp +++ b/src/diffpy/srreal/DebyeWallerPeakWidth.cpp @@ -19,6 +19,7 @@ #include #include +#include #include namespace diffpy { @@ -46,10 +47,11 @@ const string& DebyeWallerPeakWidth::type() const { } double DebyeWallerPeakWidth::calculate(const BaseBondGenerator& bnds) const { - using diffpy::mathutils::GAUSS_SIGMA_TO_FWHM; - double msdval = bnds.msd(); - double rv = (msdval < 0.0) ? 0.0 : GAUSS_SIGMA_TO_FWHM * sqrt(msdval); - return rv; + return this->calculateFromMSD(bnds.distance(), bnds.msd()); +} + +double DebyeWallerPeakWidth::calculateFromMSD(double r, double msdval) const { + return kernels::debyeWallerWidth(msdval); } double DebyeWallerPeakWidth::maxWidth(StructureAdapterPtr stru, double rmin, diff --git a/src/diffpy/srreal/DebyeWallerPeakWidth.hpp b/src/diffpy/srreal/DebyeWallerPeakWidth.hpp index ec98b3d..3e8c7d5 100644 --- a/src/diffpy/srreal/DebyeWallerPeakWidth.hpp +++ b/src/diffpy/srreal/DebyeWallerPeakWidth.hpp @@ -36,6 +36,9 @@ class DLL_EXPORT DebyeWallerPeakWidth : public PeakWidthModel { // methods virtual const std::string& type() const; virtual double calculate(const BaseBondGenerator&) const; + /// Width of this model for a bond of the distance r with the mean square + /// displacement msd. This model does not use r. + virtual double calculateFromMSD(double r, double msd) const; virtual double maxWidth(StructureAdapterPtr, double rmin, double rmax) const; private: diff --git a/src/diffpy/srreal/JeongPeakWidth.cpp b/src/diffpy/srreal/JeongPeakWidth.cpp index 0d4466d..cf79f62 100644 --- a/src/diffpy/srreal/JeongPeakWidth.cpp +++ b/src/diffpy/srreal/JeongPeakWidth.cpp @@ -19,7 +19,7 @@ *****************************************************************************/ #include -#include +#include #include namespace diffpy { @@ -63,11 +63,17 @@ double JeongPeakWidth::calculate(const BaseBondGenerator& bnds) const { double r = bnds.distance(); double corr = this->msdSharpeningRatio(r); // avoid calculating square root of negative value - double fwhm = (corr <= 0) - ? 0.0 - : (sqrt(corr) * this->DebyeWallerPeakWidth::calculate(bnds) + - pow(this->getQbroad_seperable() * r, 2)); - return fwhm; + // The bond calculates its displacement only when it is needed. + return (corr <= 0) + ? 0.0 + : kernels::sharpenedWidth(r, corr, bnds.msd(), mqbroad_seperable); +} + +double JeongPeakWidth::calculateFromMSD(double r, double msdval) const { + double corr = this->msdSharpeningRatio(r); + return (corr <= 0) + ? 0.0 + : kernels::sharpenedWidth(r, corr, msdval, mqbroad_seperable); } double JeongPeakWidth::maxWidth(StructureAdapterPtr stru, double rmin, @@ -114,12 +120,7 @@ void JeongPeakWidth::setQbroad(double qbroad) { // Private Methods ----------------------------------------------------------- double JeongPeakWidth::msdSharpeningRatio(const double& r) const { - using diffpy::mathutils::DOUBLE_EPS; - // avoid division by zero - if (r < DOUBLE_EPS) return 0.0; - double rv = 1.0 - this->getDelta1() / r - this->getDelta2() / pow(r, 2) + - pow(this->getQbroad() * r, 2); - return rv; + return kernels::sharpeningRatio(r, mdelta1, mdelta2, mqbroad); } // Registration -------------------------------------------------------------- diff --git a/src/diffpy/srreal/JeongPeakWidth.hpp b/src/diffpy/srreal/JeongPeakWidth.hpp index 021b76c..d1d88dd 100644 --- a/src/diffpy/srreal/JeongPeakWidth.hpp +++ b/src/diffpy/srreal/JeongPeakWidth.hpp @@ -38,6 +38,9 @@ class DLL_EXPORT JeongPeakWidth : public DebyeWallerPeakWidth { // methods virtual const std::string& type() const; virtual double calculate(const BaseBondGenerator&) const; + /// Width of this model for a bond of the distance r with the mean square + /// displacement msd. + double calculateFromMSD(double r, double msd) const override; virtual double maxWidth(StructureAdapterPtr, double rmin, double rmax) const; // data access diff --git a/src/diffpy/srreal/PDFCalculator.cpp b/src/diffpy/srreal/PDFCalculator.cpp index d464349..2f58b30 100644 --- a/src/diffpy/srreal/PDFCalculator.cpp +++ b/src/diffpy/srreal/PDFCalculator.cpp @@ -16,13 +16,19 @@ * *****************************************************************************/ +#include #include #include #include #include +#include #include #include +#include +#include +#include +#include #include #include #include @@ -245,19 +251,15 @@ void PDFCalculator::setQmax(double qmax) { if (_initialQstepUpdate(this)) this->resetValue(); } -const double& PDFCalculator::getQmax() const { - static double rv; - rv = min(mqmax, M_PI / this->getRstep()); - return rv; +double PDFCalculator::getQmax() const { + return min(mqmax, M_PI / this->getRstep()); } -const double& PDFCalculator::getQstep() const { - static double rv; +double PDFCalculator::getQstep() const { // replicate the zero padding as done in fftgtof - int Npad1 = this->extendedRmaxSteps(); - int Npad2 = (Npad1 > 0) ? (1 << int(ceil(log2(Npad1)))) : 0; - rv = (Npad2 > 0) ? M_PI / (Npad2 * this->getRstep()) : 0.0; - return rv; + const int n = this->extendedRmaxSteps(); + if (n <= 0) return 0.0; + return M_PI / (std::bit_ceil(std::size_t(n)) * this->getRstep()); } // R-range methods @@ -425,16 +427,35 @@ void PDFCalculator::addPairContribution(const BaseBondGenerator& bnds, double sfprod = this->sfSite(bnds.site0()) * this->sfSite(bnds.site1()); double peakscale = sfprod * bnds.multiplicity() * summationscale; double fwhm = this->getPeakWidthModel()->calculate(bnds); - const PeakProfile& pkf = *(this->getPeakProfile()); - double dist = bnds.distance(); - double xlo = dist + pkf.xboundlo(fwhm); - double xhi = dist + pkf.xboundhi(fwhm); - int i = max(0, this->calcIndex(xlo)); - int ilast = min(this->countCalcPoints(), this->calcIndex(xhi) + 1); - assert(ilast <= int(mvalue.size())); - assert(eps_gt(dist, 0.0)); - pkf.addRDFPeak(mvalue.data(), i, ilast, this->rcalcloSteps(), - this->getRstep(), dist, fwhm, peakscale); + kernels::addProfilePeak(*this->getPeakProfile(), mvalue.data(), + this->countCalcPoints(), this->rcalcloSteps(), + this->getRstep(), bnds.distance(), fwhm, peakscale); +} + +bool PDFCalculator::addAllPairContributions() { + if (typeid(*this) != typeid(PDFCalculator)) return false; + return this->addAllPairsOfPDF(); +} + +bool PDFCalculator::addAllPairsOfPDF() { + // Other peak width models and structures need per-bond evaluation. + kernels::PDFPairSumArgs args = {}; + if (!kernels::assignPairWidth(args.w, *this->getPeakWidthModel()) || + !kernels::PairTable::supports(*mstructure)) + return false; + const PeakProfile& pkf = *this->getPeakProfile(); + args.cropped = typeid(pkf) == typeid(CroppedGaussianProfile); + if (!args.cropped && typeid(pkf) != typeid(GaussianProfile)) return false; + args.height = pkf(0.0, 1.0); + args.halfboundrel = pkf.xboundhi(1.0); + args.sfsite = mstructure_cache.sfsite.data(); + args.npts = this->countCalcPoints(); + args.rlosteps = this->rcalcloSteps(); + args.rstep = this->getRstep(); + const auto& kernel = + this->preparePairSum(args, this->rcalclo(), this->rcalchi()); + mallpairs.record(kernel.pdfPairSum(args), kernel.onDevice); + return true; } void PDFCalculator::stashPartialValue() { @@ -519,11 +540,6 @@ int PDFCalculator::countCalcPoints() const { return rv; } -int PDFCalculator::calcIndex(double r) const { - int rv = int(floor(r / this->getRstep())) - this->rcalcloSteps(); - return rv; -} - void PDFCalculator::cutRipplePoints(QuantityType& y) const { if (y.empty()) return; assert(int(y.size()) == diff --git a/src/diffpy/srreal/PDFCalculator.hpp b/src/diffpy/srreal/PDFCalculator.hpp index ac89a8f..77828e9 100644 --- a/src/diffpy/srreal/PDFCalculator.hpp +++ b/src/diffpy/srreal/PDFCalculator.hpp @@ -65,8 +65,8 @@ class DLL_EXPORT PDFCalculator : public PairQuantity, void setQmin(double); const double& getQmin() const; void setQmax(double); - const double& getQmax() const; - const double& getQstep() const; + double getQmax() const; + double getQstep() const; // R-range methods QuantityType getRgrid() const; @@ -110,6 +110,10 @@ class DLL_EXPORT PDFCalculator : public PairQuantity, virtual void resetValue(); virtual void configureBondGenerator(BaseBondGenerator&) const; virtual void addPairContribution(const BaseBondGenerator&, int); + virtual bool addAllPairContributions(); + /// addAllPairContributions for derived classes that keep the + /// addPairContribution of this class. + bool addAllPairsOfPDF(); // support for PQEvaluatorOptimized virtual void stashPartialValue(); virtual void restorePartialValue(); @@ -136,8 +140,6 @@ class DLL_EXPORT PDFCalculator : public PairQuantity, int countExtendedPoints() const; /// number of points in the complete calculated r-grid int countCalcPoints() const; - /// index of a nearby point in the complete calculated r-grid - int calcIndex(double r) const; /// reduce extended grid to user-requested results grid /// by cutting away the points for termination ripples void cutRipplePoints(QuantityType& y) const; diff --git a/src/diffpy/srreal/PQEvaluator.cpp b/src/diffpy/srreal/PQEvaluator.cpp index aaa567b..0688087 100644 --- a/src/diffpy/srreal/PQEvaluator.cpp +++ b/src/diffpy/srreal/PQEvaluator.cpp @@ -28,6 +28,7 @@ #include #include #include +#include using namespace std; @@ -41,6 +42,15 @@ namespace { // tolerated load variance for splitting outer loop for parallel evaluation const double CPU_LOAD_VARIANCE = 0.1; +// Work of a sum of all pairs for each of its pairs and for each of the +// contributions that it merges them to, in the units of the work for a +// pair that is added by itself. +const double ALLPAIRS_PAIR_WORK = 0.2; +const double ALLPAIRS_PEAK_WORK = 0.85; +// The estimate is within about 15 %, so the sum of all pairs must take at +// most this share of the work of the fast update. +const double ALLPAIRS_WORK_MARGIN = 0.85; + SiteIndices complementary_indices(const int sz, const SiteIndices& indices0) { SiteIndices rv; rv.reserve(sz); @@ -73,9 +83,18 @@ void PQEvaluatorBasic::validate(PairQuantity& pq) const { return; } void PQEvaluatorBasic::updateValue(PairQuantity& pq, StructureAdapterPtr stru) { mtypeused = BASIC; + maddedallpairs = false; // The value is not valid until this update completes. mvalue_ticker = eventticker::EventTicker(); pq.setStructure(stru); + // A batch covers the complete half sum of an unpartitioned evaluation. + pq.mallpairs.ondevice = false; + const bool halfsum = !this->getFlag(USEFULLSUM) && !this->isParallel(); + if (halfsum && pq.addAllPairContributions()) { + maddedallpairs = true; + mvalue_ticker.click(); + return; + } BaseBondGeneratorPtr bnds = pq.mstructure->createBondGenerator(); pq.configureBondGenerator(*bnds); int cntsites = pq.mstructure->countSites(); @@ -141,9 +160,8 @@ void PQEvaluatorOptimized::validate(PairQuantity& pq) const { void PQEvaluatorOptimized::updateValue(PairQuantity& pq, StructureAdapterPtr stru) { mtypeused = OPTIMIZED; - // revert to normal calculation if there is no structure or - // if PairQuantity uses mask - if (pq.ticker() >= mvalue_ticker || !mlast_structure) { + if (pq.ticker() >= mvalue_ticker || !mlast_structure || + mlastkernel != &kernels::activeCalculationKernel()) { return this->updateValueCompletely(pq, stru); } // do not do fast updates if they take more work @@ -155,47 +173,52 @@ void PQEvaluatorOptimized::updateValue(PairQuantity& pq, sd.diffmethod != StructureDifference::Method::SIDEBYSIDE) { return this->updateValueCompletely(pq, stru); } + if (this->allPairsTakeLessWork(pq, sd)) { + return this->updateValueCompletely(pq, stru); + } // Remove contributions from the extra sites in the old structure assert(sd.stru0 == mlast_structure); // The value belongs to no structure until this update completes. + const StructureAdapterPtr last_structure = mlast_structure; mlast_structure.reset(); - int cntsites0 = sd.stru0->countSites(); - BaseBondGeneratorPtr bnds0 = sd.stru0->createBondGenerator(); - pq.configureBondGenerator(*bnds0); // loop counter long n = mcpuindex; bool usefullsum = this->getFlag(USEFULLSUM); - // the loop is adjusted according to usefullsum and split within - // the outer loop in case of parallel evaluation. - SiteIndices anchors = sd.pop0; - SiteIndices unchanged; + const bool hasmask = pq.hasMask(); + // A structure of the same content has no sites to remove or to add. + // It needs no bond generators, which copy all positions of a crystal. if (!sd.pop0.empty()) { - unchanged = complementary_indices(cntsites0, sd.pop0); + int cntsites0 = sd.stru0->countSites(); + BaseBondGeneratorPtr bnds0 = sd.stru0->createBondGenerator(); + pq.configureBondGenerator(*bnds0); + // the loop is adjusted according to usefullsum and split within + // the outer loop in case of parallel evaluation. + SiteIndices anchors = sd.pop0; + SiteIndices unchanged = complementary_indices(cntsites0, sd.pop0); anchors.insert(anchors.end(), unchanged.begin(), unchanged.end()); - } - bnds0->selectSites(anchors.begin(), anchors.end()); - SiteIndices::const_iterator last_anchor = - usefullsum ? anchors.end() : (anchors.begin() + sd.pop0.size()); - SiteIndices::const_iterator ii0; - bool needsreselection = usefullsum; - const bool hasmask = pq.hasMask(); - for (ii0 = anchors.begin(); ii0 != last_anchor; ++ii0) { - if (n++ % mncpu) continue; - const int& i0 = *ii0; - bnds0->selectAnchorSite(i0); - // when using half sum, deselect visited popped sites - if (!usefullsum) bnds0->selectSites(ii0, anchors.end()); - // when using full sum, select only the popped sites when - // anchored at an unchanged atom - else if (needsreselection && ii0 >= (anchors.begin() + sd.pop0.size())) { - bnds0->selectSites(sd.pop0.begin(), sd.pop0.end()); - needsreselection = false; - } - for (bnds0->rewind(); !bnds0->finished(); bnds0->next()) { - int i1 = bnds0->site1(); - if (hasmask && !pq.getPairMask(i0, i1)) continue; - const int summationscale = (usefullsum || i0 == i1) ? -1 : -2; - pq.addPairContribution(*bnds0, summationscale); + bnds0->selectSites(anchors.begin(), anchors.end()); + SiteIndices::const_iterator last_anchor = + usefullsum ? anchors.end() : (anchors.begin() + sd.pop0.size()); + SiteIndices::const_iterator ii0; + bool needsreselection = usefullsum; + for (ii0 = anchors.begin(); ii0 != last_anchor; ++ii0) { + if (n++ % mncpu) continue; + const int& i0 = *ii0; + bnds0->selectAnchorSite(i0); + // when using half sum, deselect visited popped sites + if (!usefullsum) bnds0->selectSites(ii0, anchors.end()); + // when using full sum, select only the popped sites when + // anchored at an unchanged atom + else if (needsreselection && ii0 >= (anchors.begin() + sd.pop0.size())) { + bnds0->selectSites(sd.pop0.begin(), sd.pop0.end()); + needsreselection = false; + } + for (bnds0->rewind(); !bnds0->finished(); bnds0->next()) { + int i1 = bnds0->site1(); + if (hasmask && !pq.getPairMask(i0, i1)) continue; + const int summationscale = (usefullsum || i0 == i1) ? -1 : -2; + pq.addPairContribution(*bnds0, summationscale); + } } } // Add contributions from the new atoms in the updated structure @@ -210,47 +233,78 @@ void PQEvaluatorOptimized::updateValue(PairQuantity& pq, return this->updateValueCompletely(pq, stru); } pq.restorePartialValue(); - int cntsites1 = sd.stru1->countSites(); - BaseBondGeneratorPtr bnds1 = sd.stru1->createBondGenerator(); - pq.configureBondGenerator(*bnds1); - anchors = sd.add1; - unchanged.clear(); if (!sd.add1.empty()) { - unchanged = complementary_indices(cntsites1, sd.add1); + int cntsites1 = sd.stru1->countSites(); + BaseBondGeneratorPtr bnds1 = sd.stru1->createBondGenerator(); + pq.configureBondGenerator(*bnds1); + SiteIndices anchors = sd.add1; + SiteIndices unchanged = complementary_indices(cntsites1, sd.add1); anchors.insert(anchors.begin(), unchanged.begin(), unchanged.end()); - } - bnds1->selectSites(sd.add1.begin(), sd.add1.end()); - SiteIndices::const_iterator first_anchor = - usefullsum ? anchors.begin() : (anchors.end() - sd.add1.size()); - SiteIndices::const_iterator ii1; - needsreselection = usefullsum; - for (ii1 = first_anchor; ii1 != anchors.end(); ++ii1) { - if (n++ % mncpu) continue; - const int& i0 = *ii1; - bnds1->selectAnchorSite(i0); - // when using half sum, activate the added site - if (!usefullsum) bnds1->selectSites(anchors.begin(), ii1 + 1); - // when using full sum select unchanged atoms once anchored - // at an added atom. - else if (needsreselection && ii1 >= (anchors.end() - sd.add1.size())) { - bnds1->selectSites(anchors.begin(), anchors.end()); - needsreselection = false; - } - for (bnds1->rewind(); !bnds1->finished(); bnds1->next()) { - int i1 = bnds1->site1(); - if (hasmask && !pq.getPairMask(i0, i1)) continue; - const int summationscale = (usefullsum || i0 == i1) ? +1 : +2; - pq.addPairContribution(*bnds1, summationscale); + bnds1->selectSites(sd.add1.begin(), sd.add1.end()); + SiteIndices::const_iterator first_anchor = + usefullsum ? anchors.begin() : (anchors.end() - sd.add1.size()); + SiteIndices::const_iterator ii1; + bool needsreselection = usefullsum; + for (ii1 = first_anchor; ii1 != anchors.end(); ++ii1) { + if (n++ % mncpu) continue; + const int& i0 = *ii1; + bnds1->selectAnchorSite(i0); + // when using half sum, activate the added site + if (!usefullsum) bnds1->selectSites(anchors.begin(), ii1 + 1); + // when using full sum select unchanged atoms once anchored + // at an added atom. + else if (needsreselection && ii1 >= (anchors.end() - sd.add1.size())) { + bnds1->selectSites(anchors.begin(), anchors.end()); + needsreselection = false; + } + for (bnds1->rewind(); !bnds1->finished(); bnds1->next()) { + int i1 = bnds1->site1(); + if (hasmask && !pq.getPairMask(i0, i1)) continue; + const int summationscale = (usefullsum || i0 == i1) ? +1 : +2; + pq.addPairContribution(*bnds1, summationscale); + } } } - mlast_structure = pq.getStructure()->clone(); + // The saved geometry must use the site order of the new scattering arrays. + // A sorted comparison can report empty differences after a permutation. + const bool sameorder = + sd.diffmethod == StructureDifference::Method::SIDEBYSIDE && + sd.pop0.empty() && sd.add1.empty(); + mlast_structure = sameorder ? last_structure : pq.getStructure()->clone(); mvalue_ticker.click(); } void PQEvaluatorOptimized::updateValueCompletely(PairQuantity& pq, StructureAdapterPtr stru) { + mlast_structure.reset(); this->PQEvaluatorBasic::updateValue(pq, stru); mlast_structure = pq.getStructure()->clone(); + mlastkernel = &kernels::activeCalculationKernel(); +} + +// True when a sum of all pairs is expected to take less work than the fast +// update of the changed sites. + +bool PQEvaluatorOptimized::allPairsTakeLessWork( + const PairQuantity& pq, const StructureDifference& sd) const { + // A change of whether the calculator adds all pairs forces a full + // calculation anyway. + if (!maddedallpairs) return false; + // A sum on the device beats any fast update on the CPU. + const bool changes = !sd.pop0.empty() || !sd.add1.empty(); + if (pq.mallpairs.ondevice) return changes; + // share of the pairs that have one of the sites + auto pairshare = [](const SiteIndices& sites, const StructureAdapter& s) { + if (sites.empty()) return 0.0; + const double others = 1.0 - double(sites.size()) / s.countSites(); + return 1.0 - others * others; + }; + const double pairs = pq.mallpairs.pairs; + const double fastwork = + (pairshare(sd.pop0, *sd.stru0) + pairshare(sd.add1, *sd.stru1)) * pairs; + const double fullwork = + ALLPAIRS_PAIR_WORK * pairs + ALLPAIRS_PEAK_WORK * pq.mallpairs.peaks; + return fullwork < ALLPAIRS_WORK_MARGIN * fastwork; } // Helper classes and functions for PQEvaluatorCheck ------------------------- @@ -373,6 +427,7 @@ PQEvaluatorPtr createPQEvaluator(PQEvaluatorType pqtp, PQEvaluatorPtr pqevsrc) { rv->mncpu = pqevsrc->mncpu; rv->mvalue_ticker = pqevsrc->mvalue_ticker; rv->mtypeused = pqevsrc->mtypeused; + rv->maddedallpairs = pqevsrc->maddedallpairs; } return rv; } diff --git a/src/diffpy/srreal/PQEvaluator.hpp b/src/diffpy/srreal/PQEvaluator.hpp index a4073b4..e1015e1 100644 --- a/src/diffpy/srreal/PQEvaluator.hpp +++ b/src/diffpy/srreal/PQEvaluator.hpp @@ -39,6 +39,10 @@ namespace srreal { class PairQuantity; +namespace kernels { +struct CalculationKernel; +} + /// shared pointer to PQEvaluatorBasic typedef std::shared_ptr PQEvaluatorPtr; @@ -82,6 +86,8 @@ class DLL_EXPORT PQEvaluatorBasic { eventticker::EventTicker mvalue_ticker; /// type of PQEvaluator that was actually used PQEvaluatorType mtypeused; + /// the calculator added all pairs at once in the last full calculation + bool maddedallpairs = false; private: // serialization @@ -92,6 +98,8 @@ class DLL_EXPORT PQEvaluatorBasic { } }; +class StructureDifference; + class DLL_EXPORT PQEvaluatorOptimized : public PQEvaluatorBasic { public: // methods @@ -102,9 +110,13 @@ class DLL_EXPORT PQEvaluatorOptimized : public PQEvaluatorBasic { private: // data StructureAdapterPtr mlast_structure; + // Process-local kernel identity. A restored value is recomputed on first use. + const kernels::CalculationKernel* mlastkernel = nullptr; - // helper method + // helper methods void updateValueCompletely(PairQuantity&, StructureAdapterPtr); + bool allPairsTakeLessWork(const PairQuantity&, + const StructureDifference&) const; // serialization friend class boost::serialization::access; @@ -113,6 +125,7 @@ class DLL_EXPORT PQEvaluatorOptimized : public PQEvaluatorBasic { using boost::serialization::base_object; ar& base_object(*this); ar & mlast_structure; + if constexpr (Archive::is_loading::value) mlastkernel = nullptr; } }; diff --git a/src/diffpy/srreal/PairQuantity.cpp b/src/diffpy/srreal/PairQuantity.cpp index 8efa799..89584cf 100644 --- a/src/diffpy/srreal/PairQuantity.cpp +++ b/src/diffpy/srreal/PairQuantity.cpp @@ -21,6 +21,7 @@ #include #include +#include #include #include @@ -106,7 +107,7 @@ const double& PairQuantity::getRmax() const { return mrmax; } void PairQuantity::setEvaluatorType(PQEvaluatorType evtp) { if (mevaluator.get() && mevaluator->typeint() == evtp) return; - PQEvaluatorPtr pqev = createPQEvaluator(evtp, mevaluator); + PQEvaluatorPtr pqev = createPQEvaluator(evtp, mevaluator.ptr()); // validate the new evaluator object try { pqev->validate(*this); @@ -206,11 +207,11 @@ bool PairQuantity::getPairMask(int i, int j) const { } void PairQuantity::setTypeMask(string smbli, string smblj, bool mask) { - static string upcaseall; - if (upcaseall.empty()) { - upcaseall = ALLATOMSSTR; - transform(upcaseall.begin(), upcaseall.end(), upcaseall.begin(), ::toupper); - } + static const string upcaseall = [] { + string rv = ALLATOMSSTR; + transform(rv.begin(), rv.end(), rv.begin(), ::toupper); + return rv; + }(); // accept "ALL" (upper ALLATOMSSTR) for smbli and smblj if (upcaseall == smbli) smbli = ALLATOMSSTR; if (upcaseall == smblj) smblj = ALLATOMSSTR; @@ -273,6 +274,26 @@ void PairQuantity::resetValue() { fill(mvalue.begin(), mvalue.end(), 0.0); } +const kernels::CalculationKernel& PairQuantity::preparePairSum( + kernels::PairSumArgs& args, double rmin, double rmax) { + if (!mallpairs.pairsum) + mallpairs.pairsum = std::make_shared(); + auto& state = *mallpairs.pairsum; + args.mask = this->hasMask() ? this : nullptr; + const auto& kernel = kernels::pairSumKernel(args.mask != nullptr); + args.g = state.table.geometryArgs(*mstructure, rmin, rmax); + args.workspace = state.workspaceFor(kernel); + args.out = mvalue.data(); + return kernel; +} + +void PairQuantity::AllPairsState::record(kernels::PairSumStats stats, + bool usedDevice) { + pairs = stats.pairs; + peaks = stats.peaks; + ondevice = usedDevice; +} + void PairQuantity::configureBondGenerator(BaseBondGenerator& bnds) const { bnds.setRmin(this->getRmin()); bnds.setRmax(this->getRmax()); diff --git a/src/diffpy/srreal/PairQuantity.hpp b/src/diffpy/srreal/PairQuantity.hpp index ee8fb51..e50b640 100644 --- a/src/diffpy/srreal/PairQuantity.hpp +++ b/src/diffpy/srreal/PairQuantity.hpp @@ -39,6 +39,13 @@ namespace srreal { class BaseBondGenerator; +namespace kernels { +struct PairSumState; +struct PairSumArgs; +struct PairSumStats; +struct CalculationKernel; +} // namespace kernels + class DLL_EXPORT PairQuantity : public diffpy::Attributes { public: // class constants @@ -93,6 +100,13 @@ class DLL_EXPORT PairQuantity : public diffpy::Attributes { virtual void resetValue(); virtual void configureBondGenerator(BaseBondGenerator&) const; virtual void addPairContribution(const BaseBondGenerator&, int) {} + /// For an unpartitioned half sum, add every unmasked pair as + /// addPairContribution would. Return false before changing the value + /// to use per-bond evaluation. Derived calculators can opt in when + /// their batch sum matches their per-bond contributions. + virtual bool addAllPairContributions() { return false; } + const kernels::CalculationKernel& preparePairSum(kernels::PairSumArgs&, + double rmin, double rmax); virtual void executeParallelMerge(const std::string& pdata); virtual void finishValue() {} int countSites() const; @@ -109,17 +123,66 @@ class DLL_EXPORT PairQuantity : public diffpy::Attributes { typedef std::unordered_map, bool, diffpy::PairHash> TypeMaskStorage; + /// Pairs of the last structures for calculators that add all pairs at + /// once. Neither copied nor serialized. + struct AllPairsState { + AllPairsState() = default; + void record(kernels::PairSumStats stats, bool usedDevice); + AllPairsState(const AllPairsState& other) + : ondevice(other.ondevice), pairs(other.pairs), peaks(other.peaks) {} + AllPairsState& operator=(const AllPairsState& other) { + pairsum.reset(); + ondevice = other.ondevice; + pairs = other.pairs; + peaks = other.peaks; + return *this; + } + std::shared_ptr pairsum; + /// The last full sum ran on a GPU. + bool ondevice = false; + /// Pairs and merged contributions of the last sum of all pairs, from + /// which the OPTIMIZED evaluator estimates the work of the next one. + double pairs = 0.0; + double peaks = 0.0; + }; + QuantityType mvalue; StructureAdapterPtr mstructure; double mrmin; double mrmax; - PQEvaluatorPtr mevaluator; + /// The evaluator of the calculator. A copy gets a fresh evaluator, so + /// that it does not take the last structure of the original for its own. + class EvaluatorHolder { + public: + EvaluatorHolder() {} + EvaluatorHolder(const EvaluatorHolder& other) : mptr(other.copy()) {} + EvaluatorHolder& operator=(const EvaluatorHolder& other) { + if (this != &other) mptr = other.copy(); + return *this; + } + EvaluatorHolder& operator=(PQEvaluatorPtr pqev) { + mptr = pqev; + return *this; + } + PQEvaluatorBasic* operator->() const { return mptr.get(); } + PQEvaluatorBasic* get() const { return mptr.get(); } + PQEvaluatorPtr& ptr() { return mptr; } + + private: + PQEvaluatorPtr copy() const { + return mptr ? createPQEvaluator(mptr->typeint(), mptr) : mptr; + } + PQEvaluatorPtr mptr; + }; + + EvaluatorHolder mevaluator; bool mdefaultpairmask; PairMaskStorage minvertpairmask; std::unordered_map msiteallmask; TypeMaskStorage mtypemask; int mmergedvaluescount; mutable eventticker::EventTicker mticker; + AllPairsState mallpairs; private: // methods @@ -134,7 +197,7 @@ class DLL_EXPORT PairQuantity : public diffpy::Attributes { ar & mstructure; ar & mrmin; ar & mrmax; - ar & mevaluator; + ar & mevaluator.ptr(); ar & mdefaultpairmask; ar & minvertpairmask; ar & msiteallmask; diff --git a/src/diffpy/srreal/PeriodicStructureAdapter.hpp b/src/diffpy/srreal/PeriodicStructureAdapter.hpp index 6e3bcff..443959c 100644 --- a/src/diffpy/srreal/PeriodicStructureAdapter.hpp +++ b/src/diffpy/srreal/PeriodicStructureAdapter.hpp @@ -164,6 +164,8 @@ class DLL_EXPORT PeriodicStructureBondGenerator : public BaseBondGenerator { int nextpoint = 0; }; + friend class kernels::PairTable; + /// Create generator without site1 positions, add them with addPosition. PeriodicStructureBondGenerator(StructureAdapterConstPtr, BondOrder); diff --git a/src/diffpy/srreal/ScatteringFactorTable.cpp b/src/diffpy/srreal/ScatteringFactorTable.cpp index b759d7e..2aa4497 100644 --- a/src/diffpy/srreal/ScatteringFactorTable.cpp +++ b/src/diffpy/srreal/ScatteringFactorTable.cpp @@ -100,14 +100,16 @@ unordered_set ScatteringFactorTable::getCustomSymbols() const { void ScatteringFactorTableOwner::setScatteringFactorTable( ScatteringFactorTablePtr sft) { ensureNonNull("ScatteringFactorTable", sft); - if (msftable != sft) mprivateticker.click(); - msftable = sft; + if (msftable != sft) { + msftable = sft; + mtickedtable = msftable; + mprivateticker.click(); + } } void ScatteringFactorTableOwner::setScatteringFactorTableByType( const string& tp) { - msftable = ScatteringFactorTable::createByType(tp); - mprivateticker.click(); + this->setScatteringFactorTable(ScatteringFactorTable::createByType(tp)); } ScatteringFactorTablePtr& @@ -127,6 +129,10 @@ const string& ScatteringFactorTableOwner::getRadiationType() const { } eventticker::EventTicker& ScatteringFactorTableOwner::ticker() const { + if (mtickedtable.lock() != msftable) { + mtickedtable = msftable; + mprivateticker.click(); + } if (msftable) mprivateticker.updateFrom(msftable->ticker()); return mprivateticker; } diff --git a/src/diffpy/srreal/ScatteringFactorTable.hpp b/src/diffpy/srreal/ScatteringFactorTable.hpp index c059cc2..4308b89 100644 --- a/src/diffpy/srreal/ScatteringFactorTable.hpp +++ b/src/diffpy/srreal/ScatteringFactorTable.hpp @@ -22,6 +22,7 @@ #ifndef SCATTERINGFACTORTABLE_HPP_INCLUDED #define SCATTERINGFACTORTABLE_HPP_INCLUDED +#include #include #include @@ -88,6 +89,8 @@ class DLL_EXPORT ScatteringFactorTableOwner { private: // data ScatteringFactorTablePtr msftable; + // Detect replacements made through the mutable table reference. + mutable std::weak_ptr mtickedtable; mutable eventticker::EventTicker mprivateticker; // serialization @@ -95,6 +98,7 @@ class DLL_EXPORT ScatteringFactorTableOwner { template void serialize(Archive& ar, const unsigned int version) { ar & msftable & mprivateticker; + if constexpr (Archive::is_loading::value) mtickedtable = msftable; } }; diff --git a/src/diffpy/srreal/kernels/CalculationKernelCPU.cpp b/src/diffpy/srreal/kernels/CalculationKernelCPU.cpp new file mode 100644 index 0000000..4b40e98 --- /dev/null +++ b/src/diffpy/srreal/kernels/CalculationKernelCPU.cpp @@ -0,0 +1,154 @@ +/***************************************************************************** + * + * libdiffpy Complex Modeling Initiative + * (c) 2026 The Trustees of Columbia University + * in the City of New York. All rights reserved. + * + * See AUTHORS.rst for a list of people who contributed. + * See LICENSE.rst for license information. + * + *****************************************************************************/ + +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace diffpy { +namespace srreal { +namespace kernels { + +namespace { + +// Each worker groups a bounded number of pairs before running the SIMD +// windows. Symmetric structures often need only a few distinct windows. +constexpr long long GROUP_BATCH = 65536; +constexpr long long PAIRS_PER_WORKER = 8192; + +int workerCount(long long pairs) { + if (pairs < 2 * PAIRS_PER_WORKER) return 1; + const long long hardware = std::max(1u, std::thread::hardware_concurrency()); + return int(std::min(pairs / PAIRS_PER_WORKER, hardware)); +} + +struct Worker { + PairGroups groups; + std::vector sum; + PairSumStats stats; +}; + +struct CPUState : PairSumWorkspace { + std::vector workers; +}; + +// Only immutable geometry, scattering arrays and mask data are read by +// the workers. Publish the result in range order after all workers succeed. +template +PairSumStats sumPairs(const PairSumArgs& a, int npts, + const GroupPair& groupPair, const AddPeak& addPeak) { + const long long count = pairCount(a.g); + if (!count || !npts) return {}; + const int nworkers = workerCount(count); + auto& workers = static_cast(*a.workspace).workers; + workers.resize(nworkers); + auto run = [&](int i) { + auto& w = workers[i]; + w.sum.assign(npts, 0.0); + // Count locally while neighboring workers append to their group vectors. + PairSumStats stats; + const auto geometry = a.g; + // Divide before multiplying so even a large implicit triangle fits. + const long long stride = count / nworkers; + const long long rest = count % nworkers; + const long long first = i * stride + std::min(i, rest); + const long long last = first + stride + (i < rest); + for (long long begin = first; begin < last; begin += GROUP_BATCH) { + w.groups.clear(); + const long long end = std::min(begin + GROUP_BATCH, last); + ClusterPair cluster = + geometry.translations ? ClusterPair{} : ClusterPair::at(begin); + for (long long k = begin; k < end; ++k) { + const Pair p = pairAt(geometry, k, cluster); + if (!geometry.translations) cluster.next(); + if (!p.ok) continue; + ++stats.pairs; + if (a.mask && !a.mask->getPairMask(p.site0, p.site1)) continue; + groupPair(w.groups, p, widthOf(a.w, p.dist, p.msd)); + } + for (const auto& g : w.groups.groups()) addPeak(w.sum.data(), g); + stats.peaks += w.groups.groups().size(); + } + w.stats = stats; + }; + std::vector> pending; + for (int i = 1; i < nworkers; ++i) { + pending.push_back(std::async(std::launch::async, run, i)); + } + run(0); + for (auto& task : pending) task.get(); + + PairSumStats stats; + for (const Worker& w : workers) { + stats.pairs += w.stats.pairs; + stats.peaks += w.stats.peaks; + for (int i = 0; i < npts; ++i) a.out[i] += w.sum[i]; + } + return stats; +} + +} // namespace + +void addProfilePeak(const PeakProfile& profile, double* out, int npts, + int rlosteps, double rstep, double dist, double fwhm, + double scale) { + const int ilo = + gridIndex(dist + profile.xboundlo(fwhm), rstep, rlosteps, 0, npts); + const int ihi = + gridIndex(dist + profile.xboundhi(fwhm), rstep, rlosteps, 1, npts); + profile.addRDFPeak(out, ilo, ihi, rlosteps, rstep, dist, fwhm, scale); +} + +PairSumStats cpuPDFPairSum(const PDFPairSumArgs& a, GaussianWindow gaussian) { + auto group = [&](PairGroups& groups, const Pair& p, double width) { + groups.add(p.dist, width, 0, + a.sfsite[p.site0] * a.sfsite[p.site1] * p.weight); + }; + auto peak = [&](double* out, const PairGroups::Group& g) { + if (g.width <= 0) return; + const double half = a.halfboundrel * g.width; + const int ilo = gridIndex(g.dist - half, a.rstep, a.rlosteps, 0, a.npts); + const int ihi = gridIndex(g.dist + half, a.rstep, a.rlosteps, 1, a.npts); + gaussian({out, ilo, ihi, a.rlosteps, a.rstep, g.dist, g.width, g.scale, + a.height / g.width, bool(a.cropped), a.halfboundrel}); + }; + return sumPairs(a, a.npts, group, peak); +} + +PairSumStats cpuDebyePairSum(const DebyePairSumArgs& a, DebyeSum debye) { + if (a.kqlo >= a.kqhi) return {}; + auto group = [&](PairGroups& groups, const Pair& p, double width) { + const int t0 = a.typeofsite[p.site0], t1 = a.typeofsite[p.site1]; + const auto typepair = std::int64_t(t0) * a.ntypes + t1; + groups.add(p.dist, fwhmToSigma(width), typepair, 1.0, p.weight); + }; + auto peak = [&](double* out, const PairGroups::Group& g) { + // Test the cutoff at the scale of one pair before multiplying by count. + const int t0 = int(g.tag / a.ntypes), t1 = int(g.tag % a.ntypes); + debye({out, a.kqlo, a.kqhi, a.qstep, g.dist, g.width, double(g.weight), + a.prec, a.sf + size_t(a.curve[t0]) * a.nq, + a.sf + size_t(a.curve[t1]) * a.nq, g.scale, a.occupancy[t0], + a.occupancy[t1]}); + }; + return sumPairs(a, a.nq, group, peak); +} + +PairSumWorkspace* makeCPUWorkspace() { return new CPUState; } + +} // namespace kernels +} // namespace srreal +} // namespace diffpy diff --git a/src/diffpy/srreal/kernels/CalculationKernelCuda.cu b/src/diffpy/srreal/kernels/CalculationKernelCuda.cu new file mode 100644 index 0000000..c9b05bb --- /dev/null +++ b/src/diffpy/srreal/kernels/CalculationKernelCuda.cu @@ -0,0 +1,587 @@ +/***************************************************************************** + * + * libdiffpy Complex Modeling Initiative + * (c) 2026 The Trustees of Columbia University + * in the City of New York. All rights reserved. + * + * See AUTHORS.rst for a list of people who contributed. + * See LICENSE.rst for license information. + * + ****************************************************************************** + * + * CUDA calculation kernels, which add all pairs of a structure on a device + * + * The device computes the distance and width of every PairTable pair in + * double precision. A periodic pair list stays on the device while its + * neighbor search remains valid. Each output point is summed in a fixed + * order: PDF peaks in per-warp rows, Debye terms in blocks of pairs, then + * the rows or blocks in order. The mixed kernel evaluates peaks and Debye + * terms in single precision from arguments reduced in double precision. + * + *****************************************************************************/ + +#include + +#include +#include +#include +#include +#include +#include + +#include + +namespace diffpy { +namespace srreal { +namespace kernels { + +namespace { + +// Device ---------------------------------------------------------------------- + +const char* errorDescription(cudaError_t err) { + // The Windows runtime can return nullptr when the driver is unavailable. + const char* description = cudaGetErrorString(err); + return description ? description + : "CUDA runtime error (no description available)"; +} + +void check(cudaError_t err, const char* what) { + if (err == cudaSuccess) return; + throw std::runtime_error(std::string("the CUDA device failed in ") + what + + " with '" + errorDescription(err) + "'"); +} + +int currentDevice() { + int device; + check(cudaGetDevice(&device), "the current device lookup"); + return device; +} + +/// Memory on the device that grows as needed. +template +class Buffer { + public: + Buffer() {} + Buffer(const Buffer&) = delete; + Buffer& operator=(const Buffer&) = delete; + ~Buffer() { + if (mptr) cudaFree(mptr); + } + + T* reserve(size_t n) { + if (n > mcapacity) { + if (mptr) cudaFree(mptr); + mptr = nullptr; + mcapacity = 0; + void* p = nullptr; + check(cudaMalloc(&p, n * sizeof(T)), "the allocation of memory"); + mptr = static_cast(p); + mcapacity = n; + } + return mptr; + } + + T* upload(const T* src, size_t n) { + this->reserve(n); + if (n) { + check(cudaMemcpy(mptr, src, n * sizeof(T), cudaMemcpyHostToDevice), + "the copy of data to it"); + } + return mptr; + } + + T* zero(size_t n) { + this->reserve(n); + if (n) check(cudaMemset(mptr, 0, n * sizeof(T)), "the reset of memory"); + return mptr; + } + + void download(T* dst, size_t n) const { + if (n) { + check(cudaMemcpy(dst, mptr, n * sizeof(T), cudaMemcpyDeviceToHost), + "the copy of data from it"); + } + } + + T* get() const { return mptr; } + + private: + T* mptr = nullptr; + size_t mcapacity = 0; +}; + +// Pairs ----------------------------------------------------------------------- + +/// Maximum pairs in a Debye chunk. +const long long CHUNK = 1LL << 20; +/// Threads of a block. +const int THREADS = 128; +/// Pairs that one thread of the Debye sum adds for its Q point. +const int SUBCHUNK = 1024; + +constexpr double FOUR_LN2 = 2.7725887222397811; +constexpr double TWO_PI = 6.2831853071795865; + +// Peaks of the PDF +// -------------------------------------------------------------- + +struct PeakGrid { + int npts; + int rlosteps; + double rstep; + double height; + int cropped; + double halfboundrel; +}; + +// Keep the original lower grid index as the origin for sampling the peak. +struct PeakWindow { + int origin, first, last; + double xlo; + + __device__ double x(int i, double step) const { + return xlo + (i - origin) * step; + } +}; + +__device__ PeakWindow peakWindow(const PeakGrid& grid, double dist, + double fwhm) { + const double half = grid.halfboundrel * fwhm; + const int lo = + gridIndex(dist - half, grid.rstep, grid.rlosteps, 0, grid.npts); + const int hi = + gridIndex(dist + half, grid.rstep, grid.rlosteps, 1, grid.npts); + PeakWindow win{lo, lo, hi, (double(grid.rlosteps) + lo) * grid.rstep - dist}; + if (grid.cropped) + cropGaussianWindow(win.first, win.last, grid.rlosteps, grid.rstep, dist, + fwhm, grid.halfboundrel); + return win; +} + +__device__ double samplePeak(double x, double dist, double fwhm, + double amplitude) { + const double xrel = x / fwhm; + return amplitude * exp(-FOUR_LN2 * xrel * xrel) * (x / dist + 1.0); +} + +enum { WARP = 32 }; +struct Peak { + PeakWindow win; + double dist, fwhm, amplitude, invdist, invwidth; +}; + +/// Each warp groups bit-identical distances and widths, adds their weights +/// in lane order, and accumulates the resulting peaks in its own output row. +template +__global__ void pdfPeaks(PairGeometryArgs g, PairWidthArgs w, + const double* sfsite, PeakGrid grid, long long count, + double* rows) { + __shared__ Peak peaks[THREADS]; + __shared__ double scales[THREADS]; + const int lane = threadIdx.x % WARP; + const int warp = (blockIdx.x * THREADS + threadIdx.x) / WARP; + double* row = rows + (size_t)warp * grid.npts; + Peak* batch = peaks + threadIdx.x - lane; + double* weights = scales + threadIdx.x - lane; + for (long long first = (long long)warp * WARP; first < count; + first += (long long)gridDim.x * THREADS) { + Peak pk{}; + double scale = 0.; + if (first + lane < count) { + const Pair p = pairAt(g, first + lane); + if (p.ok) { + pk.dist = p.dist; + pk.fwhm = widthOf(w, p.dist, p.msd); + scale = sfsite[p.site0] * sfsite[p.site1] * p.weight; + } + } + const unsigned valid = __ballot_sync(0xffffffffu, pk.fwhm > 0.); + const unsigned distMask = + __match_any_sync(0xffffffffu, __double_as_longlong(pk.dist)); + const unsigned widthMask = + __match_any_sync(0xffffffffu, __double_as_longlong(pk.fwhm)); + const unsigned same = valid & distMask & widthMask; + const bool leader = lane == __ffs(same) - 1; + unsigned leaders = __ballot_sync(0xffffffffu, leader); + weights[lane] = scale; + __syncwarp(); + if (leader) { + for (unsigned others = same & (same - 1); others; others &= others - 1) + scale += weights[__ffs(others) - 1]; + pk.win = peakWindow(grid, pk.dist, pk.fwhm); + pk.invdist = 1.0 / pk.dist; + pk.invwidth = 1.0 / pk.fwhm; + pk.amplitude = scale * grid.height / pk.fwhm; + batch[lane] = pk; + } + __syncwarp(); + for (; leaders; leaders &= leaders - 1) { + const Peak p = batch[__ffs(leaders) - 1]; + const int offset = (lane - p.win.first) & (WARP - 1); + for (long long i = (long long)p.win.first + offset; i < p.win.last; + i += WARP) { + const double x = p.win.x(int(i), grid.rstep); + // An unresolved peak can have an unrepresentable inverse width. + if (p.win.last - p.win.first <= 2) { + row[i] += samplePeak(x, p.dist, p.fwhm, p.amplitude); + } else { + using Real = std::conditional_t; + const Real xr = Real(x * p.invwidth); + const Real correction = Real(x) * Real(p.invdist) + Real(1); + row[i] += + Real(p.amplitude) * exp(-Real(FOUR_LN2) * xr * xr) * correction; + } + } + } + __syncwarp(); + } +} + +// Terms of the Debye sum +// -------------------------------------------------------- + +struct QGrid { + int nq; + int kqlo; + int kqhi; + double qstep; + double prec; +}; + +/// The double kernel stores distance in angstroms. The mixed kernel stores +/// the phase step as a fraction of a turn in units of 2^-64. +template +struct DebyePair { + using Phase = std::conditional_t, double, + unsigned long long>; + Phase phase; + Real sigma, amplitude; + int type0, type1, kqstop; +}; + +template +struct DebyeBuffers { + Buffer> pairs; + Buffer factors; +}; + +/// Set up the pairs of a chunk. logsf contains log(|sf|) for each type. +template +__global__ void debyePairs(PairGeometryArgs g, PairWidthArgs w, + const int* typeofsite, const int* curve, + const double* occ, const double* sf, + const float* logsf, QGrid grid, long long first, + long long count, DebyePair* pairs) { + const long long j = (long long)blockIdx.x * blockDim.x + threadIdx.x; + if (j >= count) return; + const Pair p = pairAt(g, first + j); + if (!p.ok) { + pairs[j].kqstop = grid.kqlo; + return; + } + const double sigma = fwhmToSigma(widthOf(w, p.dist, p.msd)); + const int tp0 = typeofsite[p.site0]; + const int tp1 = typeofsite[p.site1]; + const double amplitude = p.weight / p.dist; + DebyePair out; + if constexpr (std::is_same_v) { + out.phase = p.dist; + } else { + const double turns = p.dist * grid.qstep / TWO_PI; + out.phase = + (unsigned long long)((turns - floor(turns)) * 18446744073709551616.0); + } + out.sigma = Real(sigma); + out.amplitude = Real(amplitude); + out.type0 = tp0; + out.type1 = tp1; + // The sum of a pair stops at the first kq with + // |amplitude * exp(-(sigma q)^2 / 2) * sf0 * sf1| <= prec. + // Its logarithm in single precision finds where that may be, then the + // terms themselves decide. + const double* sf0 = sf + (size_t)curve[tp0] * grid.nq; + const double* sf1 = sf + (size_t)curve[tp1] * grid.nq; + const float* lsf0 = logsf + (size_t)curve[tp0] * grid.nq; + const float* lsf1 = logsf + (size_t)curve[tp1] * grid.nq; + const float decay = (float)(0.5 * sigma * sigma * grid.qstep * grid.qstep); + const float limit = (grid.prec > 0.0) + ? (float)(log(grid.prec / fabs(amplitude)) - + log(fabs(occ[tp0])) - log(fabs(occ[tp1]))) + : -800.0f; + int kq = grid.kqlo; + for (; kq < grid.kqhi; ++kq) { + const float lterm = lsf0[kq] + lsf1[kq] - decay * (float)kq * (float)kq; + if (!(lterm > limit + 0.01f)) break; + } + for (; kq < grid.kqhi; ++kq) { + const double sq = sigma * (kq * grid.qstep); + const double term = amplitude * exp(-0.5 * sq * sq) * (sf0[kq] * occ[tp0]) * + (sf1[kq] * occ[tp1]); + if (fabs(term) <= grid.prec) break; + } + out.kqstop = kq; + pairs[j] = out; +} + +/// Apply each occupancy once per Q tile in the selected precision. +template +__global__ void occupiedFactors(const double* sf, const int* curve, + const double* occupancy, QGrid tile, int ntypes, + Real* out) { + const int span = tile.kqhi - tile.kqlo; + const int q = blockIdx.x * blockDim.x + threadIdx.x; + if (q >= span) return; + for (int t = blockIdx.y; t < ntypes; t += gridDim.y) { + out[(size_t)t * span + q] = + Real(sf[(size_t)curve[t] * tile.nq + tile.kqlo + q] * occupancy[t]); + } +} + +__device__ double debyeSine(double distance, double q, int) { + return sin(q * distance); +} + +__device__ float debyeSine(unsigned long long turns, float, int kq) { + // Fold the phase to a quarter turn before converting to single precision. + const long long phase = (long long)((unsigned long long)kq * turns); + const long long quarter = 1LL << 62; + const long long folded = (phase > quarter || phase < -quarter) + ? (long long)(0x8000000000000000ULL - phase) + : phase; + return sinpif((float)folded * 0x1p-63f); +} + +/// Each thread adds one Q point for a contiguous subchunk of pairs. +template +__global__ void debyeTerms(const DebyePair* pairs, const Real* sf, + QGrid grid, int count, double* partial) { + const int i = blockIdx.x * blockDim.x + threadIdx.x; + const int span = grid.kqhi - grid.kqlo; + if (i >= span) return; + const int kq = grid.kqlo + i; + const Real q = Real(kq * grid.qstep); + const int first = blockIdx.y * SUBCHUNK; + const int end = min(first + SUBCHUNK, count); + double sum = 0.0; + for (int j = first; j < end; ++j) { + if (kq >= pairs[j].kqstop) continue; + const auto& p = pairs[j]; + const Real sq = p.sigma * q; + sum += p.amplitude * exp(Real(-0.5) * sq * sq) * + sf[(size_t)p.type0 * span + i] * sf[(size_t)p.type1 * span + i] * + debyeSine(p.phase, q, kq); + } + partial[(size_t)blockIdx.y * span + i] = sum; +} + +/// Add the sums of the blocks of pairs in their order. +__global__ void finishRows(const double* partial, int cntsub, int span, + double* total) { + const int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= span) return; + double sum = 0.0; + for (int sub = 0; sub < cntsub; ++sub) sum += partial[(size_t)sub * span + i]; + total[i] += sum; +} + +int blocksFor(long long n) { return (int)((n + THREADS - 1) / THREADS); } + +// Sums ------------------------------------------------------------------------ + +/// What the kernels keep on the device between the sums of a calculator. +struct DeviceState : PairSumWorkspace { + const int device = currentDevice(); + bool isCurrentDevice() const override { return device == currentDevice(); } + + // structure + Buffer sitebegin; + Buffer possite; + Buffer pos; + Buffer posuij; + Buffer siteuij; + Buffer anisotropy; + Buffer multiplicity; + Buffer trans; + Buffer pairsite0; + Buffer pairpos; + Buffer pairtrans; + // the list of pairs on the device + const int* listowner = nullptr; + int listversion = -1; + long long listsize = -1; + // peaks + Buffer sfsite; + Buffer total; + // Debye sum + Buffer typeofsite, curve; + Buffer occupancy; + Buffer sf; + Buffer logsf; + DebyeBuffers debye; + DebyeBuffers debyeMixed; + Buffer partial; + + /// Copy the structure to the device. Return the number of its pairs. + long long upload(const PairGeometryArgs& v, PairGeometryArgs& g) { + g = v; + g.sitebegin = sitebegin.upload(v.sitebegin, v.cntsites + 1); + g.possite = possite.upload(v.possite, v.cntpositions); + g.pos = pos.upload(v.pos, 3 * (size_t)v.cntpositions); + g.posuij = posuij.upload(v.posuij, 6 * (size_t)v.cntpositions); + g.siteuij = siteuij.upload(v.siteuij, 6 * (size_t)v.cntsites); + g.anisotropy = anisotropy.upload(v.anisotropy, v.cntsites); + g.multiplicity = multiplicity.upload(v.multiplicity, v.cntsites); + g.trans = trans.upload(v.trans, 3 * (size_t)v.cnttrans); + if (!v.translations) { + listversion = -1; + g.pairsite0 = g.pairpos = g.pairtrans = nullptr; + return pairCount(v); + } + // The list stays on the device for the structures that keep it. + const bool same = (listowner == v.pairsite0) && + (listversion == v.pairversion) && + (listsize == v.cntpairs); + if (!same) { + pairsite0.upload(v.pairsite0, v.cntpairs); + pairpos.upload(v.pairpos, v.cntpairs); + pairtrans.upload(v.pairtrans, v.cntpairs); + listowner = v.pairsite0; + listversion = v.pairversion; + listsize = v.cntpairs; + } + g.pairsite0 = pairsite0.get(); + g.pairpos = pairpos.get(); + g.pairtrans = pairtrans.get(); + return v.cntpairs; + } +}; + +template +void addPDFPairs(const PDFPairSumArgs& job) { + if (job.npts <= 0 || !pairCount(job.g)) return; + auto& m = static_cast(*job.workspace); + PairGeometryArgs g; + const long long cntpairs = m.upload(job.g, g); + const double* sfsite = m.sfsite.upload(job.sfsite, job.g.cntsites); + const PeakGrid grid = {job.npts, job.rlosteps, job.rstep, + job.height, job.cropped, job.halfboundrel}; + // Bound temporary storage to 32 MiB or four rows for a very long grid. + const int rowLimit = std::max(4, std::min(4096, (4 << 20) / job.npts)); + const int blocks = int(std::min((cntpairs + THREADS - 1) / THREADS, + rowLimit / (THREADS / WARP))); + const int rows = blocks * (THREADS / WARP); + double* partial = m.partial.zero((size_t)rows * job.npts); + double* total = m.total.zero(job.npts); + pdfPeaks + <<>>(g, job.w, sfsite, grid, cntpairs, partial); + check(cudaGetLastError(), "the start of a PDF kernel"); + finishRows<<>>(partial, rows, job.npts, total); + check(cudaGetLastError(), "the reduction of the PDF"); + std::vector result(job.npts); + m.total.download(result.data(), job.npts); + if (std::any_of(result.begin(), result.end(), + [](double x) { return !std::isfinite(x); })) + throw std::runtime_error("nonfinite PDF sum on the CUDA device"); + for (int i = 0; i < job.npts; ++i) job.out[i] += result[i]; +} + +template +void addDebyePairs(const DebyePairSumArgs& job, DeviceState& m, + DebyeBuffers& work) { + const int span = job.kqhi - job.kqlo; + if (span <= 0) return; + PairGeometryArgs g; + const long long cntpairs = m.upload(job.g, g); + if (!cntpairs) return; + const QGrid grid = {job.nq, job.kqlo, job.kqhi, job.qstep, job.prec}; + const int* typeofsite = m.typeofsite.upload(job.typeofsite, job.g.cntsites); + const int* curve = m.curve.upload(job.curve, job.ntypes); + const double* occ = m.occupancy.upload(job.occupancy, job.ntypes); + const size_t sfsize = (size_t)job.ncurves * job.nq; + const double* sf = m.sf.upload(job.sf, sfsize); + std::vector logsf(sfsize); + for (size_t i = 0; i < sfsize; ++i) { + logsf[i] = (float)std::log(std::fabs(job.sf[i])); + } + const float* dlogsf = m.logsf.upload(logsf.data(), logsf.size()); + const size_t chunk = (size_t)std::min(CHUNK, std::max(cntpairs, 1LL)); + auto* pairs = work.pairs.reserve(chunk); + const size_t maxsub = (chunk + SUBCHUNK - 1) / SUBCHUNK; + // Scratch is bounded even when each site has its own occupancy. + const int tileSize = std::min(span, std::max(1, (1 << 20) / job.ntypes)); + Real* weighted = work.factors.reserve((size_t)tileSize * job.ntypes); + double* partial = m.partial.reserve(maxsub * tileSize); + double* total = m.total.zero(span); + for (long long first = 0; first < cntpairs; first += CHUNK) { + const long long count = std::min(CHUNK, cntpairs - first); + const int cntsub = (int)((count + SUBCHUNK - 1) / SUBCHUNK); + debyePairs<<>>( + g, job.w, typeofsite, curve, occ, sf, dlogsf, grid, first, count, pairs); + check(cudaGetLastError(), "the start of a kernel"); + for (int lo = grid.kqlo; lo < grid.kqhi;) { + const int n = std::min(tileSize, grid.kqhi - lo); + QGrid tile = grid; + tile.kqlo = lo; + tile.kqhi = lo + n; + const dim3 factorBlocks(blocksFor(n), std::min(job.ntypes, 65535)); + occupiedFactors<<>>(sf, curve, occ, tile, + job.ntypes, weighted); + check(cudaGetLastError(), "the occupied scattering factors"); + const dim3 blocks(blocksFor(n), cntsub); + debyeTerms<<>>(pairs, weighted, tile, (int)count, + partial); + check(cudaGetLastError(), "the start of a kernel"); + finishRows<<>>(partial, cntsub, n, + total + (lo - grid.kqlo)); + check(cudaGetLastError(), "the reduction of the Debye sum"); + lo += n; + } + } + check(cudaDeviceSynchronize(), "the Debye sum"); + std::vector result(span); + m.total.download(result.data(), span); + for (int i = 0; i < span; ++i) job.out[job.kqlo + i] += result[i]; +} + +} // namespace + +// Public Functions ------------------------------------------------------------ + +const char* cudaKernelUnavailable() { + // Loading an actual kernel checks the binary or PTX image against the + // calling thread's current device and driver without launching any work. + cudaFuncAttributes attributes; + const cudaError_t err = cudaFuncGetAttributes(&attributes, pdfPeaks); + return err == cudaSuccess ? nullptr : errorDescription(err); +} + +PairSumStats cudaPDFPairSum(const PDFPairSumArgs& a) { + addPDFPairs(a); + return {}; +} + +PairSumStats cudaPDFPairSumMixed(const PDFPairSumArgs& a) { + addPDFPairs(a); + return {}; +} + +PairSumStats cudaDebyePairSum(const DebyePairSumArgs& a) { + auto& state = static_cast(*a.workspace); + addDebyePairs(a, state, state.debye); + return {}; +} + +PairSumStats cudaDebyePairSumMixed(const DebyePairSumArgs& a) { + auto& state = static_cast(*a.workspace); + addDebyePairs(a, state, state.debyeMixed); + return {}; +} + +PairSumWorkspace* makeCudaWorkspace() { return new DeviceState; } + +} // namespace kernels +} // namespace srreal +} // namespace diffpy + +// End of file diff --git a/src/diffpy/srreal/kernels/CalculationKernelDispatch.cpp b/src/diffpy/srreal/kernels/CalculationKernelDispatch.cpp index 7a94175..0cdabe5 100644 --- a/src/diffpy/srreal/kernels/CalculationKernelDispatch.cpp +++ b/src/diffpy/srreal/kernels/CalculationKernelDispatch.cpp @@ -20,7 +20,7 @@ #include #if defined(LIBDIFFPY_SIMD_KERNELS) -#include +#include #endif #include @@ -36,7 +36,7 @@ namespace { /// Kernels of this build that the CPU can run, from the most to the least /// preferred. CMake defines LIBDIFFPY_KERNEL_ for each SIMD kernel it /// compiles. The scalar kernel is last and always available. -const std::vector& availableKernels() { +const std::vector& cpuKernels() { static const std::vector available = [] { std::vector rv; #if defined(LIBDIFFPY_SIMD_KERNELS) @@ -63,9 +63,66 @@ const std::vector& availableKernels() { return available; } -/// The active kernel, the most preferred one is selected on first use. +// Kernel tables have process lifetime. Availability is checked on the +// current device whenever a caller lists or explicitly selects CUDA. +const char* cudaUnavailable() { +#if defined(LIBDIFFPY_KERNEL_CUDA) + return kernels::cudaKernelUnavailable(); +#else + return "libdiffpy was built without CUDA"; +#endif +} + +const std::vector& cudaKernels() { + static const std::vector cuda = [] { + std::vector rv; +#if defined(LIBDIFFPY_KERNEL_CUDA) + const CalculationKernel& cpu = *cpuKernels().front(); + static const CalculationKernel doubles = { + "cuda", + cpu.gaussianWindow, + cpu.debyeSum, + cpu.bondCandidates, + kernels::cudaPDFPairSum, + kernels::cudaDebyePairSum, + kernels::makeCudaWorkspace, + true, + }; + static const CalculationKernel mixed = { + "cuda-mixed", + cpu.gaussianWindow, + cpu.debyeSum, + cpu.bondCandidates, + kernels::cudaPDFPairSumMixed, + kernels::cudaDebyePairSumMixed, + kernels::makeCudaWorkspace, + true, + }; + rv = {&doubles, &mixed}; +#endif + return rv; + }(); + return cuda; +} + +bool isCudaName(const std::string& name) { + return name == "cuda" || name == "cuda-mixed"; +} + +/// All kernels that setCalculationKernel takes. +std::vector availableKernels() { + std::vector rv = cpuKernels(); + if (!cudaUnavailable()) { + const auto& cuda = cudaKernels(); + rv.insert(rv.end(), cuda.begin(), cuda.end()); + } + return rv; +} + +/// The active kernel, the most preferred one of the CPU is selected on +/// first use. std::atomic& activeKernelSlot() { - static std::atomic slot{availableKernels().front()}; + static std::atomic slot{cpuKernels().front()}; return slot; } @@ -85,14 +142,22 @@ std::string getCalculationKernel() { } void setCalculationKernel(const std::string& name) { - const auto& available = availableKernels(); + // No device is looked for when the name is one of the CPU. + const bool cuda = isCudaName(name); + if (cuda) { + if (const char* reason = cudaUnavailable()) { + throw std::invalid_argument("Calculation kernel '" + name + + "' is not available: " + reason + "."); + } + } + const auto& available = cuda ? cudaKernels() : cpuKernels(); const auto it = std::ranges::find_if( available, [&](const CalculationKernel* k) { return name == k->name; }); if (it == available.end()) { std::string names; - for (const CalculationKernel* k : available) { + for (const std::string& kname : availableCalculationKernels()) { if (!names.empty()) names += ", "; - names += k->name; + names += kname; } throw std::invalid_argument("Calculation kernel '" + name + "' is not available, use one of: " + names + @@ -107,6 +172,11 @@ const CalculationKernel& activeCalculationKernel() { return *activeKernelSlot().load(std::memory_order_relaxed); } +const CalculationKernel& pairSumKernel(bool cpuOnly) { + const auto& kernel = activeCalculationKernel(); + return cpuOnly && kernel.onDevice ? *cpuKernels().front() : kernel; +} + } // namespace kernels } // namespace srreal diff --git a/src/diffpy/srreal/kernels/CalculationKernelImpl.hpp b/src/diffpy/srreal/kernels/CalculationKernelImpl.hpp index 8e1a3d5..517ed8a 100644 --- a/src/diffpy/srreal/kernels/CalculationKernelImpl.hpp +++ b/src/diffpy/srreal/kernels/CalculationKernelImpl.hpp @@ -22,6 +22,7 @@ namespace diffpy { namespace srreal { +class PairQuantity; namespace kernels { /// Add one Gaussian peak to the PDFCalculator RDF array. @@ -30,8 +31,9 @@ namespace kernels { /// xlo = (rlosteps + ilo) * rstep - dist: /// out[i] += scale * G(x, fwhm) * (x / dist + 1) /// where G(x, fwhm) = peakheight * exp(-4 * ln(2) * (x / fwhm)^2), or zero -/// for |x / fwhm| >= halfboundrel when cropped. peakheight is the profile -/// value at x = 0. Nothing is added for fwhm <= 0. +/// outside the cropped support. The cutoff uses the absolute grid offset +/// (rlosteps + i) * rstep - dist. peakheight is the profile value at x = 0. +/// Nothing is added for fwhm <= 0. struct GaussianWindowArgs { double* out; int ilo; @@ -46,13 +48,15 @@ struct GaussianWindowArgs { double halfboundrel; }; -/// Add one pair to the BaseDebyeSum F(Q) array. +/// Add count equal pairs to the BaseDebyeSum F(Q) array. /// /// For kq = kqlo, kqlo + 1, ... while kq < kqhi: /// q = kq * qstep /// t = scale * exp(-0.5 * (dwsigma * q)^2) * sf0[kq] * sf1[kq] / dist /// stop at the first kq where |t| <= prec -/// out[kq] += t * sin(q * dist) +/// out[kq] += count * t * sin(q * dist) +/// The sum stops where it does for one pair, so that count pairs add +/// count times what one pair adds. struct DebyeSumArgs { double* out; int kqlo; @@ -64,6 +68,8 @@ struct DebyeSumArgs { double prec; const double* sf0; const double* sf1; + double count = 1.0; + double occ0 = 1.0, occ1 = 1.0; }; /// Select candidate bonds from n points by their squared distance. @@ -86,17 +92,140 @@ struct BondCandidatesArgs { int* idx; }; +/// Positions and pairs of a structure for a sum over all its pairs, as +/// PairTable gives them. The arrays stay with the caller. +struct PairGeometryArgs { + int cntsites; + int cntpositions; + int cnttrans; + /// number of entries in the list of pairs, unused without translations + long long cntpairs; + /// The pairs are those of the list below, of positions with lattice + /// translations. Otherwise they are all pairs of two sites. + int translations; + /// Positions of site i are [sitebegin[i], sitebegin[i + 1]). A pair + /// starts at the first position of its site0. + const int* sitebegin; + /// site of each position + const int* possite; + /// Cartesian coordinates x, y, z of each position + const double* pos; + /// Displacement tensors of the positions and of the sites, each as its + /// elements xx, yy, zz, xy, xz, yz. + const double* posuij; + const double* siteuij; + const unsigned char* anisotropy; + const int* multiplicity; + /// Cartesian coordinates of the lattice translations + const double* trans; + /// Site of the first position, second position and translation of each + /// pair in the list. + const int* pairsite0; + const int* pairpos; + const int* pairtrans; + /// changes when the list of pairs does + int pairversion; + /// The pairs are those of rmin <= distance <= rmax and of a distance + /// above zeroeps. + double rmin; + double rmax; + double zeroeps; +}; + +/// Built-in peak width model and its parameters. +struct PairWidthArgs { + enum Kind { Jeong, DebyeWaller, Constant } kind; + double delta1; + double delta2; + double qbroad; + double qbroadsep; + double width; +}; + +/// Counts used to choose between a CPU batch and an incremental update. +struct PairSumStats { + long long pairs = 0; + long long peaks = 0; +}; + +/// Owned by the calculator and reused by kernels with the same backend. +struct PairSumWorkspace { + virtual ~PairSumWorkspace() = default; + virtual bool isCurrentDevice() const { return true; } +}; +using MakeWorkspace = PairSumWorkspace* (*)(); + +struct PairSumArgs { + PairGeometryArgs g; + PairWidthArgs w; + double* out; + PairSumWorkspace* workspace; + /// Only the CPU reads the mask. A null pointer includes every pair. + const PairQuantity* mask; +}; + +/// Add the peaks of PDFCalculator for all pairs of a structure, what +/// gaussianWindow adds for each of them. +struct PDFPairSumArgs : PairSumArgs { + /// scattering factor of each site with its occupancy + const double* sfsite; + /// The peaks are added to the npts values at out, whose first one is at + /// r = rlosteps * rstep. + int npts; + int rlosteps; + double rstep; + /// height of the peak profile for the width of one + double height; + int cropped; + double halfboundrel; +}; + +/// Add the terms of BaseDebyeSum for all pairs of a structure, what +/// debyeSum adds for each of them. +struct DebyePairSumArgs : PairSumArgs { + /// Each site type maps to one scattering curve and an occupancy. + /// sf contains ncurves consecutive rows of nq unweighted factors. + const int* typeofsite; + const int* curve; + const double* occupancy; + int ntypes, ncurves; + int nq; + const double* sf; + /// The terms are added to the values of kqlo <= kq < kqhi at out. + int kqlo; + int kqhi; + double qstep; + double prec; +}; + +using GaussianWindow = void (*)(const GaussianWindowArgs&); +using DebyeSum = void (*)(const DebyeSumArgs&); + struct CalculationKernel { /// Name for setCalculationKernel, a string literal. const char* name; - void (*gaussianWindow)(const GaussianWindowArgs&); - void (*debyeSum)(const DebyeSumArgs&); + GaussianWindow gaussianWindow; + DebyeSum debyeSum; int (*bondCandidates)(const BondCandidatesArgs&); + // Sums over all pairs of a structure. + // The output is updated only after the sum succeeds. + PairSumStats (*pdfPairSum)(const PDFPairSumArgs&); + PairSumStats (*debyePairSum)(const DebyePairSumArgs&); + MakeWorkspace makeWorkspace; + bool onDevice = false; }; /// The active kernel, exported for the unit tests. DLL_EXPORT const CalculationKernel& activeCalculationKernel(); +/// Use the selected CPU kernel for CPU-only features, or the preferred CPU +/// kernel when the active kernel is on a device. +const CalculationKernel& pairSumKernel(bool cpuOnly); + +PairSumStats cpuPDFPairSum(const PDFPairSumArgs&, GaussianWindow); +PairSumStats cpuDebyePairSum(const DebyePairSumArgs&, DebyeSum); +PairSumWorkspace* makeCPUWorkspace(); + // Kernel tables per instruction set. The SIMD ones exist only when CMake // compiles them, see cmake/CalculationKernels.cmake. const CalculationKernel& scalarCalculationKernel(); @@ -105,6 +234,31 @@ const CalculationKernel& avx2CalculationKernel(); const CalculationKernel& avx512CalculationKernel(); const CalculationKernel& neonCalculationKernel(); +/// Bind both the per-pair and batch entry points to the same instruction set. +template +constexpr CalculationKernel makeCPUKernel( + const char* name, int (*candidates)(const BondCandidatesArgs&)) { + return {name, + Gaussian, + Debye, + candidates, + [](const PDFPairSumArgs& a) { return cpuPDFPairSum(a, Gaussian); }, + [](const DebyePairSumArgs& a) { return cpuDebyePairSum(a, Debye); }, + makeCPUWorkspace}; +} + +// Functions of the CUDA kernels, which exist only when CMake compiles +// them, see cmake/Cuda.cmake. The Mixed ones add in single precision +// where the result keeps six digits. + +/// Null when a CUDA device can run the pair sums, otherwise why it cannot. +const char* cudaKernelUnavailable(); +PairSumStats cudaPDFPairSum(const PDFPairSumArgs&); +PairSumStats cudaPDFPairSumMixed(const PDFPairSumArgs&); +PairSumStats cudaDebyePairSum(const DebyePairSumArgs&); +PairSumStats cudaDebyePairSumMixed(const DebyePairSumArgs&); +PairSumWorkspace* makeCudaWorkspace(); + } // namespace kernels } // namespace srreal } // namespace diffpy diff --git a/src/diffpy/srreal/kernels/CalculationKernelSIMD.ipp b/src/diffpy/srreal/kernels/CalculationKernelSIMD.ipp index 620e1fd..cef41d4 100644 --- a/src/diffpy/srreal/kernels/CalculationKernelSIMD.ipp +++ b/src/diffpy/srreal/kernels/CalculationKernelSIMD.ipp @@ -47,7 +47,7 @@ #include #include -#include +#include // The xsimd headers and all code below use the renamed namespace. #define xsimd LIBDIFFPY_XSIMD_NAMESPACE @@ -152,58 +152,36 @@ struct SIMDKernel { const B twowh(2 * wh); const B whwh(wh * wh); const B ratiostep(exp(-2 * b * wh * wh)); - const B vw(1.0 * W); const B vrstep(a.rstep); const B vxlo(xlo); const B vinvdist(1.0 / a.dist); const B vnorm(a.scale * a.peakheight); - const B vfwhm(a.fwhm); - const B vhalfbound(a.halfboundrel); const B one(1.0); - const B zero(0.0); - // Grid steps of the lanes from ilo, whole numbers are exact in double. - B steps = laneIndex(); + int first = a.ilo, last = a.ihi; + if (a.cropped) + cropGaussianWindow(first, last, a.rlosteps, a.rstep, a.dist, a.fwhm, + a.halfboundrel); + B steps = B(double(first - a.ilo)) + laneIndex(); // x / dist + 1 grows by a constant per vector step. const B vrscalestep(wh / a.dist); - for (int i = a.ilo; i < a.ihi;) { - const int iend = blockEnd(i, a.ihi); + for (int i = first; i < last;) { + const int iend = blockEnd(i, last); const int ivec = wholeEnd(i, iend); - // Same x as the scalar kernel, so cropping agrees. const B xseed = steps * vrstep + vxlo; B g = vnorm * xsimd::exp(negb * xseed * xseed); B ratio = xsimd::exp(negb * (twowh * xseed + whwh)); - if (a.cropped) { - for (; i < ivec; i += W) { - const B x = steps * vrstep + vxlo; - const B y = - xsimd::select(xsimd::abs(x / vfwhm) >= vhalfbound, zero, g); - addProduct(a.out + i, y, xsimd::fma(x, vinvdist, one)); - g *= ratio; - ratio *= ratiostep; - steps += vw; - } - if (i < iend) { - const B x = steps * vrstep + vxlo; - const B y = - xsimd::select(xsimd::abs(x / vfwhm) >= vhalfbound, zero, g); - addProductPartial(a.out + i, y, xsimd::fma(x, vinvdist, one), - iend - i); - return; - } - } else { - B rscaled = xsimd::fma(xseed, vinvdist, one); - // Advance before the loop, the next seeds then do not wait for it. - steps += B(double(ivec - i)); - for (; i < ivec; i += W) { - addProduct(a.out + i, g, rscaled); - g *= ratio; - ratio *= ratiostep; - rscaled += vrscalestep; - } - if (i < iend) { - addProductPartial(a.out + i, g, rscaled, iend - i); - return; - } + B rscaled = xsimd::fma(xseed, vinvdist, one); + // Advance before the loop, the next seeds then do not wait for it. + steps += B(double(ivec - i)); + for (; i < ivec; i += W) { + addProduct(a.out + i, g, rscaled); + g *= ratio; + ratio *= ratiostep; + rscaled += vrscalestep; + } + if (i < iend) { + addProductPartial(a.out + i, g, rscaled, iend - i); + return; } } } @@ -225,6 +203,7 @@ struct SIMDKernel { const B vdwsigma(a.dwsigma); const B vsinescale(a.scale / a.dist); const B vprec(a.prec); + const B vcount(a.count); const B neghalf(-0.5); const B lanes = laneIndex(); for (int kq = a.kqlo; kq < a.kqhi;) { @@ -237,9 +216,14 @@ struct SIMDKernel { B g = vsinescale * xsimd::exp(neghalf * dwq * dwq); B ratio = xsimd::exp(negalpha * (twow * k + ww)); auto [s, c] = xsimd::sincos(q * vdist); + // The pairs are counted in the sine, which leaves the terms for + // their comparison with the precision. The rotation keeps the + // scale of (sin, cos). + s *= vcount; + c *= vcount; for (; kq < kqvec; kq += W) { - const B t = - g * B::load_unaligned(a.sf0 + kq) * B::load_unaligned(a.sf1 + kq); + const B t = g * (B::load_unaligned(a.sf0 + kq) * B(a.occ0)) * + (B::load_unaligned(a.sf1 + kq) * B(a.occ1)); const BB small = xsimd::abs(t) <= vprec; if (xsimd::any(small)) { // Keep the lanes before the first small term. @@ -256,7 +240,8 @@ struct SIMDKernel { if (kq < kqend) { const int n = kqend - kq; // Unused lanes have zero t and do not reach the output. - B t = g * loadPartial(a.sf0 + kq, n) * loadPartial(a.sf1 + kq, n); + B t = g * (loadPartial(a.sf0 + kq, n) * B(a.occ0)) * + (loadPartial(a.sf1 + kq, n) * B(a.occ1)); const BB small = xsimd::abs(t) <= vprec; if (xsimd::any(small)) t = truncated(t, small); addProductPartial(a.out + kq, t, s, n); @@ -304,7 +289,7 @@ struct SIMDKernel { template constexpr CalculationKernel makeSIMDKernel(const char* name) { using K = SIMDKernel; - return {name, K::gaussianWindow, K::debyeSum, K::bondCandidates}; + return makeCPUKernel(name, K::bondCandidates); } } // namespace diff --git a/src/diffpy/srreal/kernels/CalculationKernelScalar.cpp b/src/diffpy/srreal/kernels/CalculationKernelScalar.cpp index b0a1f80..01dddad 100644 --- a/src/diffpy/srreal/kernels/CalculationKernelScalar.cpp +++ b/src/diffpy/srreal/kernels/CalculationKernelScalar.cpp @@ -16,7 +16,7 @@ #include #include -#include +#include namespace diffpy { namespace srreal { @@ -26,11 +26,14 @@ namespace { void gaussianWindowScalar(const GaussianWindowArgs& a) { if (a.fwhm <= 0.0) return; - const double xlo = (a.rlosteps + a.ilo) * a.rstep - a.dist; - for (int i = a.ilo; i < a.ihi; ++i) { + int first = a.ilo, last = a.ihi; + if (a.cropped) + cropGaussianWindow(first, last, a.rlosteps, a.rstep, a.dist, a.fwhm, + a.halfboundrel); + const double xlo = (double(a.rlosteps) + a.ilo) * a.rstep - a.dist; + for (int i = first; i < last; ++i) { const double x = xlo + (i - a.ilo) * a.rstep; const double xrel = x / a.fwhm; - if (a.cropped && fabs(xrel) >= a.halfboundrel) continue; const double y = a.peakheight * exp(-4 * std::numbers::ln2 * xrel * xrel); // Normalized by dist instead of r, see PeakProfile::addRDFPeak. const double yrdf = y * (x / a.dist + 1); @@ -43,9 +46,10 @@ void debyeSumScalar(const DebyeSumArgs& a) { const double q = kq * a.qstep; const double dwq = a.dwsigma * q; const double dwscale = exp(-0.5 * dwq * dwq); - const double sinescale = a.scale * dwscale * a.sf0[kq] * a.sf1[kq] / a.dist; + const double sinescale = (a.scale / a.dist) * dwscale * + (a.sf0[kq] * a.occ0) * (a.sf1[kq] * a.occ1); if (fabs(sinescale) <= a.prec) break; - a.out[kq] += sinescale * sin(q * a.dist); + a.out[kq] += a.count * sinescale * sin(q * a.dist); } } @@ -66,8 +70,9 @@ int bondCandidatesScalar(const BondCandidatesArgs& a) { } // namespace const CalculationKernel& scalarCalculationKernel() { - static constexpr CalculationKernel kernel = { - "scalar", gaussianWindowScalar, debyeSumScalar, bondCandidatesScalar}; + static constexpr auto kernel = + makeCPUKernel("scalar", + bondCandidatesScalar); return kernel; } diff --git a/src/diffpy/srreal/kernels/PairGeometry.hpp b/src/diffpy/srreal/kernels/PairGeometry.hpp new file mode 100644 index 0000000..3be414c --- /dev/null +++ b/src/diffpy/srreal/kernels/PairGeometry.hpp @@ -0,0 +1,168 @@ +/***************************************************************************** + * + * libdiffpy Complex Modeling Initiative + * (c) 2026 The Trustees of Columbia University + * in the City of New York. All rights reserved. + * + * See AUTHORS.rst for a list of people who contributed. + * See LICENSE.rst for license information. + * + *****************************************************************************/ + +#ifndef DIFFPY_KERNELS_PAIRGEOMETRY_HPP +#define DIFFPY_KERNELS_PAIRGEOMETRY_HPP + +#include +#include + +#include + +#ifdef __CUDACC__ +#define DIFFPY_HOST_DEVICE __host__ __device__ +#else +#define DIFFPY_HOST_DEVICE +#endif + +namespace diffpy { +namespace srreal { +namespace kernels { + +constexpr double PAIR_SIGMA_TO_FWHM = 2.3548200450309493; + +/// Use the per-bond rounding order for the Debye precision cutoff. +DIFFPY_HOST_DEVICE inline double fwhmToSigma(double fwhm) { + constexpr double scale = 1.0 / PAIR_SIGMA_TO_FWHM; + return fwhm * scale; +} + +inline long long pairCount(const PairGeometryArgs& g) { + return g.translations ? g.cntpairs + : (long long)g.cntsites * (g.cntsites - 1) / 2; +} + +/// Clip before converting to an integer, including for very wide peaks. +DIFFPY_HOST_DEVICE inline int gridIndex(double r, double step, int lo, + int offset, int npts) { + const double i = floor(r / step) - lo + offset; + return i > 0 ? (i < npts ? int(i) : npts) : 0; +} + +/// Crop against the given r-grid. Reconstructing x from the start of +/// the window can round a boundary sample to the other side of the cutoff. +DIFFPY_HOST_DEVICE static inline void cropGaussianWindow( + int& first, int& last, int rlosteps, double rstep, double dist, double fwhm, + double halfboundrel) { + auto outside = [&](int i) { + const double x = (double(rlosteps) + i) * rstep - dist; + return fabs(x / fwhm) >= halfboundrel; + }; + while (first < last && outside(first)) ++first; + while (first < last && outside(last - 1)) --last; +} + +struct Pair { + bool ok; + int site0; + int site1; + double dist; + double msd; + unsigned weight; +}; + +/// As meanSquareDisplacement for the tensor xx, yy, zz, xy, xz, yz. +DIFFPY_HOST_DEVICE inline double siteMSD(const double* U, double rx, double ry, + double rz, int anisotropy) { + if (!anisotropy) return U[0]; + const double sUs = U[0] * rx * rx + U[1] * ry * ry + U[2] * rz * rz + + 2 * (U[3] * rx * ry + U[4] * rx * rz + U[5] * ry * rz); + return sUs / (rx * rx + ry * ry + rz * rz); +} + +/// Coordinates of a pair in the strict lower triangle of a cluster. +struct ClusterPair { + int site0, site1; + + DIFFPY_HOST_DEVICE static ClusterPair at(long long k) { + // k = i0 * (i0 - 1) / 2 + i1 with i1 < i0. + long long row = (long long)((1.0 + sqrt(1.0 + 8.0 * (double)k)) / 2); + while (row * (row - 1) / 2 > k) --row; + while ((row + 1) * row / 2 <= k) ++row; + return {int(row), int(k - row * (row - 1) / 2)}; + } + + void next() { + if (++site1 == site0) { + ++site0; + site1 = 0; + } + } +}; + +DIFFPY_HOST_DEVICE inline Pair pairAt(const PairGeometryArgs& g, long long k, + ClusterPair cluster) { + Pair rv{}; + int p1 = cluster.site1; + rv.site0 = cluster.site0; + double tx = 0.0, ty = 0.0, tz = 0.0; + if (g.translations) { + rv.site0 = g.pairsite0[k]; + p1 = g.pairpos[k]; + const double* t = g.trans + 3LL * g.pairtrans[k]; + tx = t[0]; + ty = t[1]; + tz = t[2]; + } + rv.site1 = g.possite[p1]; + const double* r0 = g.pos + 3LL * g.sitebegin[rv.site0]; + const double* r1 = g.pos + 3LL * p1; + const double rx = r1[0] + tx - r0[0]; + const double ry = r1[1] + ty - r0[1]; + const double rz = r1[2] + tz - r0[2]; + rv.dist = sqrt(rx * rx + ry * ry + rz * rz); + rv.ok = !(rv.dist < g.rmin || rv.dist > g.rmax || rv.dist <= g.zeroeps); + if (!rv.ok) return rv; + rv.msd = + siteMSD(g.siteuij + 6LL * rv.site0, rx, ry, rz, g.anisotropy[rv.site0]) + + siteMSD(g.posuij + 6LL * p1, rx, ry, rz, g.anisotropy[rv.site1]); + const unsigned m = g.multiplicity[rv.site0]; + rv.weight = (g.translations && rv.site0 == rv.site1) ? m : 2 * m; + return rv; +} + +/// Random access for independent CUDA threads. +DIFFPY_HOST_DEVICE inline Pair pairAt(const PairGeometryArgs& g, long long k) { + return pairAt(g, k, g.translations ? ClusterPair{} : ClusterPair::at(k)); +} + +DIFFPY_HOST_DEVICE inline double debyeWallerWidth(double msd) { + return msd < 0.0 ? 0.0 : PAIR_SIGMA_TO_FWHM * sqrt(msd); +} + +DIFFPY_HOST_DEVICE inline double sharpeningRatio(double r, double delta1, + double delta2, double qbroad) { + if (r < DBL_EPSILON) return 0.0; + const double qr = qbroad * r; + return 1.0 - delta1 / r - delta2 / (r * r) + qr * qr; +} + +DIFFPY_HOST_DEVICE inline double sharpenedWidth(double r, double corr, + double msd, double qbroadsep) { + const double qr = qbroadsep * r; + return sqrt(corr) * debyeWallerWidth(msd) + qr * qr; +} + +/// FWHM of the built-in peak width models for one pair. +DIFFPY_HOST_DEVICE inline double widthOf(const PairWidthArgs& w, double dist, + double msd) { + if (w.kind == PairWidthArgs::Constant) return w.width; + if (w.kind == PairWidthArgs::DebyeWaller) return debyeWallerWidth(msd); + const double corr = sharpeningRatio(dist, w.delta1, w.delta2, w.qbroad); + return corr <= 0.0 ? 0.0 : sharpenedWidth(dist, corr, msd, w.qbroadsep); +} + +} // namespace kernels +} // namespace srreal +} // namespace diffpy + +#undef DIFFPY_HOST_DEVICE +#endif diff --git a/src/diffpy/srreal/kernels/PairTable.cpp b/src/diffpy/srreal/kernels/PairTable.cpp new file mode 100644 index 0000000..6269bbe --- /dev/null +++ b/src/diffpy/srreal/kernels/PairTable.cpp @@ -0,0 +1,336 @@ +/***************************************************************************** + * + * libdiffpy Complex Modeling Initiative + * (c) 2026 The Trustees of Columbia University + * in the City of New York. All rights reserved. + * + * See AUTHORS.rst for a list of people who contributed. + * See LICENSE.rst for license information. + * + ****************************************************************************** + * + * class PairTable -- all pairs of a structure for a sum over them + * + *****************************************************************************/ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace diffpy { +namespace srreal { +namespace kernels { + +namespace { + +/// The pairs of a periodic structure are searched for distances up to this +/// much beyond the r-range. They stay valid while the positions and the +/// lattice change the distances by less. +const double SKIN = 0.25; + +bool isPeriodicClass(const StructureAdapter& s) { + return typeid(s) == typeid(PeriodicStructureAdapter) || + typeid(s) == typeid(CrystalStructureAdapter); +} + +} // namespace + +// Public Methods ------------------------------------------------------------ + +PairTable::View PairTable::view(const StructureAdapter& stru) { + View rv = {&stru, nullptr, nullptr}; + while (true) { + const StructureAdapter& s = *rv.source; + if (typeid(s) == typeid(NoMetaStructureAdapter)) { + const auto& nm = static_cast(s); + rv.source = nm.getSourceStructure().get(); + } else if (typeid(s) == typeid(NoSymmetryStructureAdapter)) { + const auto& ns = static_cast(s); + if (!rv.cluster) rv.cluster = &s; + rv.source = ns.getSourceStructure().get(); + } else { + break; + } + } + if (isPeriodicClass(*rv.source)) { + rv.periodic = static_cast(rv.source); + } + return rv; +} + +bool PairTable::supports(const StructureAdapter& stru) { + const View v = view(stru); + // The empty structure has no pairs whatever its class. + if (stru.countSites() == 0) return true; + if (typeid(*v.source) == typeid(AtomicStructureAdapter)) return true; + return v.periodic; +} + +// Positions, translations and pairs of the structure for the range. + +void PairTable::preparePeriodic(const PeriodicStructureAdapter& stru, + double rmin, double rmax) { + Cell& cell = mcell; + fillCell(stru, cell); + if (!this->hasPairsFor(stru, cell, rmin, rmax)) { + this->searchPairs(stru, cell, rmin, rmax); + } + // site of each position + const int cntsites = stru.countSites(); + mpossite.resize(cell.pos.size()); + for (int i = 0; i < cntsites; ++i) { + for (int p = cell.sitebegin[i]; p < cell.sitebegin[i + 1]; ++p) { + mpossite[p] = i; + } + } +} + +PairGeometryArgs PairTable::geometryArgs(const StructureAdapter& stru, + double rmin, double rmax) { + PairGeometryArgs rv = {}; + Geometry& g = mgeometry; + const View v = view(stru); + const StructureAdapter& sites = v.sites(); + const int cntsites = sites.countSites(); + auto tensor = [](std::vector& out, const R3::Matrix& U) { + for (double u : {U(0, 0), U(1, 1), U(2, 2), U(0, 1), U(0, 2), U(1, 2)}) { + out.push_back(u); + } + }; + rv.translations = v.hasTranslations(); + g.pos.clear(); + g.posuij.clear(); + g.trans.clear(); + if (rv.translations) { + this->preparePeriodic(*v.periodic, rmin, rmax); + for (std::size_t p = 0; p < mcell.pos.size(); ++p) { + g.pos.insert(g.pos.end(), mcell.pos[p].begin(), mcell.pos[p].end()); + tensor(g.posuij, mcell.uij[p]); + } + for (const auto& mno : mmno) { + const auto t = + v.periodic->getLattice().cartesian(R3::Vector(mno[0], mno[1], mno[2])); + g.trans.insert(g.trans.end(), t.begin(), t.end()); + } + rv.cntpairs = mpairsite0.size(); + rv.pairsite0 = mpairsite0.data(); + rv.pairpos = mpairpos.data(); + rv.pairtrans = mpairtrans.data(); + rv.pairversion = msearches; + } else { + mcell.sitebegin.resize(cntsites + 1); + mpossite.resize(cntsites); + for (int i = 0; i < cntsites; ++i) { + mcell.sitebegin[i] = mpossite[i] = i; + const R3::Vector r = sites.siteCartesianPosition(i); + g.pos.insert(g.pos.end(), r.begin(), r.end()); + } + mcell.sitebegin[cntsites] = cntsites; + } + g.siteuij.clear(); + g.anisotropy.resize(cntsites); + g.multiplicity.resize(cntsites); + for (int i = 0; i < cntsites; ++i) { + tensor(g.siteuij, sites.siteCartesianUij(i)); + g.anisotropy[i] = sites.siteAnisotropy(i); + g.multiplicity[i] = sites.siteMultiplicity(i); + } + rv.cntsites = cntsites; + rv.cntpositions = int(mpossite.size()); + rv.cnttrans = int(g.trans.size() / 3); + rv.sitebegin = mcell.sitebegin.data(); + rv.possite = mpossite.data(); + rv.pos = g.pos.data(); + rv.posuij = rv.translations ? g.posuij.data() : g.siteuij.data(); + rv.siteuij = g.siteuij.data(); + rv.anisotropy = g.anisotropy.data(); + rv.multiplicity = g.multiplicity.data(); + rv.trans = g.trans.data(); + rv.rmin = rmin; + rv.rmax = rmax; + // as eps_eq(distance, 0.0) + rv.zeroeps = mathutils::SQRT_DOUBLE_EPS; + return rv; +} + +// The positions of PeriodicStructureBondGenerator and of +// CrystalStructureBondGenerator. + +void PairTable::fillCell(const PeriodicStructureAdapter& stru, Cell& cell) { + const int cntsites = stru.countSites(); + cell.sitebegin.assign(1, 0); + cell.pos.clear(); + cell.uij.clear(); + if (typeid(stru) == typeid(CrystalStructureAdapter)) { + const auto& cstru = static_cast(stru); + for (int i = 0; i < cntsites; ++i) { + for (const Atom& a : cstru.getEquivalentAtoms(i)) { + cell.pos.push_back(a.xyz_cartn); + cell.uij.push_back(a.uij_cartn); + } + cell.sitebegin.push_back(int(cell.pos.size())); + } + return; + } + const Lattice& L = stru.getLattice(); + for (int i = 0; i < cntsites; ++i) { + const Atom& a = stru[i]; + cell.pos.push_back(L.ucvCartesian(a.xyz_cartn)); + cell.uij.push_back(a.uij_cartn); + cell.sitebegin.push_back(int(cell.pos.size())); + } +} + +// Whether the remembered pairs include all pairs of the cell in the range. +// Keep the same cell images as a fresh bond generator at cutoff boundaries. + +bool PairTable::hasPairsFor(const PeriodicStructureAdapter& stru, + const Cell& cell, double rmin, double rmax) const { + if (cell.sitebegin != msitebegin0) return false; + const Lattice& L = stru.getLattice(); + // largest change of a position and of a lattice translation + double dmax = 0.0; + for (std::size_t i = 0; i < mpos0.size(); ++i) { + const double d = R3::distance(cell.pos[i], mpos0[i]); + // NaN makes the pairs invalid + if (!(d <= SKIN)) return false; + if (d > dmax) dmax = d; + } + double tmax = 0.0; + if (L.base() != mbase0) { + // An in-range bond has translation t = r01 - (p1 - p0), even if the + // previous search missed it. Twice the maximum distance from the + // first position bounds |p1 - p0|. + double radius = 0.0; + for (const auto& p : cell.pos) { + radius = std::max(radius, R3::distance(p, cell.pos.front())); + } + // t * strain is the change back to its old translation. The Frobenius + // norm bounds this change for every direction, including in skew cells. + const R3::Matrix strain = R3::prod(L.recbase(), mbase0 - L.base()); + double norm2 = 0.0; + for (double x : strain.data()) norm2 += x * x; + tmax = (rmax + 2 * radius) * std::sqrt(norm2); + } + // No pair that was out of the searched range can be in the range now. + const double change = 2 * dmax + tmax; + const bool lowok = (mrlo0 <= 0.0) || (rmin - change >= mrlo0); + const bool highok = (rmax + change <= mrhi0); + return lowok && highok; +} + +// Search the pairs with the bond generator of the structure. + +void PairTable::searchPairs(const PeriodicStructureAdapter& stru, + const Cell& cell, double rmin, double rmax) { + // An empty site-offset array never matches a cell (even an empty cell has + // the end offset). A failed search must be retried before reusing its lists. + msitebegin0.clear(); + const Lattice& L = stru.getLattice(); + const int cntsites = stru.countSites(); + mrlo0 = rmin - SKIN; + mrhi0 = rmax + SKIN; + mpos0 = cell.pos; + mbase0 = L.base(); + mmno.clear(); + mpairsite0.clear(); + mpairpos.clear(); + mpairtrans.clear(); + // index of each translation by its lattice coordinates + std::map, int> transindex; + auto indexOf = [&](const std::array& mno) { + const auto [it, added] = transindex.try_emplace(mno, int(mmno.size())); + if (added) mmno.push_back(mno); + return it->second; + }; + auto latticeCoordinates = [&](const R3::Vector& t) { + const R3::Vector f = L.fractional(t); + return std::array{int(std::lround(f[0])), int(std::lround(f[1])), + int(std::lround(f[2]))}; + }; + // Keep distinct coincident positions in the candidate list. pairAt tests + // their current distance each time the table is used. + auto owner = stru.createBondGenerator(); + auto& bnds = static_cast(*owner); + bnds.mkeepcoincident = true; + bnds.setRmin(mrlo0); + bnds.setRmax(mrhi0); + std::vector> coincident; + for (int i0 = 0; i0 < cntsites; ++i0) { + auto append = [&](int position, int translation) { + mpairsite0.push_back(i0); + mpairpos.push_back(position); + mpairtrans.push_back(translation); + }; + bnds.selectAnchorSite(i0); + bnds.selectSiteRange(0, i0 + 1); + for (bnds.rewind(); !bnds.finished(); bnds.next()) { + const int p = bnds.mcellindex; + const auto mno = latticeCoordinates(bnds.r1() - cell.pos[p]); + if (p == cell.sitebegin[i0] && mno == std::array{}) continue; + const int t = indexOf(mno); + if (mathutils::eps_eq(bnds.distance(), 0.0)) + coincident.emplace_back(p, t); + else + append(p, t); + } + // Reserve the trailing slots of each anchor for pairs that may separate. + // Interleaving them shifts the regular bonds between CUDA reduction rows. + for (auto [p, t] : coincident) append(p, t); + coincident.clear(); + } + msitebegin0 = cell.sitebegin; + ++msearches; +} + +////////////////////////////////////////////////////////////////////////////// +// Built-in peak width models +////////////////////////////////////////////////////////////////////////////// + +bool assignPairWidth(PairWidthArgs& rv, const PeakWidthModel& model) { + rv = {}; + if (typeid(model) == typeid(JeongPeakWidth)) { + const auto& jeong = static_cast(model); + rv = {PairWidthArgs::Jeong, jeong.getDelta1(), jeong.getDelta2(), + jeong.getQbroad(), jeong.getQbroad_seperable(), 0.0}; + } else if (typeid(model) == typeid(DebyeWallerPeakWidth)) { + rv.kind = PairWidthArgs::DebyeWaller; + } else if (typeid(model) == typeid(ConstantPeakWidth)) { + rv.kind = PairWidthArgs::Constant; + rv.width = static_cast(model).getWidth(); + } else { + return false; + } + return true; +} + +////////////////////////////////////////////////////////////////////////////// +// struct PairSumState +////////////////////////////////////////////////////////////////////////////// + +PairSumWorkspace* PairSumState::workspaceFor(const CalculationKernel& kernel) { + if (mfactory != kernel.makeWorkspace || !mworkspace->isCurrentDevice()) { + mworkspace.reset(kernel.makeWorkspace()); + mfactory = kernel.makeWorkspace; + } + return mworkspace.get(); +} + +} // namespace kernels +} // namespace srreal +} // namespace diffpy + +// End of file diff --git a/src/diffpy/srreal/kernels/PairTable.hpp b/src/diffpy/srreal/kernels/PairTable.hpp new file mode 100644 index 0000000..e2d799a --- /dev/null +++ b/src/diffpy/srreal/kernels/PairTable.hpp @@ -0,0 +1,231 @@ +/***************************************************************************** + * + * libdiffpy Complex Modeling Initiative + * (c) 2026 The Trustees of Columbia University + * in the City of New York. All rights reserved. + * + * See AUTHORS.rst for a list of people who contributed. + * See LICENSE.rst for license information. + * + ****************************************************************************** + * + * class PairTable -- all pairs of a structure for a sum over them + * class PairGroups -- pairs that add the same peak + * + * Internal header, not installed. + * + * PairTable remembers the pairs of a periodic structure and only updates + * their distances while the structure stays close to that of the search, + * as in a refinement. PairGroups merges pairs of the same distance and + * width, so that each kind of peak is added once. + * + *****************************************************************************/ + +#ifndef KERNELS_PAIRTABLE_HPP_INCLUDED +#define KERNELS_PAIRTABLE_HPP_INCLUDED + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace diffpy { +namespace srreal { + +class PeriodicStructureAdapter; +class PeakWidthModel; +class PeakProfile; + +namespace kernels { + +// Exported for the unit tests. +class DLL_EXPORT PairTable { + public: + /// True when the table can give the pairs of the structure. These are + /// the structures of the classes AtomicStructureAdapter, + /// PeriodicStructureAdapter and CrystalStructureAdapter, not of classes + /// derived from them, also within the nometa and nosymmetry wrappers. + static bool supports(const StructureAdapter& stru); + + /// Number of searches for the pairs of periodic structures so far. + int countSearches() const { return msearches; } + + /// The geometry for a kernel that adds all pairs of the structure in + /// the range. Its arrays are those of this table, they are valid until + /// the next call. + PairGeometryArgs geometryArgs(const StructureAdapter& stru, double rmin, + double rmax); + + private: + /// Sources of the sites and lattice translations. + struct View { + /// The structure within any nometa and nosymmetry wrappers. + const StructureAdapter* source; + /// The outermost nosymmetry wrapper or null. Its pairs are those of + /// the sites alone, without symmetry positions and lattice + /// translations. + const StructureAdapter* cluster; + /// The source when it has a lattice, also for a cluster of its sites. + const PeriodicStructureAdapter* periodic; + /// True for the pairs of symmetry positions and lattice translations. + bool hasTranslations() const { return periodic && !cluster; } + /// The structure whose sites are those of the pairs. + const StructureAdapter& sites() const { + return cluster ? *cluster : *source; + } + }; + static View view(const StructureAdapter& stru); + + /// Contiguous storage for the arrays read by both CPU and CUDA kernels. + struct Geometry { + std::vector pos, posuij, siteuij, trans; + std::vector anisotropy; + std::vector multiplicity; + }; + + /// Positions and displacement tensors of a periodic structure as its + /// bond generator has them. + struct Cell { + /// Positions of site i are [sitebegin[i], sitebegin[i + 1]), the anchor + /// of the site is the first one. + std::vector sitebegin; + std::vector pos; + std::vector uij; + }; + + // methods + void preparePeriodic(const PeriodicStructureAdapter& stru, double rmin, + double rmax); + static void fillCell(const PeriodicStructureAdapter& stru, Cell& cell); + bool hasPairsFor(const PeriodicStructureAdapter& stru, const Cell& cell, + double rmin, double rmax) const; + void searchPairs(const PeriodicStructureAdapter& stru, const Cell& cell, + double rmin, double rmax); + + // data + Cell mcell; + // Pairs of a periodic structure and what they were searched for. + /// Positions and lattice basis at the search. + std::vector msitebegin0; + std::vector mpos0; + R3::Matrix mbase0; + /// Range of the distances at the search. + double mrlo0 = 0.0; + double mrhi0 = -1.0; + /// Lattice translations of the pairs in lattice coordinates. + std::vector> mmno; + /// Anchor site, position of the other site and lattice translation of + /// each pair, ordered by the anchor site. + std::vector mpairsite0; + std::vector mpairpos; + std::vector mpairtrans; + int msearches = 0; + // work arrays + std::vector mpossite; + Geometry mgeometry; +}; + +/// Sums of the scales of the pairs that have the same distance and width. +/// +/// Even adjacent floating-point values can straddle a peak or Debye cutoff. +class PairGroups { + public: + struct Group { + double dist; + double width; + /// Tells apart pairs that the distance and the width do not, for + /// example by their scattering factors. + std::int64_t tag; + /// Sum of the scales of the pairs. + double scale; + /// Multiplicity scale of a single Debye pair, before its precision cutoff. + unsigned weight; + }; + + void clear() { + mgroups.clear(); + mslots.assign(mslots.size(), -1); + } + + void add(double dist, double width, std::int64_t tag, double scale, + unsigned weight = 1) { + const std::uint64_t kd = key(dist); + const std::uint64_t kw = key(width); + if (2 * (mgroups.size() + 1) > mslots.size()) this->grow(); + const std::size_t mask = mslots.size() - 1; + std::size_t i = hash(kd, kw, tag, weight) & mask; + for (;; i = (i + 1) & mask) { + const int g = mslots[i]; + if (g < 0) break; + Group& group = mgroups[g]; + if (key(group.dist) == kd && key(group.width) == kw && group.tag == tag && + group.weight == weight) { + group.scale += scale; + return; + } + } + mslots[i] = int(mgroups.size()); + mgroups.push_back({dist, width, tag, scale, weight}); + } + + const std::vector& groups() const { return mgroups; } + + private: + static std::uint64_t key(double x) { return std::bit_cast(x); } + + static std::size_t hash(std::uint64_t kd, std::uint64_t kw, std::int64_t tag, + unsigned weight) { + std::uint64_t h = kd * 0x9e3779b97f4a7c15ull; + h ^= (h >> 29) ^ (kw * 0xbf58476d1ce4e5b9ull); + h ^= (h >> 31) ^ (std::uint64_t(tag) * 0x94d049bb133111ebull); + return std::size_t((h ^ (h >> 32)) + weight); + } + + void grow() { + const std::size_t n = std::max(1024, 2 * mslots.size()); + mslots.assign(n, -1); + const std::size_t mask = n - 1; + for (std::size_t g = 0; g < mgroups.size(); ++g) { + const Group& group = mgroups[g]; + std::size_t i = + hash(key(group.dist), key(group.width), group.tag, group.weight) & mask; + while (mslots[i] >= 0) i = (i + 1) & mask; + mslots[i] = int(g); + } + } + + std::vector mgroups; + /// Open addressing table of the group indices, -1 for a free slot. + std::vector mslots; +}; + +/// Peak width of the built-in models from the values of a bond. +bool assignPairWidth(PairWidthArgs&, const PeakWidthModel&); + +/// Add a peak with the bounds and values supplied by the profile. +void addProfilePeak(const PeakProfile&, double* out, int npts, int rlosteps, + double rstep, double dist, double fwhm, double scale); + +/// What a calculator keeps for its sums over all pairs. +struct PairSumState { + PairTable table; + PairSumWorkspace* workspaceFor(const CalculationKernel& kernel); + + private: + MakeWorkspace mfactory = nullptr; + std::unique_ptr mworkspace; +}; + +} // namespace kernels +} // namespace srreal +} // namespace diffpy + +#endif // KERNELS_PAIRTABLE_HPP_INCLUDED diff --git a/src/diffpy/srreal/scatteringfactordata.cpp b/src/diffpy/srreal/scatteringfactordata.cpp index 6d38fc9..d1191dc 100644 --- a/src/diffpy/srreal/scatteringfactordata.cpp +++ b/src/diffpy/srreal/scatteringfactordata.cpp @@ -109,12 +109,10 @@ class wksmbl_hash { typedef unordered_set SetOfWKFormulas; -const SetOfWKFormulas& getWKFormulasSet() { +SetOfWKFormulas loadWKFormulasSet() { using namespace diffpy::runtimepath; using diffpy::validators::ensureFileOK; - static unique_ptr the_set; - if (the_set) return *the_set; - the_set.reset(new SetOfWKFormulas); + SetOfWKFormulas the_set; string wkfile = datapath("f0_WaasKirf.dat"); ifstream fp(wkfile.c_str()); ensureFileOK(wkfile, fp); @@ -144,16 +142,23 @@ const SetOfWKFormulas& getWKFormulasSet() { throw line.format_error( wkfile, "Line should contain 11 floating point values."); } - if (the_set->count(wk)) { + if (the_set.count(wk)) { string emsg = "Duplicate atom symbol \""; emsg += wk.symbol + "\"."; throw line.format_error(wkfile, emsg); } - the_set->insert(wk); + the_set.insert(wk); wk.symbol.clear(); } } - return getWKFormulasSet(); + return the_set; +} + +/// The table is loaded on the first call, also when the first calls come +/// from several threads. A failed load is repeated on the next call. +const SetOfWKFormulas& getWKFormulasSet() { + static const SetOfWKFormulas the_set = loadWKFormulasSet(); + return the_set; } SetOfWKFormulas::const_iterator findWKFormula(const string& smbl) { @@ -186,56 +191,57 @@ SetOfWKFormulas::const_iterator findWKFormula(const string& smbl) { typedef unordered_map ElectronNumberStorage; -ElectronNumberStorage& getElectronNumberTable() { +ElectronNumberStorage loadElectronNumberTable() { using namespace diffpy::runtimepath; using diffpy::validators::ensureFileOK; - static unique_ptr entable; + ElectronNumberStorage entable; typedef ElectronNumberStorage::value_type ENPair; - if (!entable) { - entable.reset(new ElectronNumberStorage); - string ionfile = datapath("ionlist.dat"); - ifstream fp0(ionfile.c_str()); - ensureFileOK(ionfile, fp0); - LineReader line; - while (fp0 >> line) { - if (line.isignored()) continue; - istringstream fpline(line.line); - string element; - int z = 0; - fpline >> element >> z; - if (!fpline) { - throw line.format_error(ionfile, - "Expected at least 2 columns for (symbol, Z)."); - } - entable->insert(ENPair(element, z)); - for (int v; fpline >> v;) { - ostringstream smbl; - smbl << element << abs(v) << ((v > 0) ? '+' : '-'); - entable->insert(ENPair(smbl.str(), z - v)); - } + string ionfile = datapath("ionlist.dat"); + ifstream fp0(ionfile.c_str()); + ensureFileOK(ionfile, fp0); + LineReader line; + while (fp0 >> line) { + if (line.isignored()) continue; + istringstream fpline(line.line); + string element; + int z = 0; + fpline >> element >> z; + if (!fpline) { + throw line.format_error(ionfile, + "Expected at least 2 columns for (symbol, Z)."); } - const size_t mintablesize = 447; - if (entable->size() < mintablesize) { - ostringstream emsg; - emsg << "Incomplete file. Expected " << mintablesize << " items loaded " - << entable->size() << "."; - throw line.format_error(ionfile, emsg.str()); + entable.insert(ENPair(element, z)); + for (int v; fpline >> v;) { + ostringstream smbl; + smbl << element << abs(v) << ((v > 0) ? '+' : '-'); + entable.insert(ENPair(smbl.str(), z - v)); } } - return *entable; + const size_t mintablesize = 447; + if (entable.size() < mintablesize) { + ostringstream emsg; + emsg << "Incomplete file. Expected " << mintablesize << " items loaded " + << entable.size() << "."; + throw line.format_error(ionfile, emsg.str()); + } + return entable; +} + +/// Loaded on the first call as the Waasmaier-Kirfel formulas. +const ElectronNumberStorage& getElectronNumberTable() { + static const ElectronNumberStorage entable = loadElectronNumberTable(); + return entable; } // Neutron scattering lengths typedef unordered_map NeutronBCStorage; -const NeutronBCStorage& getNeutronBCTable() { +NeutronBCStorage loadNeutronBCTable() { using namespace diffpy::runtimepath; using diffpy::validators::ensureFileOK; typedef NeutronBCStorage::value_type BCPair; - static unique_ptr bctable; - if (bctable) return *bctable; - bctable.reset(new NeutronBCStorage); + NeutronBCStorage bctable; string nsffile = datapath("nsftable.dat"); ifstream fp(nsffile.c_str()); ensureFileOK(nsffile, fp); @@ -264,12 +270,12 @@ const NeutronBCStorage& getNeutronBCTable() { if (!fpbc) { throw line.format_error(nsffile, "Invalid b_c value."); } - if (bctable->count(smbl)) { + if (bctable.count(smbl)) { string emsg = "Duplicate atom symbol \""; emsg += smbl + "\"."; throw line.format_error(nsffile, emsg); } - bctable->insert(BCPair(smbl, bc)); + bctable.insert(BCPair(smbl, bc)); // elements are not explicitly included if there is just one isotope // or if all isotopes are unstable size_t p2 = smbl.find_first_of('-'); @@ -278,23 +284,29 @@ const NeutronBCStorage& getNeutronBCTable() { // there is just one isotope const string& chlf = line.words[1]; bool addel = - (chlf == "100") || (!bctable->count(el) && *chlf.rbegin() == 'Y'); + (chlf == "100") || (!bctable.count(el) && *chlf.rbegin() == 'Y'); if (addel) { - if (bctable->count(el)) { + if (bctable.count(el)) { string emsg = "Duplicate element entry for \""; emsg += el + "\"."; throw line.format_error(nsffile, emsg); ; } - bctable->insert(BCPair(el, bc)); + bctable.insert(BCPair(el, bc)); } } } // define aliases for neutron, deuterium and tritium - bctable->insert(BCPair("n", bctable->at("1-n"))); - bctable->insert(BCPair("D", bctable->at("2-H"))); - bctable->insert(BCPair("T", bctable->at("3-H"))); - return getNeutronBCTable(); + bctable.insert(BCPair("n", bctable.at("1-n"))); + bctable.insert(BCPair("D", bctable.at("2-H"))); + bctable.insert(BCPair("T", bctable.at("3-H"))); + return bctable; +} + +/// Loaded on the first call as the Waasmaier-Kirfel formulas. +const NeutronBCStorage& getNeutronBCTable() { + static const NeutronBCStorage bctable = loadNeutronBCTable(); + return bctable; } } // namespace diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 08632a3..1e00ad6 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -61,6 +61,11 @@ set_property(SOURCE TestCalculationKernel.cpp test_helpers.cpp target_include_directories(alltests PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}") target_link_libraries(alltests PRIVATE libdiffpy::diffpy Catch2::Catch2WithMain Threads::Threads) +if(TARGET diffpy_cuda) + # The device-switch test selects the current GPU through the CUDA runtime. + target_link_libraries(alltests PRIVATE CUDA::cudart_static) + target_compile_definitions(alltests PRIVATE LIBDIFFPY_TEST_CUDA_RUNTIME) +endif() # test_helpers.cpp finds the testdata directory from this macro. set_property(SOURCE test_helpers.cpp APPEND PROPERTY COMPILE_DEFINITIONS "DIFFPYTESTSDIRPATH=${CMAKE_CURRENT_SOURCE_DIR}") @@ -97,6 +102,9 @@ list(REMOVE_DUPLICATES _suite_tags) foreach(tag IN LISTS _suite_tags) string(REGEX REPLACE "^\"\\[|\\]\"$" "" name "${tag}") add_test(NAME "${name}" COMMAND alltests "[${name}]") + # Catch2 returns 4 when it skipped all tests of the tag, as those of the + # CUDA kernels without a device. + set_tests_properties("${name}" PROPERTIES SKIP_RETURN_CODE 4) if(WIN32 AND LIBDIFFPY_TEST_INSTALLED) set_tests_properties("${name}" PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") diff --git a/src/tests/TestBondGeneratorBatching.cpp b/src/tests/TestBondGeneratorBatching.cpp index f422e24..3cd2676 100644 --- a/src/tests/TestBondGeneratorBatching.cpp +++ b/src/tests/TestBondGeneratorBatching.cpp @@ -196,6 +196,17 @@ CrystalStructureAdapterPtr loadTestCrystalStructure(const std::string& f) { return rv; } +/// The same crystal built anew, without any cached symmetry positions. +CrystalStructureAdapterPtr rebuiltCrystal(const CrystalStructureAdapter& s) { + const Lattice& L = s.getLattice(); + CrystalStructureAdapterPtr rv(new CrystalStructureAdapter); + rv->setLatPar(L.a(), L.b(), L.c(), L.alpha(), L.beta(), L.gamma()); + rv->assign(s.begin(), s.end()); + rv->setSymmetryPrecision(s.getSymmetryPrecision()); + for (int i = 0; i < s.countSymOps(); ++i) rv->addSymOp(s.getSymOp(i)); + return rv; +} + /// Select bonds up to 6 from the site 1, used with CaTiO3. void nearSite1(BaseBondGenerator& bnds) { bnds.setRmax(6.0); @@ -277,23 +288,13 @@ TEST_CASE("TestBondGeneratorBatching::test_crystal_changes", bnds->selectAnchorSite(0); return generated(*bnds); }; - // the same structure rebuilt without cached symmetry positions - auto rebuilt = [](const CrystalStructureAdapter& s) { - const Lattice& L = s.getLattice(); - CrystalStructureAdapterPtr rv(new CrystalStructureAdapter); - rv->setLatPar(L.a(), L.b(), L.c(), L.alpha(), L.beta(), L.gamma()); - rv->assign(s.begin(), s.end()); - rv->setSymmetryPrecision(s.getSymmetryPrecision()); - for (int i = 0; i < s.countSymOps(); ++i) rv->addSymOp(s.getSymOp(i)); - return rv; - }; auto same = [](const std::vector& a, const std::vector& b) { return a.size() == b.size() && std::equal(a.begin(), a.end(), b.begin(), sameBond); }; std::vector last = bonds(*stru); REQUIRE(!last.empty()); - checkSameBonds(last, bonds(*rebuilt(*stru))); + checkSameBonds(last, bonds(*rebuiltCrystal(*stru))); R3::Matrix R = R3::identity(); R(2, 2) = -1.0; const std::pair> changes[] = { @@ -310,7 +311,7 @@ TEST_CASE("TestBondGeneratorBatching::test_crystal_changes", change(); const std::vector now = bonds(*stru); CHECK(!same(last, now)); - checkSameBonds(now, bonds(*rebuilt(*stru))); + checkSameBonds(now, bonds(*rebuiltCrystal(*stru))); last = now; } }); @@ -346,6 +347,49 @@ TEST_CASE("TestBondGeneratorBatching::test_crystal_refresh", }); } +TEST_CASE("TestBondGeneratorBatching::test_crystal_site_expansion", + "[BondGeneratorBatching]") { + // A changed atom expands only its own site again. All sites then have + // the positions of a new expansion. + CrystalStructureAdapterPtr stru = loadTestCrystalStructure("CaTiO3.stru"); + const int cntsites = stru->countSites(); + auto arrays = [&] { + std::vector rv; + for (int i = 0; i < cntsites; ++i) { + rv.push_back(stru->getEquivalentAtoms(i).data()); + } + return rv; + }; + auto checkAsRebuilt = [&] { + const CrystalStructureAdapterPtr fresh = rebuiltCrystal(*stru); + for (int i = 0; i < cntsites; ++i) { + INFO("site " << i); + CHECK(fresh->getEquivalentAtoms(i) == stru->getEquivalentAtoms(i)); + } + }; + const std::vector before = arrays(); + stru->at(3).xyz_cartn[1] += 0.2; + stru->at(5).uij_cartn(0, 0) += 0.01; + const R3::Vector moved = stru->getEquivalentAtoms(3)[0].xyz_cartn; + const std::vector after = arrays(); + for (int i = 0; i < cntsites; ++i) { + if (i == 3 || i == 5) continue; + INFO("site " << i); + CHECK(before[i] == after[i]); + } + checkAsRebuilt(); + // an atom that moves back gets its first positions again + stru->at(3).xyz_cartn[1] -= 0.2; + CHECK(moved != stru->getEquivalentAtoms(3)[0].xyz_cartn); + checkAsRebuilt(); + // a new lattice moves the positions of every site + stru->setLatPar(5.5, 7.7, 5.4, 90.0, 95.0, 90.0); + checkAsRebuilt(); + // sites that take the atoms of their neighbors + std::swap(stru->at(1), stru->at(2)); + checkAsRebuilt(); +} + TEST_CASE("TestBondGeneratorBatching::test_crystal_constructor", "[BondGeneratorBatching]") { // The public constructor finds the symmetry positions when needed. diff --git a/src/tests/TestCalculationKernel.cpp b/src/tests/TestCalculationKernel.cpp index 7f3c7a0..9081e98 100644 --- a/src/tests/TestCalculationKernel.cpp +++ b/src/tests/TestCalculationKernel.cpp @@ -234,10 +234,17 @@ TEST_CASE("TestCalculationKernel::test_available", "[CalculationKernel]") { return std::ranges::find(names, name) != names.end(); }; REQUIRE(!names.empty()); - // The scalar kernel is the last resort. - CHECK(names.back() == "scalar"); - const std::vector known = {"avx512", "avx2", "sse2", "neon", - "scalar"}; + // The scalar kernel is the last resort of the CPU. The CUDA kernels + // follow it, so that they are never the default. + const size_t cntcpu = names.size() - (has("cuda") ? 2 : 0); + CHECK(names[cntcpu - 1] == "scalar"); + CHECK(has("cuda") == has("cuda-mixed")); + if (has("cuda")) { + CHECK(names[cntcpu] == "cuda"); + CHECK(names[cntcpu + 1] == "cuda-mixed"); + } + const std::vector known = {"avx512", "avx2", "sse2", "neon", + "scalar", "cuda", "cuda-mixed"}; for (const std::string& kname : names) { INFO("kernel " << kname); CHECK(std::ranges::count(names, kname) == 1); @@ -273,6 +280,22 @@ TEST_CASE("TestCalculationKernel::test_selection", "[CalculationKernel]") { } else { CHECK_THROWS_AS(setCalculationKernel("sse2"), std::invalid_argument); } + // The CUDA kernels tell why they are not available. + if (std::ranges::find(names, "cuda") == names.end()) { + for (const char* cuda : {"cuda", "cuda-mixed"}) { + INFO("kernel '" << cuda << "'"); + try { + setCalculationKernel(cuda); + FAIL("The kernel was taken without a device."); + } catch (const std::invalid_argument& e) { + const std::string what = e.what(); + INFO(what); + CHECK(what.find("is not available: ") != std::string::npos); + CHECK(what.find("use one of") == std::string::npos); + } + CHECK(active == getCalculationKernel()); + } + } } TEST_CASE("TestCalculationKernel::test_match_scalar", "[CalculationKernel]") { @@ -355,29 +378,33 @@ TEST_CASE("TestCalculationKernel::test_window_lengths", "[CalculationKernel]") { // The scale is negative when a calculator removes a pair. std::vector sf(hi); for (int kq = 0; kq < hi; ++kq) sf[kq] = std::exp(-0.01 * kq); + // Several equal pairs add what one pair adds that many times. for (double scale : {2.0, -2.0}) { for (double prec : {0.0, 1e-3, 0.1}) { - INFO("lo " << lo << ", n " << n << ", scale " << scale << ", prec " - << prec); - QuantityType reference(hi + padding, fill); - QuantityType actual(reference); - DebyeSumArgs args = { - .out = reference.data(), - .kqlo = lo, - .kqhi = hi, - .qstep = 0.05, - .dist = 1.7, - .dwsigma = 0.1, - .scale = scale, - .prec = prec, - .sf0 = sf.data(), - .sf1 = sf.data(), - }; - scalar.debyeSum(args); - args.out = actual.data(); - kernel.debyeSum(args); - CHECK(maxRelativeError(actual, reference) <= KERNEL_TOLERANCE); - CHECK(sameUntouched(actual, reference, fill)); + for (int count : {1, 3}) { + INFO("lo " << lo << ", n " << n << ", scale " << scale + << ", prec " << prec << ", count " << count); + QuantityType reference(hi + padding, fill); + QuantityType actual(reference); + DebyeSumArgs args = { + .out = reference.data(), + .kqlo = lo, + .kqhi = hi, + .qstep = 0.05, + .dist = 1.7, + .dwsigma = 0.1, + .scale = scale, + .prec = prec, + .sf0 = sf.data(), + .sf1 = sf.data(), + }; + for (int k = 0; k < count; ++k) scalar.debyeSum(args); + args.out = actual.data(); + args.count = count; + kernel.debyeSum(args); + CHECK(maxRelativeError(actual, reference) <= KERNEL_TOLERANCE); + CHECK(sameUntouched(actual, reference, fill)); + } } } } diff --git a/src/tests/TestCalculationKernelCuda.cpp b/src/tests/TestCalculationKernelCuda.cpp new file mode 100644 index 0000000..627e31d --- /dev/null +++ b/src/tests/TestCalculationKernelCuda.cpp @@ -0,0 +1,527 @@ +/***************************************************************************** + * + * libdiffpy Complex Modeling Initiative + * (c) 2026 The Trustees of Columbia University + * in the City of New York. All rights reserved. + * + * See AUTHORS.rst for a list of people who contributed. + * See LICENSE.rst for license information. + * + ****************************************************************************** + * + * TestCalculationKernelCuda -- the "cuda" and "cuda-mixed" kernels must + * give the results of the CPU. Skipped without a CUDA device. + * + *****************************************************************************/ + +#include + +#ifdef LIBDIFFPY_TEST_CUDA_RUNTIME +#include +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +// The internal kernel header is missing when testing an installed library. +#if __has_include() +#include +#define LIBDIFFPY_TEST_KERNEL_INTERNALS 1 +#endif + +#include "pair_test_helpers.hpp" + +using namespace diffpy::srreal; +using namespace pairtests; + +namespace { + +/// Agreement with the CPU. In mixed precision, the results other than the +/// raw PDF peaks cancel larger terms (Debye sums, PDF after the baseline) +/// and get the looser tolerance. +const double TOLERANCE = 1e-12; +const double MIXED_TOLERANCE = 1e-6; +const double MIXED_REST_TOLERANCE = 1e-5; + +void skipWithoutDevice() { + KernelGuard guard; + try { + setCalculationKernel("cuda"); + } catch (const std::invalid_argument& e) { + SKIP(e.what()); + } +} + +/// The kernel of the CPU that the calculators use by default. +std::string cpuKernel() { return availableCalculationKernels().front(); } + +/// Evaluate with the kernel, then restore the one that was active. +const QuantityType& evalWith(const std::string& kernel, PairQuantity& calc, + StructureAdapterPtr stru) { + KernelGuard guard; + setCalculationKernel(kernel); + return calc.eval(stru); +} + +/// Ni atoms of the fcc lattice within a sphere. +std::shared_ptr makeSphere(double radius) { + auto rv = std::make_shared(); + Atom a; + a.atomtype = "Ni"; + a.uij_cartn = 0.005 * R3::identity(); + const double alat = 3.52; + const int n = int(radius / alat) + 1; + const R3::Vector basis[] = {R3::Vector(0, 0, 0), R3::Vector(0, 0.5, 0.5), + R3::Vector(0.5, 0, 0.5), R3::Vector(0.5, 0.5, 0)}; + for (int i = -n; i <= n; ++i) { + for (int j = -n; j <= n; ++j) { + for (int k = -n; k <= n; ++k) { + for (const R3::Vector& b : basis) { + a.xyz_cartn = alat * (b + R3::Vector(i, j, k)); + if (R3::norm(a.xyz_cartn) <= radius) rv->append(a); + } + } + } + } + return rv; +} + +/// Results of a calculator of the class T with the kernel. +template +std::vector evaluate(const Setup& setup, StructureAdapterPtr s, + const std::string& kernel) { + KernelGuard guard; + setCalculationKernel(kernel); + AllPairs calc; + calc.setEvaluatorType(BASIC); + setup(calc); + calc.eval(s); + CHECK(calc.addedAllPairs()); + CHECK((kernel != cpuKernel()) == calc.addedOnDevice()); + std::vector rv = {calc.value(), calc.getPDF(), calc.getF()}; + // The same again to the last bit, also from the calculator of the + // library. + T again; + again.setEvaluatorType(BASIC); + setup(again); + again.eval(s); + CHECK(rv[0] == again.value()); + return rv; +} + +/// Check that the device gives what the CPU does. +template +void checkAsCPU(const Setup& setup, StructureAdapterPtr s) { + const auto expected = evaluate(setup, s, cpuKernel()); + REQUIRE(*std::ranges::max_element(expected[0]) > 0.0); + const auto results = evaluate(setup, s, "cuda"); + const auto mixed = evaluate(setup, s, "cuda-mixed"); + for (size_t i = 0; i < expected.size(); ++i) { + CHECK(maxRelativeError(results[i], expected[i]) <= TOLERANCE); + const bool peaks = !i && std::is_same_v; + const double tolerance = peaks ? MIXED_TOLERANCE : MIXED_REST_TOLERANCE; + CHECK(maxRelativeError(mixed[i], expected[i]) <= tolerance); + } +} + +} // namespace + +TEST_CASE("TestCalculationKernelCuda::test_results_of_calculators", "[cuda]") { + skipWithoutDevice(); + forEachCalculatorCase([](const Setup& setup, StructureAdapterPtr structure) { + checkAsCPU(setup, structure); + checkAsCPU(setup, structure); + }); +} + +#ifdef LIBDIFFPY_TEST_CUDA_RUNTIME +TEST_CASE("TestCalculationKernelCuda::test_device_switch", "[cuda]") { + skipWithoutDevice(); + int count, original; + REQUIRE(cudaGetDeviceCount(&count) == cudaSuccess); + if (count < 2) SKIP("Requires two CUDA devices"); + REQUIRE(cudaGetDevice(&original) == cudaSuccess); + struct RestoreDevice { + int device; + ~RestoreDevice() { cudaSetDevice(device); } + } restore{original}; + + const auto stru = loadPeriodic("CaTiO3.stru"); + PDFCalculator pdf; + DebyePDFCalculator debye; + PairQuantity* calculators[] = {&pdf, &debye}; + for (PairQuantity* calc : calculators) { + setupDefault(*calc); + calc->setEvaluatorType(BASIC); + const QuantityType expected = evalWith("scalar", *calc, stru); + for (const int device : {original, (original + 1) % count, original}) { + REQUIRE(cudaSetDevice(device) == cudaSuccess); + skipWithoutDevice(); + CHECK(maxRelativeError(evalWith("cuda", *calc, stru), expected) <= + TOLERANCE); + } + } +} +#endif + +TEST_CASE("TestCalculationKernelCuda::test_peak_profiles_and_cutoffs", + "[cuda]") { + skipWithoutDevice(); + const StructureAdapterPtr stru = loadPeriodic("CaTiO3.stru", true); + const Setup cropped = [](PairQuantity& c) { + setupDefault(c); + dynamic_cast(c).setPeakProfileByType("croppedgaussian"); + }; + const Setup precise = [](PairQuantity& c) { + setupDefault(c); + c.setDoubleAttr("peakprecision", 1e-12); + }; + // peaks that are cut at one percent of their height + const Setup cut = [](PairQuantity& c) { + setupDefault(c); + dynamic_cast(c).setPeakProfileByType("croppedgaussian"); + c.setDoubleAttr("peakprecision", 0.01); + }; + checkAsCPU(cropped, stru); + checkAsCPU(cut, stru); + checkAsCPU(precise, stru); + // Debye sums that stop early, late and never + for (double prec : {1e-3, 1e-6, 1e-12, 0.0}) { + INFO("debyeprecision " << prec); + const Setup debye = [=](PairQuantity& c) { + setupDefault(c); + c.setDoubleAttr("debyeprecision", prec); + }; + checkAsCPU(debye, stru); + checkAsCPU(debye, + asCluster(*loadPeriodic("Ni.stru", true))); + } +} + +TEST_CASE("TestCalculationKernelCuda::test_narrow_jeong_peak", "[cuda]") { + skipWithoutDevice(); + auto stru = std::make_shared(); + Atom atom; + atom.atomtype = "Ni"; + atom.uij_cartn = 0.05 * R3::identity(); + stru->append(atom); + atom.xyz_cartn[0] = 1.0; + stru->append(atom); + checkAsCPU( + [](PairQuantity& c) { + c.setRmax(2.0); + c.setDoubleAttr("rstep", 0.01); + c.setDoubleAttr("delta2", 1.0 - 2.5e-8); + }, + stru); +} + +TEST_CASE("TestCalculationKernelCuda::test_sampled_peak_windows", "[cuda]") { + skipWithoutDevice(); + auto stru = std::make_shared(); + Atom atom; + atom.atomtype = "C"; + stru->append(atom); + stru->append(atom); + PDFCalculator calc; + calc.setRmax(4); + calc.setRstep(0.01); + calc.setMaxExtension(0); + calc.setEvaluatorType(BASIC); + SECTION("an unresolved peak between samples leaves the other peaks intact") { + (*stru)[1].xyz_cartn[0] = 1.005; + atom.xyz_cartn[1] = 2; + atom.uij_cartn = 0.01 * R3::identity(); + stru->append(atom); + calc.setDoubleAttr("qbroad_seperable", 1e-10); + } + SECTION("a 3 angstrom bond at the inclusive upper distance cutoff") { + (*stru)[1].xyz_cartn = R3::Vector(0.4, 2.0, 2.2); + (*stru)[1].uij_cartn = 0.02 * R3::identity(); + calc.setRmax(3); + } + SECTION("a sample at the cropped Gaussian boundary") { + (*stru)[1].xyz_cartn[0] = 3; + calc.setPeakWidthModelByType("constant"); + calc.setDoubleAttr("width", 0.2); + calc.setPeakProfileByType("croppedgaussian"); + calc.setDoubleAttr("peakprecision", 0.5); + } + const QuantityType reference = evalWith("scalar", calc, stru); + REQUIRE(*std::ranges::max_element(reference) > 0.0); + CHECK(maxRelativeError(evalWith("cuda", calc, stru), reference) <= TOLERANCE); + CHECK(maxRelativeError(evalWith("cuda-mixed", calc, stru), reference) <= + MIXED_TOLERANCE); +} + +TEST_CASE("TestCalculationKernelCuda::test_weak_peaks", "[cuda]") { + skipWithoutDevice(); + auto stru = std::make_shared(); + Atom atom; + atom.atomtype = "Ni"; + atom.occupancy = 0.01; + for (int x = 0; x < 8; ++x) + for (int y = 0; y < 8; ++y) + for (int z = 0; z < 8; ++z) { + atom.xyz_cartn = R3::Vector(x, y, z); + stru->append(atom); + } + atom.occupancy = 1; + atom.xyz_cartn = R3::Vector(0, 100, 0); + stru->append(atom); + atom.xyz_cartn[0] = 2.7; + stru->append(atom); + PDFCalculator calc; + calc.setEvaluatorType(BASIC); + calc.setRmax(5); + calc.setMaxExtension(0); + calc.setPeakWidthModelByType("constant"); + calc.setDoubleAttr("width", 0.1); + const QuantityType reference = evalWith("scalar", calc, stru); + const QuantityType actual = evalWith("cuda", calc, stru); + CHECK(maxRelativeError(actual, reference) <= TOLERANCE); + CHECK(actual == evalWith("cuda", calc, stru)); +} + +TEST_CASE("TestCalculationKernelCuda::test_split_site_debye_sum", "[cuda]") { + skipWithoutDevice(); + auto stru = std::make_shared(); + Atom atom; + atom.atomtype = "Ni"; + atom.uij_cartn = 0.01 * R3::identity(); + stru->append(atom); + atom.xyz_cartn[0] = 2.0; + stru->append(atom); + DebyePDFCalculator calc; + calc.setRmax(4); + calc.setQstep(0.01); + calc.setEvaluatorType(BASIC); + const QuantityType reference = evalWith("scalar", calc, stru); + // Split the first site into coincident partial occupancies. All its + // terms for a given Q have the same sign and their sum is unchanged. + (*stru)[0].occupancy = 1.0 / 2048; + atom = (*stru)[0]; + for (int i = 1; i < 2048; ++i) stru->append(atom); + // Put the unsplit site last so that one subchunk sums 1024 equal terms. + std::swap((*stru)[1], stru->at(2048)); + CHECK(maxRelativeError(evalWith("cuda-mixed", calc, stru), reference) <= + MIXED_TOLERANCE); +} + +TEST_CASE("TestCalculationKernelCuda::test_many_occupancies_on_a_fine_q_grid", + "[cuda]") { + skipWithoutDevice(); + auto stru = std::make_shared(); + Atom atom; + atom.atomtype = "Ni"; + // Only the first two sites form a bond. All 129 occupancies enter the + // factor table, which spans more than one Q tile at this Q step. + for (int i = 0; i < 129; ++i) { + atom.occupancy = (i + 1) / 129.0; + atom.xyz_cartn[0] = (i == 1) ? 2.0 : 10.0 * i; + stru->append(atom); + } + DebyePDFCalculator calc; + calc.setEvaluatorType(BASIC); + calc.setRmax(4); + calc.setMaxExtension(0); + calc.setQmax(9); + calc.setQstep(0.001); + const QuantityType expected = evalWith("scalar", calc, stru); + REQUIRE(*std::ranges::max_element(expected) > 0); + CHECK(maxRelativeError(evalWith("cuda", calc, stru), expected) <= TOLERANCE); + CHECK(maxRelativeError(evalWith("cuda-mixed", calc, stru), expected) <= + MIXED_TOLERANCE); +} + +TEST_CASE("TestCalculationKernelCuda::test_change_precision", "[cuda]") { + skipWithoutDevice(); + const auto stru = loadPeriodic("CaTiO3.stru", true); + PDFCalculator pdf; + DebyePDFCalculator debye; + PairQuantity* calculators[] = {&pdf, &debye}; + for (PairQuantity* calc : calculators) { + setupDefault(*calc); + calc->setEvaluatorType(OPTIMIZED); + const QuantityType mixed = evalWith("cuda-mixed", *calc, stru); + const QuantityType doubles = evalWith("cuda", *calc, stru); + CHECK(calc->getEvaluatorTypeUsed() == BASIC); + calc->setEvaluatorType(BASIC); + const QuantityType expected = evalWith("cuda", *calc, stru); + REQUIRE(mixed != expected); + CHECK(maxRelativeError(doubles, expected) == 0.0); + } +} + +TEST_CASE("TestCalculationKernelCuda::test_many_pairs", "[cuda]") { + skipWithoutDevice(); + // more pairs than the device takes at once + const StructureAdapterPtr sphere = makeSphere(19.5); + const long long n = sphere->countSites(); + INFO(n << " atoms"); + REQUIRE(n * (n - 1) / 2 > (3LL << 20)); + const Setup setup = [](PairQuantity& c) { + c.setRmax(41.0); + c.setDoubleAttr("qmax", 20.0); + }; + checkAsCPU(setup, sphere); + checkAsCPU(setup, sphere); + // and of a periodic structure + const StructureAdapterPtr periodic = + loadPeriodic("alpha_K2Bi8Se13.stru", true); + const Setup far = [](PairQuantity& c) { + c.setRmax(60.0); + c.setDoubleAttr("qmax", 15.0); + }; + checkAsCPU(far, periodic); +} + +TEST_CASE("TestCalculationKernelCuda::test_evaluations_in_turn", "[cuda]") { + skipWithoutDevice(); + // The list of pairs stays on the device while the structure changes + // little. + const auto periodic = loadPeriodic("CaTiO3.stru", true); + for (std::shared_ptr stru : + {periodic, + std::shared_ptr(asCrystal(*periodic))}) { + AllPairs cpu, device; + AllPairs dcpu, ddevice; + for (PairQuantity* c : {(PairQuantity*)&cpu, (PairQuantity*)&device, + (PairQuantity*)&dcpu, (PairQuantity*)&ddevice}) { + setupDefault(*c); + } + const Lattice L = stru->getLattice(); + for (int k = 0; k < 12; ++k) { + INFO("evaluation " << k); + const int n = stru->countSites(); + // every fourth change is beyond the margin of the pairs + const double step = (k % 4 == 3) ? 0.3 : 0.004; + stru->at(k % n).xyz_cartn[k % 3] += step * ((k % 5) - 2); + stru->at((3 * k) % n).uij_cartn *= 1.0 + 0.01 * ((k % 3) - 1); + for (Atom& a : *stru) stru->toFractional(a); + const double f = 1.0 + 1e-4 * ((k % 7) - 3); + stru->setLatPar(f * L.a(), L.b(), L.c() / f, L.alpha(), L.beta(), + L.gamma()); + for (Atom& a : *stru) stru->toCartesian(a); + StructureAdapterPtr copy = stru->clone(); + // every third sum of the device is in mixed precision + const bool mixed = (k % 3 == 2); + const std::string kernel = mixed ? "cuda-mixed" : "cuda"; + const double tolerance = mixed ? MIXED_REST_TOLERANCE : TOLERANCE; + CHECK(maxRelativeError(evalWith(kernel, device, copy), cpu.eval(copy)) <= + tolerance); + CHECK(maxRelativeError(evalWith(kernel, ddevice, copy), + dcpu.eval(copy)) <= tolerance); + CHECK(device.addedOnDevice()); + CHECK(ddevice.addedOnDevice()); + CHECK(!cpu.addedOnDevice()); + // The device adds all pairs again rather than those of a few sites. + if (k > 0) CHECK(BASIC == device.getEvaluatorTypeUsed()); + } + // a structure of the same content takes no sum + evalWith("cuda-mixed", device, stru->clone()); + CHECK(OPTIMIZED == device.getEvaluatorTypeUsed()); + } +} + +TEST_CASE("TestCalculationKernelCuda::test_sums_on_the_cpu", "[cuda]") { + skipWithoutDevice(); + struct DerivedProfile : GaussianProfile {}; + const StructureAdapterPtr stru = loadPeriodic("CaTiO3.stru", true); + // What the device does not take gives the results of the CPU to the + // last bit. + auto check = [&](PairQuantity& calc, PairQuantity& cpu) { + for (PairQuantity* c : {&calc, &cpu}) { + setupDefault(*c); + c->setEvaluatorType(BASIC); + } + for (const char* kernel : {"cuda", "cuda-mixed"}) { + INFO("kernel " << kernel); + CHECK(evalWith(kernel, calc, stru) == cpu.eval(stru)); + } + }; + // all pairs at once, with the kernel of the CPU + AllPairs masked, maskedcpu; + masked.setPairMask(0, 3, false); + maskedcpu.setPairMask(0, 3, false); + check(masked, maskedcpu); + CHECK(masked.addedAllPairs()); + CHECK(!masked.addedOnDevice()); + AllPairs dmasked, dmaskedcpu; + dmasked.setTypeMask("Ti4+", "O2-", false); + dmaskedcpu.setTypeMask("Ti4+", "O2-", false); + check(dmasked, dmaskedcpu); + CHECK(dmasked.addedAllPairs()); + CHECK(!dmasked.addedOnDevice()); + AllPairs profiled, profiledcpu; + profiled.setPeakProfile(PeakProfilePtr(new DerivedProfile)); + profiledcpu.setPeakProfile(PeakProfilePtr(new DerivedProfile)); + check(profiled, profiledcpu); + CHECK_FALSE(profiled.addedAllPairs()); + CHECK(!profiled.addedOnDevice()); + // the pairs one by one + PairByPair derived, derivedcpu; + check(derived, derivedcpu); + BVSCalculator bvsc, bvscpu; + CHECK(evalWith("cuda", bvsc, stru) == bvscpu.eval(stru)); + // all pairs of a calculator without any of these go to the device + AllPairs pdfc; + PDFCalculator pdfcpu; + for (PairQuantity* c : {(PairQuantity*)&pdfc, (PairQuantity*)&pdfcpu}) { + setupDefault(*c); + c->setEvaluatorType(BASIC); + } + CHECK(maxRelativeError(evalWith("cuda", pdfc, stru), pdfcpu.eval(stru)) <= + TOLERANCE); + CHECK(pdfc.addedOnDevice()); + // and back to the CPU with its kernels + CHECK(pdfc.eval(stru) == pdfcpu.value()); + CHECK(pdfc.addedAllPairs()); + CHECK(!pdfc.addedOnDevice()); + // then to the device again + CHECK(maxRelativeError(evalWith("cuda-mixed", pdfc, stru), pdfcpu.value()) <= + MIXED_TOLERANCE); + CHECK(pdfc.addedOnDevice()); +} + +#ifdef LIBDIFFPY_TEST_KERNEL_INTERNALS + +TEST_CASE("TestCalculationKernelCuda::test_kernels_of_the_cpu", "[cuda]") { + skipWithoutDevice(); + // The CUDA kernels are those of the CPU for everything else. + KernelGuard guard; + setCalculationKernel(cpuKernel()); + const kernels::CalculationKernel cpu = kernels::activeCalculationKernel(); + CHECK_FALSE(cpu.onDevice); + CHECK(cpu.pdfPairSum); + CHECK(cpu.debyePairSum); + for (const char* name : {"cuda", "cuda-mixed"}) { + INFO("kernel " << name); + setCalculationKernel(name); + const kernels::CalculationKernel& k = kernels::activeCalculationKernel(); + CHECK(std::string(name) == k.name); + CHECK(k.gaussianWindow == cpu.gaussianWindow); + CHECK(k.debyeSum == cpu.debyeSum); + CHECK(k.bondCandidates == cpu.bondCandidates); + CHECK(k.pdfPairSum); + CHECK(k.debyePairSum); + CHECK(k.makeWorkspace); + CHECK(k.onDevice); + } +} + +#endif // LIBDIFFPY_TEST_KERNEL_INTERNALS diff --git a/src/tests/TestDebyePDFCalculator.cpp b/src/tests/TestDebyePDFCalculator.cpp index bda39e7..0fcf5d6 100644 --- a/src/tests/TestDebyePDFCalculator.cpp +++ b/src/tests/TestDebyePDFCalculator.cpp @@ -19,7 +19,10 @@ #include #include +#include #include +#include +#include #include #include @@ -352,4 +355,171 @@ TEST_CASE_METHOD(TestDebyePDFCalculatorFixture, CHECK(BASIC == mpdfc->getEvaluatorTypeUsed()); } +TEST_CASE_METHOD(TestDebyePDFCalculatorFixture, + "TestDebyePDFCalculator::test_DBPDF_site_occupancies", + "[DebyePDFCalculator]") { + // Sites of one element with different occupancies scatter as sites of + // different atom types with those occupancies. + AtomicStructureAdapterPtr mixed = + std::make_shared(*mstru10); + AtomicStructureAdapterPtr named = + std::make_shared(*mstru10); + for (int i : {0, 3, 4, 8}) { + mixed->at(i).occupancy = 0.4; + named->at(i).occupancy = 0.4; + named->at(i).atomtype = "C2"; + } + mpdfc->getScatteringFactorTable()->setCustomAs("C2", "C"); + for (PQEvaluatorType tp : {BASIC, OPTIMIZED}) { + INFO("evaluator " << tp); + mpdfc->setEvaluatorType(tp); + mpdfc->eval(named); + const QuantityType gnamed = mpdfc->getPDF(); + const QuantityType fnamed = mpdfc->getF(); + mpdfc->eval(mstru10); + CHECK(!allclose(gnamed, mpdfc->getPDF())); + mpdfc->eval(mixed); + CHECK(allclose(gnamed, mpdfc->getPDF())); + CHECK(allclose(fnamed, mpdfc->getF())); + } + // A new occupancy of the first site of an element is a change of that + // site only, which the fast update must get right. + mpdfc->setEvaluatorType(OPTIMIZED); + mpdfc->eval(mixed); + mixed->at(0).occupancy = 0.7; + named->at(0).occupancy = 0.7; + mpdfc->eval(mixed); + CHECK(OPTIMIZED == mpdfc->getEvaluatorTypeUsed()); + const QuantityType gfast = mpdfc->getPDF(); + mpdfc->setEvaluatorType(BASIC); + mpdfc->eval(named); + CHECK(allclose(gfast, mpdfc->getPDF())); +} + +TEST_CASE_METHOD(TestDebyePDFCalculatorFixture, + "TestDebyePDFCalculator::test_retry_unknown_atom", + "[DebyePDFCalculator]") { + const QuantityType expected = mpdfc->eval(mstru10); + mstru10->at(1).atomtype = "Bogus"; + CHECK_THROWS_AS(mpdfc->eval(mstru10), invalid_argument); + CHECK_THROWS_AS(mpdfc->eval(mstru10), invalid_argument); + mstru10->at(1).atomtype = "C"; + CHECK(mpdfc->eval(mstru10) == expected); +} + +TEST_CASE_METHOD(TestDebyePDFCalculatorFixture, + "TestDebyePDFCalculator::test_replace_table_by_reference", + "[DebyePDFCalculator]") { + DebyePDFCalculator reference; + reference.setScatteringFactorTableByType("neutron"); + const auto neutron = reference.eval(mstru10); + for (const auto evaluator : {BASIC, OPTIMIZED}) { + DebyePDFCalculator calc; + calc.setEvaluatorType(evaluator); + auto& table = calc.getScatteringFactorTable(); + REQUIRE(calc.eval(mstru10) != neutron); + table = reference.getScatteringFactorTable(); + CHECK(calc.eval(mstru10) == neutron); + CHECK(calc.getEvaluatorTypeUsed() == BASIC); + } +} + +TEST_CASE_METHOD(TestDebyePDFCalculatorFixture, + "TestDebyePDFCalculator::test_DBPDF_kept_scattering_factors", + "[DebyePDFCalculator]") { + // The kept scattering factors must follow every change of the calculator + // and of the occupancies. + typedef std::function Config; + std::vector history; + auto configure = [&](Config cfg) { + cfg(*mpdfc); + history.push_back(cfg); + }; + // the calculator that is checked + DebyePDFCalculator* calc = mpdfc.get(); + auto check = [&](StructureAdapterPtr stru) { + DebyePDFCalculator fresh; + for (const Config& cfg : history) cfg(fresh); + fresh.setEvaluatorType(BASIC); + fresh.eval(stru); + calc->eval(stru); + CHECK(allclose(fresh.getPDF(), calc->getPDF())); + CHECK(allclose(fresh.getF(), calc->getF())); + }; + check(mstru10); + check(mstru10d1); + configure( + [](DebyePDFCalculator& c) { c.setScatteringFactorTableByType("neutron"); }); + check(mstru10d1); + configure([](DebyePDFCalculator& c) { + c.getScatteringFactorTable()->setCustomAs("C", "Au"); + }); + check(mstru10d1); + configure([](DebyePDFCalculator& c) { c.setQmax(15.0); }); + check(mstru10d1); + configure([](DebyePDFCalculator& c) { c.setRstep(0.02); }); + check(mstru10d1); + // Many occupancies in turn, which leave arrays that no site uses. + AtomicStructureAdapterPtr stru = + std::make_shared(*mstru10d1); + // a calculator of a derived class adds its pairs one by one + struct DerivedDebyePDFCalculator : DebyePDFCalculator {}; + DerivedDebyePDFCalculator onebyone; + for (const Config& cfg : history) cfg(onebyone); + for (DebyePDFCalculator* c : {(DebyePDFCalculator*)&onebyone, mpdfc.get()}) { + const bool pairbypair = (c == &onebyone); + calc = c; + check(stru); + for (int k = 0; k < 40; ++k) { + INFO((pairbypair ? "pair by pair" : "all pairs") << " step " << k); + stru->at(3).occupancy = 0.5 + 0.01 * k; + stru->at(7).occupancy = 0.9 - 0.01 * k; + check(stru); + if (pairbypair) CHECK(OPTIMIZED == calc->getEvaluatorTypeUsed()); + } + } + calc = mpdfc.get(); + // a restored calculator builds its scattering factors again + stringstream storage(ios::in | ios::out | ios::binary); + diffpy::serialization::oarchive oa(storage, ios::binary); + oa << mpdfc; + diffpy::serialization::iarchive ia(storage, ios::binary); + std::shared_ptr pdfc1; + ia >> pdfc1; + const QuantityType g = mpdfc->getPDF(); + stru->at(3).occupancy = 0.3; + pdfc1->eval(stru); + CHECK(!allclose(g, pdfc1->getPDF())); + mpdfc.swap(pdfc1); + check(stru); +} + +TEST_CASE_METHOD(TestDebyePDFCalculatorFixture, + "TestDebyePDFCalculator::test_structure_dependent_scattering", + "[DebyePDFCalculator]") { + struct SiteFactor : DebyePDFCalculator { + double sfSiteAtQ(int site, const double&) const override { + // Both site-specific and structure-wide dependence. + return mstructure->siteCartesianUij(site)(0, 0) + + mstructure->siteCartesianUij(0)(1, 1); + } + }; + for (auto evaluator : {BASIC, OPTIMIZED}) { + SiteFactor calc; + calc.setEvaluatorType(evaluator); + auto stru = std::make_shared(*mstru10); + for (double u : {0.004, 0.05, 0.02, 0.004}) { + (*stru)[0].uij_cartn = u * R3::identity(); + // Give the reference sites unique keys. It must always evaluate each + // site's curve, including sites whose own parameters stayed unchanged. + auto named = std::make_shared(*stru); + for (int i = 0; i < named->countSites(); ++i) + (*named)[i].atomtype = std::to_string(i); + SiteFactor fresh; + fresh.setEvaluatorType(BASIC); + CHECK(allclose(calc.eval(stru), fresh.eval(named))); + } + } +} + // End of file diff --git a/src/tests/TestPQEvaluator.cpp b/src/tests/TestPQEvaluator.cpp index 08cb8c5..9ca7019 100644 --- a/src/tests/TestPQEvaluator.cpp +++ b/src/tests/TestPQEvaluator.cpp @@ -25,6 +25,8 @@ #include #include #include +#include +#include #include #include #include @@ -92,12 +94,14 @@ CrystalStructureAdapterPtr inversionCrystal(double x) { // TestPQEvaluator ////////////////////////////////////////////////////////////////////////////// -class TestPQEvaluatorFixture { +/// Calculators of the class PDF with the BASIC and OPTIMIZED evaluators. +template +class TestPQEvaluatorFixtureOf { protected: // data EpsilonEqual allclose; - PDFCalculator mpdfcb; - PDFCalculator mpdfco; + PDF mpdfcb; + PDF mpdfco; QuantityType mzeros; AtomicStructureAdapterPtr mstru10; AtomicStructureAdapterPtr mstru10d1; @@ -117,7 +121,7 @@ class TestPQEvaluatorFixture { } public: - TestPQEvaluatorFixture() { + TestPQEvaluatorFixtureOf() { // setup PDFCalculator instances and the zero vector for comparison mpdfcb.setEvaluatorType(BASIC); mpdfco.setEvaluatorType(OPTIMIZED); @@ -140,7 +144,12 @@ class TestPQEvaluatorFixture { mstru9 = std::make_shared(*mstru10); mstru9->erase(9); } -}; // class TestPQEvaluatorFixture +}; // class TestPQEvaluatorFixtureOf + +typedef TestPQEvaluatorFixtureOf TestPQEvaluatorFixture; +// Calculators that add their pairs one by one also in a full calculation. +typedef TestPQEvaluatorFixtureOf> + TestPQEvaluatorPairByPairFixture; TEST_CASE_METHOD(TestPQEvaluatorFixture, "TestPQEvaluator::test_initial_evaluator_type_used", @@ -187,6 +196,40 @@ TEST_CASE_METHOD(TestPQEvaluatorFixture, CHECK(!allclose(gb1, gb2)); } +TEST_CASE_METHOD(TestPQEvaluatorFixture, + "TestPQEvaluator::test_PDF_copies_of_calculator", + "[PQEvaluator]") { + // A copy of a calculator has an evaluator of its own. + for (PQEvaluatorType tp : {OPTIMIZED, PQEvaluatorType::CHECK, BASIC}) { + INFO("evaluator " << tp); + PDFCalculator pdfc; + pdfc.setEvaluatorType(tp); + PDFCalculator early = pdfc; + const QuantityType v10 = pdfc.eval(mstru10); + CHECK(tp == early.getEvaluatorType()); + CHECK(NONE == early.getEvaluatorTypeUsed()); + CHECK(v10 == early.eval(mstru10)); + CHECK(BASIC == early.getEvaluatorTypeUsed()); + // a copy with a value and its assignment + PDFCalculator late = pdfc; + PDFCalculator assigned; + assigned = pdfc; + const QuantityType v9 = pdfc.eval(mstru9); + CHECK(!allclose(v9, v10)); + for (PDFCalculator* c : {&late, &assigned}) { + CHECK(tp == c->getEvaluatorType()); + CHECK(v10 == c->value()); + CHECK(v10 == c->eval(mstru10)); + CHECK(BASIC == c->getEvaluatorTypeUsed()); + CHECK(allclose(v9, c->eval(mstru9))); + if (tp != BASIC) CHECK(tp == c->getEvaluatorTypeUsed()); + } + // the calculator is as it was + CHECK(v9 == pdfc.eval(mstru9)); + if (tp != BASIC) CHECK(tp == pdfc.getEvaluatorTypeUsed()); + } +} + TEST_CASE_METHOD(TestPQEvaluatorFixture, "TestPQEvaluator::test_PDF_reverse_atoms", "[PQEvaluator]") { mpdfco.eval(mstru10); @@ -228,8 +271,10 @@ TEST_CASE_METHOD(TestPQEvaluatorFixture, "TestPQEvaluator::test_PDF_type_mask", CHECK(!allclose(gb0, gb2)); } -TEST_CASE_METHOD(TestPQEvaluatorFixture, "TestPQEvaluator::test_PDF_pair_mask", - "[PQEvaluator]") { +TEST_CASE_METHOD(TestPQEvaluatorPairByPairFixture, + "TestPQEvaluator::test_PDF_pair_mask", "[PQEvaluator]") { + // The fast updates below are of pairs that are added one by one. A sum + // of all pairs takes less work for some changes of this small structure. mpdfcb.setPairMask(0, 3, false); mpdfco.setPairMask(0, 3, false); CHECK(mzeros == this->pdfcdiff(mstru10)); @@ -365,6 +410,152 @@ TEST_CASE_METHOD(TestPQEvaluatorFixture, "TestPQEvaluator::test_failed_update", } } +TEST_CASE_METHOD(TestPQEvaluatorFixture, + "TestPQEvaluator::test_PDF_unchanged_structure", + "[PQEvaluator]") { + // New structures of the same content keep the value, and later changes + // are still found. + mpdfco.eval(mstru10); + const QuantityType v10 = mpdfco.value(); + for (int i = 0; i < 3; ++i) { + StructureAdapterPtr copy = mstru10->clone(); + mpdfco.eval(copy); + CHECK(OPTIMIZED == mpdfco.getEvaluatorTypeUsed()); + CHECK(copy == mpdfco.getStructure()); + CHECK(v10 == mpdfco.value()); + } + CHECK(allclose(mzeros, this->pdfcdiff(mstru10d1))); + CHECK(OPTIMIZED == mpdfco.getEvaluatorTypeUsed()); + CHECK(allclose(mzeros, this->pdfcdiff(mstru10d1->clone()))); + CHECK(OPTIMIZED == mpdfco.getEvaluatorTypeUsed()); + CHECK(allclose(mzeros, this->pdfcdiff(mstru10))); + CHECK(OPTIMIZED == mpdfco.getEvaluatorTypeUsed()); +} + +TEST_CASE_METHOD(TestPQEvaluatorFixture, + "TestPQEvaluator::test_PDF_unchanged_structure_new_baseline", + "[PQEvaluator]") { + // The slope of a linear baseline comes from the structure at every + // evaluation, also when the structure did not change. + PeriodicStructureAdapterPtr stru(new PeriodicStructureAdapter); + stru->setLatPar(11.0, 5.0, 6.0, 90.0, 90.0, 90.0); + stru->assign(mstru10->begin(), mstru10->end()); + CHECK(allclose(mzeros, this->pdfcdiff(stru))); + const QuantityType g0 = mpdfco.getPDF(); + mpdfcb.setBaselineByType("linear"); + mpdfco.setBaselineByType("linear"); + CHECK(allclose(mzeros, this->pdfcdiff(stru->clone()))); + CHECK(OPTIMIZED == mpdfco.getEvaluatorTypeUsed()); + CHECK(allclose(g0, mpdfco.getPDF())); +} + +TEST_CASE_METHOD(TestPQEvaluatorFixture, + "TestPQEvaluator::test_PDF_structure_kind", "[PQEvaluator]") { + // A periodic structure differs from a cluster of the same atoms. + PeriodicStructureAdapterPtr periodic(new PeriodicStructureAdapter); + periodic->setLatPar(11.0, 5.0, 6.0, 90.0, 90.0, 90.0); + periodic->assign(mstru10->begin(), mstru10->end()); + CrystalStructureAdapterPtr crystal(new CrystalStructureAdapter); + crystal->setLatPar(11.0, 5.0, 6.0, 90.0, 90.0, 90.0); + crystal->assign(mstru10->begin(), mstru10->end()); + crystal->addSymOp(R3::identity(), R3::zerovector); + crystal->addSymOp(R3::identity(), R3::Vector(0.0, 0.5, 0.5)); + const StructureAdapterPtr strus[] = {mstru10, periodic, crystal, mstru10, + crystal, periodic, mstru10}; + QuantityType glast; + for (const StructureAdapterPtr& stru : strus) { + CHECK(allclose(mzeros, this->pdfcdiff(stru))); + const QuantityType g = mpdfcb.getPDF(); + CHECK(!allclose(glast, g)); + glast = g; + } +} + +TEST_CASE_METHOD(TestPQEvaluatorFixture, + "TestPQEvaluator::test_PDF_derived_structure_class", + "[PQEvaluator]") { + // A class that adds nothing to a built-in adapter, as one defined in + // Python, has copies of the built-in class and keeps its fast updates. + struct DerivedPeriodic : PeriodicStructureAdapter {}; + struct DerivedCrystal : CrystalStructureAdapter {}; + std::shared_ptr periodic(new DerivedPeriodic); + periodic->setLatPar(11.0, 5.0, 6.0, 90.0, 90.0, 90.0); + periodic->assign(mstru10->begin(), mstru10->end()); + std::shared_ptr crystal(new DerivedCrystal); + crystal->setLatPar(11.0, 5.0, 6.0, 90.0, 90.0, 90.0); + crystal->assign(mstru10->begin(), mstru10->end()); + crystal->addSymOp(R3::identity(), R3::zerovector); + crystal->addSymOp(R3::identity(), R3::Vector(0.0, 0.5, 0.5)); + const std::shared_ptr strus[] = {periodic, crystal}; + for (const auto& stru : strus) { + REQUIRE(typeid(*stru) != typeid(*stru->clone())); + CHECK(allclose(mzeros, this->pdfcdiff(stru))); + CHECK(BASIC == mpdfco.getEvaluatorTypeUsed()); + const QuantityType g0 = mpdfcb.getPDF(); + CHECK(allclose(mzeros, this->pdfcdiff(stru))); + CHECK(OPTIMIZED == mpdfco.getEvaluatorTypeUsed()); + stru->at(3).xyz_cartn[1] += 0.2; + CHECK(allclose(mzeros, this->pdfcdiff(stru))); + CHECK(OPTIMIZED == mpdfco.getEvaluatorTypeUsed()); + CHECK(!allclose(g0, mpdfcb.getPDF())); + } +} + +TEST_CASE_METHOD(TestPQEvaluatorFixture, + "TestPQEvaluator::test_PDF_structure_kind_in_wrappers", + "[PQEvaluator]") { + // Wrappers compare their source structures. + PeriodicStructureAdapterPtr periodic(new PeriodicStructureAdapter); + periodic->setLatPar(11.0, 5.0, 6.0, 90.0, 90.0, 90.0); + periodic->assign(mstru10->begin(), mstru10->end()); + const StructureAdapterPtr strus[] = {nometa(mstru10), nometa(periodic), + nometa(mstru10), nosymmetry(periodic), + nosymmetry(mstru10)}; + QuantityType glast; + for (const StructureAdapterPtr& stru : strus) { + CHECK(allclose(mzeros, this->pdfcdiff(stru))); + const QuantityType g = mpdfcb.getPDF(); + glast = g; + } + CHECK(!glast.empty()); +} + +TEST_CASE_METHOD(TestPQEvaluatorFixture, + "TestPQEvaluator::test_crystal_site_changes", + "[PQEvaluator]") { + // Changes of single sites of a crystal, which expand only their own + // symmetry positions again. + mpdfcb.setRmax(7.0); + mpdfco.setRmax(7.0); + mzeros.assign(mpdfcb.getRgrid().size(), 0.0); + CrystalStructureAdapterPtr stru = inversionCrystal(0.3); + CHECK(allclose(mzeros, this->pdfcdiff(stru))); + QuantityType glast = mpdfcb.getPDF(); + const std::function changes[] = { + [](CrystalStructureAdapter& s) { s.at(2).xyz_cartn[1] += 0.1; }, + [](CrystalStructureAdapter& s) { s.at(3).uij_cartn *= 1.5; }, + [](CrystalStructureAdapter& s) { s.at(4).occupancy = 0.5; }, + // to a special position and away from it + [](CrystalStructureAdapter& s) { s.at(0).xyz_cartn[0] = 0.0; }, + [](CrystalStructureAdapter& s) { s.at(0).xyz_cartn[0] = 0.2; }, + [](CrystalStructureAdapter& s) { s.at(1).xyz_cartn[2] -= 0.1; }, + }; + bool inplace = false; + for (const auto& change : changes) { + // every second change is made in place + if (!inplace) { + stru = std::dynamic_pointer_cast(stru->clone()); + } + inplace = !inplace; + change(*stru); + CHECK(allclose(mzeros, this->pdfcdiff(stru))); + CHECK(OPTIMIZED == mpdfco.getEvaluatorTypeUsed()); + const QuantityType g = mpdfcb.getPDF(); + CHECK(!allclose(glast, g)); + glast = g; + } +} + TEST_CASE_METHOD(TestPQEvaluatorFixture, "TestPQEvaluator::test_structure_changed_in_place", "[PQEvaluator]") { diff --git a/src/tests/TestPairTable.cpp b/src/tests/TestPairTable.cpp new file mode 100644 index 0000000..16ee71d --- /dev/null +++ b/src/tests/TestPairTable.cpp @@ -0,0 +1,1028 @@ +/***************************************************************************** + * + * libdiffpy Complex Modeling Initiative + * (c) 2026 The Trustees of Columbia University + * in the City of New York. All rights reserved. + * + * See AUTHORS.rst for a list of people who contributed. + * See LICENSE.rst for license information. + * + ****************************************************************************** + * + * TestPairTable -- calculators that add all pairs of a structure at once + * + * The pairs must be those of the bond generators and the results those of + * the pairs added one by one, within rounding errors. + * + *****************************************************************************/ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if __has_include() +#define LIBDIFFPY_TEST_PAIR_TABLE +#include +#include +#endif + +#include "pair_test_helpers.hpp" + +using namespace diffpy::srreal; +using namespace pairtests; +#ifdef LIBDIFFPY_TEST_PAIR_TABLE +using kernels::PairGroups; +using kernels::PairTable; +#endif + +namespace { + +const double TOLERANCE = 1e-12; + +// Pairs --------------------------------------------------------------------- + +#ifdef LIBDIFFPY_TEST_PAIR_TABLE + +struct Pair { + int site0; + int site1; + double dist; + double msd; + double weight; +}; + +/// Pairs from the bond generator, as the evaluator adds them one by one. +std::vector generatedPairs(const StructureAdapter& stru, double rmin, + double rmax) { + std::vector rv; + BaseBondGeneratorPtr bnds = stru.createBondGenerator(); + bnds->setRmin(rmin); + bnds->setRmax(rmax); + for (int i0 = 0; i0 < stru.countSites(); ++i0) { + bnds->selectAnchorSite(i0); + bnds->selectSiteRange(0, i0 + 1); + for (bnds->rewind(); !bnds->finished(); bnds->next()) { + const int scale = (i0 == bnds->site1()) ? 1 : 2; + rv.push_back({i0, bnds->site1(), bnds->distance(), bnds->msd(), + double(scale * bnds->multiplicity())}); + } + } + return rv; +} + +std::vector tablePairs(PairTable& table, const StructureAdapter& stru, + double rmin, double rmax) { + std::vector rv; + const auto g = table.geometryArgs(stru, rmin, rmax); + for (long long i = 0; i < kernels::pairCount(g); ++i) { + const auto p = kernels::pairAt(g, i); + if (p.ok) rv.push_back({p.site0, p.site1, p.dist, p.msd, double(p.weight)}); + } + return rv; +} + +/// The pairs are the same within rounding errors. Distances that differ +/// in their last bits may sort differently, so the distances and the +/// displacements of every pair of sites are compared in their own orders. +void checkSamePairs(const std::vector& expected, + const std::vector& pairs) { + REQUIRE(expected.size() == pairs.size()); + auto sorted = [](std::vector v, double Pair::* member) { + std::sort(v.begin(), v.end(), [=](const Pair& a, const Pair& b) { + return std::tie(a.site0, a.site1, a.weight, a.*member) < + std::tie(b.site0, b.site1, b.weight, b.*member); + }); + return v; + }; + int mismatches = 0; + double maxerror = 0.0; + for (double Pair::* member : {&Pair::dist, &Pair::msd}) { + const std::vector e = sorted(expected, member); + const std::vector p = sorted(pairs, member); + for (size_t i = 0; i < p.size(); ++i) { + mismatches += (e[i].site0 != p[i].site0) || (e[i].site1 != p[i].site1) || + (e[i].weight != p[i].weight); + const double error = std::fabs(e[i].*member - p[i].*member); + maxerror = std::max(maxerror, error / std::fabs(e[i].*member)); + } + } + CHECK(0 == mismatches); + CHECK(maxerror <= TOLERANCE); +} + +#endif + +// Calculators --------------------------------------------------------------- + +/// Value and results of the calculator after its setup. +template +std::vector resultsOf(C&& calc, const Setup& setup, + StructureAdapterPtr s) { + calc.setEvaluatorType(BASIC); + setup(calc); + calc.eval(s); + return {calc.value(), calc.getPDF(), calc.getF()}; +} + +/// Check that a calculator of the class T adds all pairs at once and gives +/// what it does with the pairs one by one. +template +void checkAllPairs(const Setup& setup, StructureAdapterPtr s) { + const auto expected = resultsOf(PairByPair(), setup, s); + AllPairs calc; + const auto results = resultsOf(calc, setup, s); + CHECK(calc.addedAllPairs()); + REQUIRE(expected.size() == results.size()); + // the structure has some pairs + REQUIRE(*std::ranges::max_element(expected[0]) > 0.0); + for (size_t i = 0; i < results.size(); ++i) { + CHECK(maxRelativeError(results[i], expected[i]) <= TOLERANCE); + } + const auto builtin = resultsOf(T(), setup, s); + if constexpr (std::is_same_v) { + CHECK(results[0] == builtin[0]); + // Normalization may round the occupied scattering factors differently. + for (size_t i = 1; i < results.size(); ++i) { + CHECK(maxRelativeError(results[i], builtin[i]) <= TOLERANCE); + } + } else { + CHECK(results == builtin); + } +} + +/// Check that a calculator of the class T adds the pairs one by one. +template +void checkPairByPair(const Setup& setup, StructureAdapterPtr s) { + const auto expected = resultsOf(PairByPair(), setup, s); + REQUIRE(*std::ranges::max_element(expected[0]) > 0.0); + AllPairs calc; + CHECK(expected == resultsOf(calc, setup, s)); + CHECK(!calc.addedAllPairs()); + CHECK(expected == resultsOf(T(), setup, s)); +} + +double maxAbsolute(const QuantityType& v) { + double rv = 0.0; + for (double x : v) rv = std::max(rv, std::fabs(x)); + return rv; +} + +} // namespace + +// Tests --------------------------------------------------------------------- + +#ifdef LIBDIFFPY_TEST_PAIR_TABLE +TEST_CASE("TestPairTable::test_pair_groups", "[PairTable]") { + PairGroups groups; + CHECK(groups.groups().empty()); + const double d = 2.5; + const double dnext = std::nextafter(d, 3.0); + groups.add(d, 0.1, 0, 1.0); + groups.add(d, 0.1, 0, 2.0); + // Adjacent representable inputs can fall on opposite sides of a cutoff. + groups.add(dnext, 0.1, 0, 4.0); + groups.add(d, std::nextafter(0.1, 1.0), 0, 8.0); + REQUIRE(3u == groups.groups().size()); + CHECK(d == groups.groups()[0].dist); + CHECK(0.1 == groups.groups()[0].width); + CHECK(3.0 == groups.groups()[0].scale); + CHECK(4.0 == groups.groups()[1].scale); + CHECK(8.0 == groups.groups()[2].scale); + groups.add(d * (1 + 1e-12), 0.1, 0, 1.0); + groups.add(d, 0.1 * (1 + 1e-12), 0, 1.0); + groups.add(d, 0.1, 1, 1.0); + groups.add(-d, 0.1, 0, 1.0); + groups.add(d, 0.0, 0, 1.0); + CHECK(8u == groups.groups().size()); + CHECK(3.0 == groups.groups()[0].scale); + // many groups keep their sums + groups.clear(); + CHECK(groups.groups().empty()); + const int n = 5000; + for (int repeat = 0; repeat < 3; ++repeat) { + for (int i = 0; i < n; ++i) groups.add(1.0 + i, 0.1, i % 7, i); + } + REQUIRE(size_t(n) == groups.groups().size()); + for (int i = 0; i < n; ++i) { + const PairGroups::Group& g = groups.groups()[i]; + CHECK((1.0 + i == g.dist && i % 7 == g.tag && 3.0 * i == g.scale)); + } +} + +#endif + +TEST_CASE("TestPairTable::test_debye_cutoff_of_adjacent_distances", + "[PairTable]") { + auto stru = std::make_shared(); + Atom atom; + atom.atomtype = "H"; + atom.occupancy = std::nextafter(0.001, 1.0); + stru->append(atom); + atom.xyz_cartn[0] = 2.0; + stru->append(atom); + atom.xyz_cartn = R3::Vector(0, 30, 0); + stru->append(atom); + atom.xyz_cartn[0] = 2.0 + 4 * (std::nextafter(2.0, 3.0) - 2.0); + stru->append(atom); + DebyePDFCalculator grouped; + PairByPair individual; + for (DebyePDFCalculator* calc : + {&grouped, static_cast(&individual)}) { + calc->setRmax(5); + calc->setMaxExtension(0); + calc->setQstep(0.1); + calc->setScatteringFactorTableByType("neutron"); + calc->getScatteringFactorTable()->setCustomAs("H", "H", 1.0); + } + testAllKernels([&] { + const QuantityType reference = individual.eval(stru); + REQUIRE(maxAbsolute(reference) > 0); + CHECK(maxRelativeError(grouped.eval(stru), reference) <= TOLERANCE); + }); +} + +TEST_CASE("TestPairTable::test_ordered_scattering_at_cutoff", "[PairTable]") { + auto stru = std::make_shared(); + Atom atom; + atom.atomtype = "Ni"; + atom.occupancy = 0.0002; + stru->append(atom); + atom.xyz_cartn[0] = 3.0; + atom.occupancy = 7.069469318503158e-05; + stru->append(atom); + DebyePDFCalculator batch; + PairByPair individual; + for (DebyePDFCalculator* c : + {&batch, static_cast(&individual)}) { + c->setEvaluatorType(BASIC); + c->setScatteringFactorTableByType("neutron"); + c->setRmax(8); + } + // Reversing the site factors rounds the first term above the cutoff. + testAllKernels([&] { + CHECK(batch.eval(stru) == individual.eval(stru)); + CHECK(maxAbsolute(batch.value()) == 0.0); + }); +} + +TEST_CASE("TestPairTable::test_debye_width_at_cutoff", "[PairTable]") { + auto stru = std::make_shared(); + Atom atom; + stru->append(atom); + atom.xyz_cartn[0] = 2; + stru->append(atom); + BaseDebyeSum batch; + PairByPair individual; + for (BaseDebyeSum* c : {&batch, static_cast(&individual)}) { + c->setEvaluatorType(BASIC); + c->setRmax(3); + c->setQmin(18); + c->setQmax(19); + c->setQstep(1); + c->setPeakWidthModelByType("constant"); + c->getPeakWidthModel()->setDoubleAttr("width", 0.63); + // Midway between the cutoffs from multiplying and dividing by the + // Gaussian conversion constants. + c->setDebyePrecision(9.20959530020065e-6); + } + KernelGuard guard; + for (const auto& name : availableCalculationKernels()) { + INFO(name); + setCalculationKernel(name); + CHECK(individual.eval(stru).at(18) == 0.0); + CHECK(batch.eval(stru).at(18) == 0.0); + } +} + +TEST_CASE("TestPairTable::test_wrapped_cropped_peak", "[PairTable]") { + auto stru = std::make_shared(); + stru->setLatPar(8, 20, 20, 90, 90, 90); + Atom atom; + atom.atomtype = "Ni"; + atom.xyz_cartn[0] = 7.99; + stru->append(atom); + atom.xyz_cartn[0] = 2.02; + stru->append(atom); + PDFCalculator cached; + cached.setEvaluatorType(BASIC); + cached.setRmax(4); + cached.setMaxExtension(0); + cached.setPeakWidthModelByType("constant"); + cached.setDoubleAttr("width", 0.2); + cached.setPeakProfileByType("croppedgaussian"); + cached.setDoubleAttr("peakprecision", 0.5); + PDFCalculator fresh = cached; + cached.eval(stru); + (*stru)[0].xyz_cartn[0] = 0.02; + CHECK(cached.eval(stru) == fresh.eval(stru)); +} + +TEST_CASE("TestPairTable::test_incremental_update_after_reordering", + "[PairTable]") { + const auto original = asCluster(*loadPeriodic("CaTiO3.stru")); + auto reversed = std::make_shared(); + reversed->assign(original->rbegin(), original->rend()); + auto moved = std::make_shared(*reversed); + (*moved)[2].xyz_cartn[2] += 0.09; + auto check = [&](auto& calc) { + calc.setRmax(8); + calc.eval(original); + calc.eval(reversed); + REQUIRE(calc.getEvaluatorTypeUsed() == OPTIMIZED); + const QuantityType incremental = calc.eval(moved); + REQUIRE(calc.getEvaluatorTypeUsed() == OPTIMIZED); + calc.setEvaluatorType(BASIC); + CHECK(maxRelativeError(incremental, calc.eval(moved)) <= TOLERANCE); + }; + PDFCalculator pdf; + DebyePDFCalculator debye; + check(pdf); + check(debye); +} + +#ifdef LIBDIFFPY_TEST_PAIR_TABLE +TEST_CASE("TestPairTable::test_pairs_of_bond_generator", "[PairTable]") { + const std::pair ranges[] = { + {0.0, 9.7}, {5.0, 4.0}, {0.0, 0.0}, {6.1, 6.9}, {2.0, 12.0}}; + for (const auto& [name, stru] : allStructures()) { + INFO(name); + CHECK(PairTable::supports(*stru)); + PairTable table; + for (const auto& [rmin, rmax] : ranges) { + INFO("range " << rmin << " to " << rmax); + const std::vector expected = generatedPairs(*stru, rmin, rmax); + checkSamePairs(expected, tablePairs(table, *stru, rmin, rmax)); + // and again from the remembered pairs + checkSamePairs(expected, tablePairs(table, *stru, rmin, rmax)); + } + } +} + +TEST_CASE("TestPairTable::test_unsupported_structures", "[PairTable]") { + struct DerivedPeriodic : PeriodicStructureAdapter {}; + struct DerivedAtomic : AtomicStructureAdapter {}; + const auto periodic = loadPeriodic("Ni.stru"); + auto derived = std::make_shared(); + derived->setLatPar(3.52, 3.52, 3.52, 90, 90, 90); + derived->assign(periodic->begin(), periodic->end()); + CHECK(!PairTable::supports(*derived)); + CHECK(!PairTable::supports(*nometa(derived))); + CHECK(!PairTable::supports(*nosymmetry(derived))); + auto cluster = std::make_shared(); + cluster->assign(periodic->begin(), periodic->end()); + CHECK(!PairTable::supports(*cluster)); + // structures without sites have no pairs + CHECK(PairTable::supports(*emptyStructureAdapter())); + CHECK(PairTable::supports(*std::make_shared())); + PairTable table; + CHECK(tablePairs(table, *emptyStructureAdapter(), 0.0, 10.0).empty()); + // The calculators add the pairs of such structures one by one. + checkPairByPair(setupDefault, derived); + checkPairByPair(setupDefault, cluster); +} + +TEST_CASE("TestPairTable::test_remembered_pairs", "[PairTable]") { + // Small changes of a periodic structure keep its pairs, large ones or + // a longer range search them again. + for (const char* kind : {"periodic", "crystal"}) { + INFO(kind); + const auto periodic = loadPeriodic("CaTiO3.stru"); + std::shared_ptr stru = periodic; + // The first sites of the crystal are at special positions, where a + // site that moves gets more symmetry positions and other pairs. + int i0 = 0; + if (std::string(kind) == "crystal") { + stru = asCrystal(*periodic); + i0 = 4; + } + PairTable table; + const double rmin = 0.0, rmax = 8.0; + auto check = [&](int added = -1) { + const int previous = table.countSearches(); + checkSamePairs(generatedPairs(*stru, rmin, rmax), + tablePairs(table, *stru, rmin, rmax)); + if (added >= 0) CHECK(previous + added == table.countSearches()); + }; + check(1); + check(0); + // changes within the margin of the search + stru->at(i0 + 1).xyz_cartn[0] += 0.05; + // An equivalent site of the crystal crosses the unit cell here. + check(stru == periodic ? 0 : 1); + stru->at(i0).xyz_cartn[2] += 0.03; + stru->at(i0 + 1).uij_cartn *= 1.3; + stru->at(i0 + 2).occupancy = 0.5; + check(0); + const Lattice L = stru->getLattice(); + for (Atom& a : *stru) stru->toFractional(a); + stru->setLatPar(1.0005 * L.a(), 0.9995 * L.b(), L.c(), L.alpha(), 90.02, + L.gamma()); + for (Atom& a : *stru) stru->toCartesian(a); + check(); + // an atom that crosses the unit cell appears at its other side + const R3::Vector xyz0 = stru->at(i0).xyz_cartn; + stru->at(i0).xyz_cartn = + xyz0 - stru->getLattice().cartesian(R3::Vector(0.0, 0.0, 1e-5)); + check(); + stru->at(i0).xyz_cartn -= R3::Vector(0.0, 0.0, 0.06); + check(); + stru->at(i0).xyz_cartn = xyz0; + check(); + // a shorter range needs no search, a longer one does + const int beforeRangeChange = table.countSearches(); + checkSamePairs(generatedPairs(*stru, 1.0, 6.5), + tablePairs(table, *stru, 1.0, 6.5)); + CHECK(beforeRangeChange == table.countSearches()); + checkSamePairs(generatedPairs(*stru, 0.0, 9.0), + tablePairs(table, *stru, 0.0, 9.0)); + CHECK(beforeRangeChange + 1 == table.countSearches()); + check(0); + // changes beyond the margin + stru->at(i0 + 1).xyz_cartn[1] += 0.4; + check(1); + for (Atom& a : *stru) stru->toFractional(a); + stru->setLatPar(1.03 * L.a(), L.b(), L.c(), L.alpha(), L.beta(), L.gamma()); + for (Atom& a : *stru) stru->toCartesian(a); + check(1); + // A site that moves by less than the margin can still bring pairs + // from beyond the margin into the range, with the site that moves + // the other way. + stru->at(i0 + 1).xyz_cartn[2] += 0.2; + check(1); + stru->at(i0 + 2).xyz_cartn[2] -= 0.1; + check(0); + stru->at(i0 + 2).xyz_cartn[2] -= 0.1; + check(1); + // other sites + stru->append(stru->at(i0)); + stru->at(stru->countSites() - 1).xyz_cartn[1] += 1.0; + check(1); + stru->erase(1); + check(1); + // The pairs below the range of a search are not remembered. + PairTable other; + checkSamePairs(generatedPairs(*stru, 4.0, 7.0), + tablePairs(other, *stru, 4.0, 7.0)); + checkSamePairs(generatedPairs(*stru, 3.9, 7.1), + tablePairs(other, *stru, 3.9, 7.1)); + CHECK(1 == other.countSearches()); + checkSamePairs(generatedPairs(*stru, 3.0, 7.0), + tablePairs(other, *stru, 3.0, 7.0)); + CHECK(2 == other.countSearches()); + stru->at(i0).xyz_cartn[0] += 0.07; + checkSamePairs(generatedPairs(*stru, 2.9, 7.0), + tablePairs(other, *stru, 2.9, 7.0)); + CHECK(2 == other.countSearches()); + checkSamePairs(generatedPairs(*stru, 2.85, 7.0), + tablePairs(other, *stru, 2.85, 7.0)); + CHECK(3 == other.countSearches()); + } +} + +TEST_CASE("TestPairTable::test_strained_lattice", "[PairTable]") { + // The pairs are searched again when the changes of the positions and of + // the translations together exceed the margin, even if neither does. + auto stru = std::make_shared(); + stru->setLatPar(3.0, 3.0, 3.0, 90.0, 90.0, 90.0); + Atom a; + a.atomtype = "Ni"; + a.uij_cartn = 0.005 * R3::identity(); + stru->append(a); + a.xyz_cartn = R3::Vector(1.5, 0.0, 0.0); + stru->append(a); + PairTable table; + auto check = [&](double alat, int searches) { + INFO("lattice parameter " << alat); + for (Atom& ai : *stru) stru->toFractional(ai); + stru->setLatPar(alat, 3.0, 3.0, 90.0, 90.0, 90.0); + for (Atom& ai : *stru) stru->toCartesian(ai); + checkSamePairs(generatedPairs(*stru, 0.0, 8.0), + tablePairs(table, *stru, 0.0, 8.0)); + CHECK(searches == table.countSearches()); + }; + check(3.0, 1); + // The second atom is up to three cells from the first one within the + // range: the translation changes by 0.15 and the position by 0.025. + check(3.05, 1); + // by 0.21 and by 0.035, which is 0.28 for the distance of two atoms + check(3.07, 2); + check(3.05, 2); +} + +TEST_CASE("TestPairTable::test_new_neighbors_after_contraction", + "[PairTable]") { + auto stru = std::make_shared(); + stru->append(Atom()); + // The first search finds either no translations or only translations + // along b. Shortening a brings a new pair of neighbors into range. + for (double b : {6.0, 2.0}) { + PairTable table; + stru->setLatPar(6, b, 6, 90, 90, 90); + const auto before = tablePairs(table, *stru, 0, 5); + stru->setLatPar(4.9, b, 6, 90, 90, 90); + const auto expected = generatedPairs(*stru, 0, 5); + REQUIRE(expected.size() > before.size()); + checkSamePairs(expected, tablePairs(table, *stru, 0, 5)); + } +} + +TEST_CASE("TestPairTable::test_retry_interrupted_search", "[PairTable]") { + // A stack adapter cannot supply the shared owner needed by its generator. + PeriodicStructureAdapter stack; + stack.setLatPar(3, 3, 3, 90, 90, 90); + stack.append(Atom{}); + PairTable table; + REQUIRE_THROWS_AS(table.geometryArgs(stack, 0, 3.1), std::bad_weak_ptr); + auto owned = std::make_shared(stack); + const auto expected = generatedPairs(*owned, 0, 3.1); + REQUIRE(expected.size() == 6); + checkSamePairs(expected, tablePairs(table, *owned, 0, 3.1)); +} + +TEST_CASE("TestPairTable::test_near_coincidence_on_a_short_axis", + "[PairTable]") { + auto stru = std::make_shared(); + stru->setLatPar(0.005, 1, 1, 90, 90, 90); + Atom site; + site.xyz_cartn = R3::Vector(0.00125, 0.2, 0.2); + stru->append(site); + site.xyz_cartn[0] += 1.3e-8; + stru->append(site); + PairTable table; + REQUIRE(generatedPairs(*stru, 0, 0.001).empty()); + REQUIRE(tablePairs(table, *stru, 0, 0.001).empty()); + // The dormant pair follows the regular neighbors of the last anchor, + // preserving their assignment to reduction rows. + const auto g = table.geometryArgs(*stru, 0, 0.001); + REQUIRE(g.cntpairs > 0); + const auto last = kernels::pairAt(g, g.cntpairs - 1); + CHECK(last.site0 == 1); + CHECK(last.site1 == 0); + CHECK(last.dist <= g.zeroeps); + stru->at(1).xyz_cartn[0] += 0.0001; + const auto expected = generatedPairs(*stru, 0, 0.001); + REQUIRE(expected.size() == 1); + checkSamePairs(expected, tablePairs(table, *stru, 0, 0.001)); + CHECK(table.countSearches() == 1); +} + +TEST_CASE("TestPairTable::test_coinciding_sites", "[PairTable]") { + // Sites at the same position make no bond until they move apart. + for (const char* kind : {"periodic", "crystal", "cluster"}) { + INFO(kind); + const auto periodic = loadPeriodic("CaTiO3.stru"); + // a second atom on the first site and one a lattice vector away + periodic->append(periodic->at(0)); + periodic->at(periodic->countSites() - 1).atomtype = "Sr"; + Atom away = periodic->at(3); + away.xyz_cartn += periodic->getLattice().cartesian(R3::Vector(1, -2, 0)); + periodic->append(away); + std::shared_ptr stru = periodic; + if (std::string(kind) == "crystal") { + auto crystal = asCrystal(*periodic); + crystal->append(crystal->at(0)); + stru = crystal; + } + if (std::string(kind) == "cluster") stru = asCluster(*periodic); + PairTable table; + auto check = [&] { + checkSamePairs(generatedPairs(*stru, 0.0, 7.0), + tablePairs(table, *stru, 0.0, 7.0)); + }; + check(); + const size_t cntpairs = tablePairs(table, *stru, 0.0, 7.0).size(); + const int last = stru->countSites() - 1; + stru->at(last).xyz_cartn[0] += 0.01; + stru->at(last - 1).xyz_cartn[1] += 0.02; + check(); + CHECK(cntpairs < tablePairs(table, *stru, 0.0, 7.0).size()); + // The sites of the crystal get other symmetry positions and their + // pairs are searched again. + if (std::string(kind) != "crystal") CHECK(1 >= table.countSearches()); + stru->at(last).xyz_cartn[0] -= 0.01; + stru->at(last - 1).xyz_cartn[1] -= 0.02; + check(); + CHECK(cntpairs == tablePairs(table, *stru, 0.0, 7.0).size()); + } +} + +TEST_CASE("TestPairTable::test_special_positions", "[PairTable]") { + // A site that moves to a special position has fewer positions. + auto stru = std::make_shared(); + stru->setLatPar(4.0, 4.5, 5.0, 90.0, 90.0, 90.0); + stru->addSymOp(R3::identity(), R3::zerovector); + stru->addSymOp(-1.0 * R3::identity(), R3::zerovector); + Atom a; + a.atomtype = "Ni"; + a.uij_cartn = 0.005 * R3::identity(); + for (const R3::Vector& xyz : + {R3::Vector(0.3, 0.0, 0.0), R3::Vector(0.9, 0.6, 1.3), + R3::Vector(1.7, 1.2, 0.5)}) { + a.xyz_cartn = xyz; + stru->append(a); + } + PairTable table; + auto check = [&] { + checkSamePairs(generatedPairs(*stru, 0.0, 7.0), + tablePairs(table, *stru, 0.0, 7.0)); + }; + check(); + CHECK(2 == stru->siteMultiplicity(0)); + stru->at(0).xyz_cartn[0] = 0.0; + check(); + CHECK(1 == stru->siteMultiplicity(0)); + stru->at(0).xyz_cartn[0] = 0.01; + check(); + stru->addSymOp(R3::identity(), R3::Vector(0.5, 0.5, 0.5)); + check(); + stru->setSymmetryPrecision(0.5); + check(); +} + +#endif + +TEST_CASE("TestPairTable::test_results_of_calculators", "[PairTable]") { + testAllKernels([] { + forEachCalculatorCase( + [](const Setup& setup, StructureAdapterPtr structure) { + checkAllPairs(setup, structure); + checkAllPairs(setup, structure); + }); + }); +} + +TEST_CASE("TestPairTable::test_peak_profiles", "[PairTable]") { + // Any peak profile is linear in the scale of its peaks. + struct WideGaussian : GaussianProfile { + double operator()(double x, double fwhm) const override { + return this->GaussianProfile::operator()(x, 1.5 * fwhm); + } + }; + const auto periodic = loadPeriodic("CaTiO3.stru"); + const Setup cropped = [](PairQuantity& c) { + setupDefault(c); + dynamic_cast(c).setPeakProfileByType("croppedgaussian"); + }; + const Setup wide = [](PairQuantity& c) { + setupDefault(c); + PeakProfilePtr pkf(new WideGaussian); + pkf->setPrecision(1e-5); + dynamic_cast(c).setPeakProfile(pkf); + }; + testAllKernels([&] { + for (StructureAdapterPtr stru : + {StructureAdapterPtr(periodic), + StructureAdapterPtr(asCrystal(*periodic)), + StructureAdapterPtr(asCluster(*periodic))}) { + checkAllPairs(cropped, stru); + checkPairByPair(wide, stru); + } + }); +} + +TEST_CASE("TestPairTable::test_pairs_added_one_by_one", "[PairTable]") { + // A derived class may add its pairs in another way, as these that add + // every pair twice. + struct TwicePDFCalculator : PDFCalculator { + void addPairContribution(const BaseBondGenerator& bnds, + int summationscale) override { + this->PDFCalculator::addPairContribution(bnds, 2 * summationscale); + } + }; + struct TwiceDebyePDFCalculator : DebyePDFCalculator { + void addPairContribution(const BaseBondGenerator& bnds, + int summationscale) override { + this->DebyePDFCalculator::addPairContribution(bnds, 2 * summationscale); + } + }; + const StructureAdapterPtr stru = loadPeriodic("CaTiO3.stru"); + auto twice = [](QuantityType v) { + for (double& x : v) x *= 2; + return v; + }; + const auto once = resultsOf(PDFCalculator(), setupDefault, stru); + const auto doubled = resultsOf(TwicePDFCalculator(), setupDefault, stru); + CHECK(maxRelativeError(doubled[0], twice(once[0])) <= TOLERANCE); + const auto donce = resultsOf(DebyePDFCalculator(), setupDefault, stru); + const auto ddoubled = + resultsOf(TwiceDebyePDFCalculator(), setupDefault, stru); + CHECK(maxRelativeError(ddoubled[0], twice(donce[0])) <= TOLERANCE); + // The other peak width models need the bond of each pair. + struct DerivedWidth : JeongPeakWidth {}; + const Setup derivedwidth = [](PairQuantity& c) { + setupDefault(c); + dynamic_cast(c).setPeakWidthModel( + PeakWidthModelPtr(new DerivedWidth)); + }; + checkPairByPair(derivedwidth, stru); + checkPairByPair(derivedwidth, stru); + // One part of a parallel run has a share of the pairs. + const Setup parallel = [](PairQuantity& c) { + setupDefault(c); + c.setupParallelRun(0, 2); + }; + checkPairByPair(parallel, stru); +} + +TEST_CASE("TestPairTable::test_masks", "[PairTable]") { + // The masks leave out the same pairs as for the pairs added one by one. + const std::pair setups[] = { + {"without the pairs of two types", + [](PairQuantity& c) { + setupDefault(c); + c.setTypeMask("Ti4+", "O2-", false); + }}, + {"only the pairs of one type with any other", + [](PairQuantity& c) { + setupDefault(c); + c.maskAllPairs(false); + c.setTypeMask("Ca2+", PairQuantity::ALLATOMSSTR, true); + }}, + {"without some pairs of sites", + [](PairQuantity& c) { + setupDefault(c); + c.setPairMask(0, 5, false); + c.setPairMask(2, 2, false); + c.setPairMask(9, 1, false); + }}, + {"without one site", + [](PairQuantity& c) { + setupDefault(c); + c.setPairMask(4, PairQuantity::ALLATOMSINT, false); + }}, + {"only one site with itself", + [](PairQuantity& c) { + setupDefault(c); + c.maskAllPairs(false); + c.setPairMask(5, 5, true); + }}, + }; + const auto periodic = loadPeriodic("CaTiO3.stru"); + const std::pair structures[] = { + {"periodic", periodic}, {"crystal", asCrystal(*periodic)}}; + for (const auto& [kind, stru] : structures) { + AllPairs pdfc; + AllPairs dpdfc; + setupDefault(pdfc); + setupDefault(dpdfc); + const QuantityType vall = pdfc.eval(stru); + const QuantityType dvall = dpdfc.eval(stru); + for (const auto& [name, setup] : setups) { + INFO(kind << " " << name); + checkAllPairs(setup, stru); + checkAllPairs(setup, stru); + // the mask leaves out some pairs + setup(pdfc); + setup(dpdfc); + CHECK(maxRelativeError(pdfc.eval(stru), vall) > 1e-3); + CHECK(maxRelativeError(dpdfc.eval(stru), dvall) > 1e-3); + pdfc.maskAllPairs(true); + dpdfc.maskAllPairs(true); + } + // A mask of all pairs changes nothing in the sum. + const int cntsites = stru->countSites(); + for (PairQuantity* c : {(PairQuantity*)&pdfc, (PairQuantity*)&dpdfc}) { + c->maskAllPairs(false); + CHECK(maxAbsolute(c->eval(stru)) == 0.0); + for (int i = 0; i < cntsites; ++i) { + for (int j = 0; j <= i; ++j) c->setPairMask(i, j, true); + } + } + CHECK(vall == pdfc.eval(stru)); + CHECK(dvall == dpdfc.eval(stru)); + CHECK(pdfc.addedAllPairs()); + CHECK(dpdfc.addedAllPairs()); + } +} + +TEST_CASE("TestPairTable::test_evaluators", "[PairTable]") { + const StructureAdapterPtr stru = loadPeriodic("CaTiO3.stru"); + const QuantityType expected = + resultsOf(PairByPair(), setupDefault, stru)[0]; + for (PQEvaluatorType tp : {BASIC, OPTIMIZED, PQEvaluatorType::CHECK}) { + INFO("evaluator " << tp); + AllPairs pdfc; + setupDefault(pdfc); + pdfc.setEvaluatorType(tp); + CHECK(!pdfc.addedAllPairs()); + pdfc.eval(stru); + CHECK(pdfc.addedAllPairs()); + CHECK(maxRelativeError(pdfc.value(), expected) <= TOLERANCE); + const QuantityType v = pdfc.value(); + pdfc.eval(stru); + if (tp != BASIC) CHECK(tp == pdfc.getEvaluatorTypeUsed()); + CHECK(v == pdfc.value()); + } +} + +TEST_CASE("TestPairTable::test_fast_updates", "[PairTable]") { + // The OPTIMIZED evaluator updates the sum of all pairs for the sites + // that change, pair by pair. + for (const auto& [name, stru0] : allStructures()) { + INFO(name); + PairByPair basic; + PairByPair dbasic; + AllPairs optimized; + AllPairs doptimized; + for (PairQuantity* c : {(PairQuantity*)&basic, (PairQuantity*)&dbasic}) { + c->setEvaluatorType(BASIC); + } + for (PairQuantity* c : + {(PairQuantity*)&basic, (PairQuantity*)&optimized, + (PairQuantity*)&dbasic, (PairQuantity*)&doptimized}) { + setupDefault(*c); + } + StructureAdapterPtr stru = stru0->clone(); + for (int step = 0; step < 4; ++step) { + INFO("step " << step); + if (step > 0) { + // one changed site of the source structure + StructureAdapter* s = stru.get(); + while (!dynamic_cast(s)) { + if (auto* nm = dynamic_cast(s)) { + s = nm->getSourceStructure().get(); + } else { + s = dynamic_cast(*s) + .getSourceStructure() + .get(); + } + } + auto& atoms = dynamic_cast(*s); + atoms.at(step % atoms.countSites()).xyz_cartn[step % 3] += 0.07; + } + basic.eval(stru); + optimized.eval(stru); + dbasic.eval(stru); + doptimized.eval(stru); + CHECK(maxRelativeError(optimized.getPDF(), basic.getPDF()) <= 1e-10); + CHECK(maxRelativeError(doptimized.getPDF(), dbasic.getPDF()) <= 1e-10); + // the first calculation is of all pairs + if (!step) CHECK(optimized.addedAllPairs()); + if (!step) CHECK(doptimized.addedAllPairs()); + } + } +} + +TEST_CASE("TestPairTable::test_fast_update_or_all_pairs", "[PairTable]") { + // The OPTIMIZED evaluator adds all pairs again when that takes less work + // than the pairs of the changed sites one by one. + struct Case { + const char* structure; + /// sites that move + std::vector sites; + /// evaluator for these sites when the calculator can add all pairs + PQEvaluatorType used; + }; + const Case cases[] = { + // Four sites, the pairs merge to few peaks. + {"Ni.stru", {1}, BASIC}, + // Twenty sites: one of them takes less work by itself, the four of + // one element do not. + {"CaTiO3.stru", {5}, OPTIMIZED}, + {"CaTiO3.stru", {4, 5, 6, 7}, BASIC}, + // No pairs merge without symmetry. + {"alpha_K2Bi8Se13.stru", {3}, OPTIMIZED}, + {"alpha_K2Bi8Se13.stru", {2, 3, 4, 5}, OPTIMIZED}, + }; + for (const Case& c : cases) { + INFO(c.structure << " with " << c.sites.size() << " moved sites"); + const auto stru = loadPeriodic(c.structure); + // calculators that add all pairs at once, then ones that do not + auto check = [&](PairQuantity& basic, PairQuantity& optimized, + PairQuantity& dbasic, PairQuantity& doptimized, + bool pairbypair) { + INFO((pairbypair ? "pair by pair" : "all pairs")); + for (PairQuantity* calc : {&basic, &optimized, &dbasic, &doptimized}) { + setupDefault(*calc); + } + basic.setEvaluatorType(BASIC); + dbasic.setEvaluatorType(BASIC); + optimized.eval(stru); + doptimized.eval(stru); + const PQEvaluatorType used = pairbypair ? OPTIMIZED : c.used; + for (int step = 0; step < 3; ++step) { + for (int i : c.sites) stru->at(i).xyz_cartn[step] += 0.02; + StructureAdapterPtr copy = stru->clone(); + CHECK(maxRelativeError(optimized.eval(copy), basic.eval(copy)) <= + 1e-10); + CHECK(maxRelativeError(doptimized.eval(copy), dbasic.eval(copy)) <= + 1e-10); + CHECK(used == optimized.getEvaluatorTypeUsed()); + CHECK(used == doptimized.getEvaluatorTypeUsed()); + } + // a structure of the same content takes no work in either way + optimized.eval(stru->clone()); + CHECK(OPTIMIZED == optimized.getEvaluatorTypeUsed()); + // a removed site has half the work of a moved one + stru->erase(c.sites[0]); + CHECK(maxRelativeError(optimized.eval(stru), basic.eval(stru)) <= 1e-10); + CHECK(OPTIMIZED == optimized.getEvaluatorTypeUsed()); + stru->insert(c.sites[0], Atom(stru->at(0))); + }; + PDFCalculator basic, optimized; + DebyePDFCalculator dbasic, doptimized; + check(basic, optimized, dbasic, doptimized, false); + PairByPair pbasic, poptimized; + PairByPair pdbasic, pdoptimized; + check(pbasic, poptimized, pdbasic, pdoptimized, true); + } +} + +TEST_CASE("TestPairTable::test_copies_and_archives", "[PairTable]") { + const StructureAdapterPtr stru = loadPeriodic("CaTiO3.stru"); + AllPairs pdfc; + setupDefault(pdfc); + pdfc.eval(stru); + const QuantityType v0 = pdfc.value(); + // a copy has pairs of its own + AllPairs copy = pdfc; + copy.setEvaluatorType(BASIC); + copy.eval(nosymmetry(stru)); + CHECK(copy.addedAllPairs()); + CHECK(maxRelativeError(copy.value(), v0) > 0.01); + pdfc.setEvaluatorType(BASIC); + pdfc.eval(stru); + CHECK(maxRelativeError(pdfc.value(), v0) <= TOLERANCE); + AllPairs assigned; + assigned = pdfc; + assigned.eval(stru); + CHECK(assigned.addedAllPairs()); + CHECK(maxRelativeError(assigned.value(), v0) <= TOLERANCE); + // and so has a calculator from an archive + auto saved = std::make_shared(); + setupDefault(*saved); + saved->eval(stru); + std::stringstream storage(std::ios::in | std::ios::out | std::ios::binary); + diffpy::serialization::oarchive oa(storage, std::ios::binary); + oa << saved; + diffpy::serialization::iarchive ia(storage, std::ios::binary); + std::shared_ptr loaded; + ia >> loaded; + CHECK(loaded->value() == v0); + loaded->setEvaluatorType(BASIC); + loaded->eval(stru); + CHECK(loaded->value() == v0); +} + +TEST_CASE("TestPairTable::test_many_evaluations", "[PairTable]") { + // A refinement of a crystal: positions, displacements and the lattice + // change a little at every evaluation. + const auto periodic = loadPeriodic("CaTiO3.stru"); + for (std::shared_ptr stru : + {periodic, + std::shared_ptr(asCrystal(*periodic))}) { + PairByPair onebyone; + PairByPair donebyone; + AllPairs atonce; + AllPairs datonce; + for (PairQuantity* c : + {(PairQuantity*)&onebyone, (PairQuantity*)&atonce, + (PairQuantity*)&donebyone, (PairQuantity*)&datonce}) { + setupDefault(*c); + c->setEvaluatorType(BASIC); + } + const Lattice L = stru->getLattice(); + for (int k = 0; k < 40; ++k) { + INFO("evaluation " << k); + const int n = stru->countSites(); + stru->at(k % n).xyz_cartn[k % 3] += 0.004 * ((k % 5) - 2); + stru->at((3 * k) % n).uij_cartn *= 1.0 + 0.01 * ((k % 3) - 1); + for (Atom& a : *stru) stru->toFractional(a); + const double f = 1.0 + 1e-4 * ((k % 7) - 3); + stru->setLatPar(f * L.a(), L.b(), L.c() / f, L.alpha(), L.beta(), + L.gamma()); + for (Atom& a : *stru) stru->toCartesian(a); + onebyone.setDoubleAttr("delta2", 0.5 + 0.05 * (k % 4)); + atonce.setDoubleAttr("delta2", 0.5 + 0.05 * (k % 4)); + StructureAdapterPtr copy = stru->clone(); + onebyone.eval(copy); + atonce.eval(copy); + donebyone.eval(copy); + datonce.eval(copy); + CHECK(atonce.addedAllPairs()); + CHECK(datonce.addedAllPairs()); + CHECK(maxRelativeError(atonce.value(), onebyone.value()) <= TOLERANCE); + CHECK(maxRelativeError(datonce.value(), donebyone.value()) <= TOLERANCE); + } + } +} diff --git a/src/tests/TestPeakWidthModel.cpp b/src/tests/TestPeakWidthModel.cpp index 3f3ee0e..4154846 100644 --- a/src/tests/TestPeakWidthModel.cpp +++ b/src/tests/TestPeakWidthModel.cpp @@ -104,6 +104,9 @@ TEST_CASE_METHOD(TestPeakWidthModelFixture, CHECK_THAT(w, WithinAbs(mjepw->calculate(*bnds), meps)); mcjepw->setDelta2(0.75); CHECK_THAT(0.5 * w, WithinAbs(mjepw->calculate(*bnds), meps)); + const DebyeWallerPeakWidth& model = *mcjepw; + CHECK_THAT(model.calculateFromMSD(bnds->distance(), bnds->msd()), + WithinAbs(0.5 * w, meps)); mcjepw->setDelta2(0); mcjepw->setQbroad(sqrt(3)); mcjepw->setQbroad_seperable(0); diff --git a/src/tests/TestThreads.cpp b/src/tests/TestThreads.cpp new file mode 100644 index 0000000..22f09ca --- /dev/null +++ b/src/tests/TestThreads.cpp @@ -0,0 +1,165 @@ +/***************************************************************************** + * + * libdiffpy Complex Modeling Initiative + * (c) 2026 The Trustees of Columbia University + * in the City of New York. All rights reserved. + * + * See AUTHORS.rst for a list of people who contributed. + * See LICENSE.rst for license information. + * + ****************************************************************************** + * + * TestThreads -- calculators that evaluate in several threads at once + * + * Catch2 assertions are not thread-safe, the threads only collect results + * and the main thread checks them. + * + *****************************************************************************/ + +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "test_helpers.hpp" + +using namespace diffpy::srreal; + +namespace { + +const int NTHREADS = 4; + +/// Results of a short refinement-like sequence of evaluations. The job +/// number selects the calculator and makes the sequences differ. +std::vector evaluateSequence(int job, StructureAdapterPtr stru0) { + PDFCalculator pdfc; + DebyePDFCalculator dpdfc; + const bool usepdf = (job % 2 == 0); + PairQuantity& calc = usepdf ? static_cast(pdfc) + : static_cast(dpdfc); + calc.setDoubleAttr("rmax", 8.0 + job); + calc.setDoubleAttr("qmax", 20.0); + std::vector rv; + for (int k = 0; k < 6; ++k) { + // The first evaluation reads stru0 itself, which other threads may + // read too. The later ones get a new structure with a moved atom and + // changed calculator attributes. + StructureAdapterPtr stru = stru0; + if (k > 0) { + stru = stru0->clone(); + auto& atoms = dynamic_cast(*stru); + atoms[0].xyz_cartn[0] += 0.01 * k * (job + 1); + } + if (k == 3) calc.setDoubleAttr("delta2", 1.0 + 0.1 * job); + if (k == 4) calc.setDoubleAttr("scale", 1.5); + if (k == 5) calc.setDoubleAttr("qmax", 17.0 + job); + calc.eval(stru); + rv.push_back(usepdf ? pdfc.getPDF() : dpdfc.getPDF()); + rv.push_back(usepdf ? pdfc.getF() : dpdfc.getF()); + } + return rv; +} + +/// Cubic crystal with a face-centered Ni site and a body-centered Cu site. +std::shared_ptr makeCrystal() { + auto stru = std::make_shared(); + stru->setLatPar(3.52, 3.52, 3.52, 90, 90, 90); + const double centering[4][3] = { + {0.0, 0.0, 0.0}, {0.0, 0.5, 0.5}, {0.5, 0.0, 0.5}, {0.5, 0.5, 0.0}}; + for (const auto& t : centering) { + stru->addSymOp(R3::identity(), R3::Vector(t[0], t[1], t[2])); + } + Atom a; + a.atomtype = "Ni"; + a.uij_cartn(0, 0) = a.uij_cartn(1, 1) = a.uij_cartn(2, 2) = 0.004; + stru->append(a); + a.atomtype = "Cu"; + a.occupancy = 0.5; + a.xyz_cartn = R3::Vector(0.5, 0.5, 0.5); + stru->toCartesian(a); + stru->append(a); + return stru; +} + +/// Call fn(job) in NTHREADS threads that start together. +template +void runInThreads(Fn fn) { + std::latch start(NTHREADS); + std::vector threads; + try { + for (int job = 0; job < NTHREADS; ++job) { + threads.emplace_back([&, job] { + start.arrive_and_wait(); + fn(job); + }); + } + } catch (...) { + start.count_down(NTHREADS - threads.size()); + throw; + } +} + +} // namespace + +TEST_CASE("TestThreads::test_separate_calculators", "[Threads]") { + const StructureAdapterPtr stru = loadTestPeriodicStructure("CaTiO3.stru"); + std::vector> serial(NTHREADS); + for (int job = 0; job < NTHREADS; ++job) { + serial[job] = evaluateSequence(job, stru->clone()); + } + // Every thread has its own calculator and its own structure. + for (int repeat = 0; repeat < 5; ++repeat) { + std::vector> threaded(NTHREADS); + std::vector strus; + for (int job = 0; job < NTHREADS; ++job) strus.push_back(stru->clone()); + runInThreads( + [&](int job) { threaded[job] = evaluateSequence(job, strus[job]); }); + CHECK(serial == threaded); + } +} + +TEST_CASE("TestThreads::test_shared_structure", "[Threads]") { + std::vector> serial(NTHREADS); + for (int job = 0; job < NTHREADS; ++job) { + serial[job] = evaluateSequence(job, makeCrystal()); + } + // The threads read one crystal, whose symmetry positions are expanded on + // the first use by any of them. + for (int repeat = 0; repeat < 5; ++repeat) { + std::vector> threaded(NTHREADS); + const StructureAdapterPtr shared = makeCrystal(); + runInThreads( + [&](int job) { threaded[job] = evaluateSequence(job, shared); }); + CHECK(serial == threaded); + } +} + +TEST_CASE("TestThreads::test_event_ticker", "[Threads]") { + using diffpy::eventticker::EventTicker; + const int nclicks = 20000; + EventTicker before, after; + std::vector unordered(NTHREADS, 0); + before.click(); + runInThreads([&](int job) { + EventTicker last, current; + for (int i = 0; i < nclicks; ++i) { + current.click(); + if (!(last < current)) ++unordered[job]; + last = current; + } + }); + after.click(); + // No click was lost and every click gave a newer value. + CHECK(before.value().first == after.value().first); + CHECK(long(NTHREADS) * nclicks + 1 == + after.value().second - before.value().second); + CHECK(std::vector(NTHREADS, 0) == unordered); +} diff --git a/src/tests/pair_test_helpers.hpp b/src/tests/pair_test_helpers.hpp new file mode 100644 index 0000000..85294fd --- /dev/null +++ b/src/tests/pair_test_helpers.hpp @@ -0,0 +1,157 @@ +/***************************************************************************** + * + * libdiffpy Complex Modeling Initiative + * (c) 2026 The Trustees of Columbia University + * in the City of New York. All rights reserved. + * + * See AUTHORS.rst for a list of people who contributed. + * See LICENSE.rst for license information. + * + *****************************************************************************/ + +#ifndef PAIR_TEST_HELPERS_HPP_INCLUDED +#define PAIR_TEST_HELPERS_HPP_INCLUDED + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "test_helpers.hpp" + +namespace pairtests { +using namespace diffpy::srreal; + +using Setup = std::function; + +inline std::shared_ptr loadPeriodic( + const std::string& file, bool varyOccupancy = false) { + auto rv = std::dynamic_pointer_cast( + loadTestPeriodicStructure(file)); + REQUIRE(rv); + for (int i = 0; i < rv->countSites(); ++i) { + Atom& a = rv->at(i); + if (a.uij_cartn == R3::zeromatrix()) { + a.uij_cartn = (0.004 + 0.001 * (i % 3)) * R3::identity(); + } + if (varyOccupancy) a.occupancy = 1.0 - 0.1 * (i % 3); + } + return rv; +} + +inline std::shared_ptr asCluster( + const PeriodicStructureAdapter& stru) { + auto rv = std::make_shared(); + rv->assign(stru.begin(), stru.end()); + return rv; +} + +/// The first half of the atoms with the operations of a space group that +/// suits the lattice. +inline std::shared_ptr asCrystal( + const PeriodicStructureAdapter& stru) { + const Lattice& L = stru.getLattice(); + auto rv = std::make_shared(); + rv->setLatPar(L.a(), L.b(), L.c(), L.alpha(), L.beta(), L.gamma()); + const int n = std::max(1, stru.countSites() / 2); + for (int i = 0; i < n; ++i) rv->append(stru[i]); + rv->addSymOp(R3::identity(), R3::zerovector); + rv->addSymOp(-1.0 * R3::identity(), R3::zerovector); + if (L.alpha() == 90.0 && L.beta() == 90.0) { + R3::Matrix R = R3::identity(); + R(0, 0) = -1.0; + R(1, 1) = -1.0; + rv->addSymOp(R, R3::Vector(0.5, 0.5, 0.0)); + rv->addSymOp(-1.0 * R, R3::Vector(0.5, 0.5, 0.0)); + } + return rv; +} + +inline std::vector> allStructures( + bool varyOccupancy = false) { + std::vector> rv; + for (const char* f : {"ZnS_wurtzite.stru", "CaTiO3.stru", "Ni.stru", + "alpha_K2Bi8Se13.stru"}) { + const auto periodic = loadPeriodic(f, varyOccupancy); + const auto crystal = asCrystal(*periodic); + const std::string name = f; + rv.emplace_back(name + " periodic", periodic); + rv.emplace_back(name + " crystal", crystal); + rv.emplace_back(name + " cluster", asCluster(*periodic)); + rv.emplace_back(name + " nometa periodic", nometa(periodic)); + rv.emplace_back(name + " nosymmetry crystal", nosymmetry(crystal)); + rv.emplace_back(name + " nometa nosymmetry periodic", + nometa(nosymmetry(periodic))); + } + return rv; +} + +inline void setupDefault(PairQuantity& calc) { + calc.setRmax(9.5); + calc.setDoubleAttr("qmax", 22.0); +} + +template +void forEachCalculatorCase(Check check) { + for (const auto& [name, structure] : allStructures(true)) { + INFO(name); + check(setupDefault, structure); + } + const auto perovskite = loadPeriodic("CaTiO3.stru", true); + const std::pair setups[] = { + {"jeong", + [](PairQuantity& c) { + setupDefault(c); + c.setDoubleAttr("delta1", 0.3); + c.setDoubleAttr("delta2", 1.2); + c.setDoubleAttr("qbroad", 0.03); + c.setDoubleAttr("qbroad_seperable", 0.02); + }}, + {"debye-waller", + [](PairQuantity& c) { + setupDefault(c); + dynamic_cast(c).setPeakWidthModelByType( + "debye-waller"); + }}, + {"constant", + [](PairQuantity& c) { + setupDefault(c); + dynamic_cast(c).setPeakWidthModelByType( + "constant"); + c.setDoubleAttr("width", 0.21); + }}, + {"sharpened peaks", + [](PairQuantity& c) { + setupDefault(c); + c.setDoubleAttr("delta2", 5.5); + }}, + {"rmin", + [](PairQuantity& c) { + setupDefault(c); + c.setRmin(3.0); + c.setRmax(11.0); + }}, + {"neutrons", + [](PairQuantity& c) { + setupDefault(c); + dynamic_cast(c) + .setScatteringFactorTableByType("neutron"); + }}, + }; + for (const auto& [name, setup] : setups) { + INFO(name); + check(setup, perovskite); + } +} + +} // namespace pairtests +#endif diff --git a/src/tests/test_helpers.hpp b/src/tests/test_helpers.hpp index 16ab01f..1120929 100644 --- a/src/tests/test_helpers.hpp +++ b/src/tests/test_helpers.hpp @@ -50,18 +50,58 @@ class KernelGuard { std::string msaved; }; -/// Call fn() with each available calculation kernel active, then restore the -/// kernel that was active before. Failed checks report their kernel. -/// Do not use SECTION in fn, it would run only with the first kernel. +/// Call fn() with each available CPU kernel active, then restore the kernel +/// that was active before. Do not use SECTION in fn, it would run only with +/// the first kernel. template void testAllKernels(Fn&& fn) { KernelGuard guard; for (const std::string& kname : diffpy::srreal::availableCalculationKernels()) { + if (kname.starts_with("cuda")) continue; diffpy::srreal::setCalculationKernel(kname); INFO("kernel " << kname); fn(); } } +// Sums of all pairs --------------------------------------------------------- + +/// A calculator of the class T that adds its pairs one by one, as the +/// calculators of the library do in every class derived from them. +template +struct PairByPair : T {}; + +/// A calculator of the class T that adds all pairs at once where T does, +/// and tells whether it did. +template +class AllPairs : public T { + public: + /// True when the last full calculation added all pairs at once. + bool addedAllPairs() const { return maddedallpairs; } + /// True when it added them on the GPU. + bool addedOnDevice() const { + return maddedallpairs && this->mallpairs.ondevice; + } + + protected: + // every calculation starts with the value reset + void resetValue() override { + maddedallpairs = false; + this->T::resetValue(); + } + + bool addAllPairContributions() override { + if constexpr (requires { this->addAllPairsOfPDF(); }) { + maddedallpairs = this->addAllPairsOfPDF(); + } else { + maddedallpairs = this->addAllPairsOfDebyeSum(); + } + return maddedallpairs; + } + + private: + bool maddedallpairs = false; +}; + #endif // TEST_HELPERS_HPP_INCLUDED