diff --git a/form/form_source.cpp b/form/form_source.cpp index 7ced2789c..611e24e23 100644 --- a/form/form_source.cpp +++ b/form/form_source.cpp @@ -191,7 +191,7 @@ PHLEX_REGISTER_SOURCE(s, config) } // Register the source object with Phlex - s.source( + s.add_source( module_label, input_cfg, tech_cfg, actual_creator, advertised_creator, products); std::cout << "FORM input source registered successfully\n"; diff --git a/phlex/app/load_module.cpp b/phlex/app/load_module.cpp index 48d52f72c..ecdb54c4a 100644 --- a/phlex/app/load_module.cpp +++ b/phlex/app/load_module.cpp @@ -113,16 +113,17 @@ namespace phlex::experimental { creator(g.source_proxy(config), config); } - driver_bundle load_driver(boost::json::object const& raw_config) + void load_driver(framework_graph& g, boost::json::object const& raw_config) { configuration const config{raw_config}; auto const& spec = config.get("cpp"); + auto const required_sources = config.get>("uses_sources", {}); // False positive: clang-analyzer cannot trace ownership through Boost's is_any_of // internal reference counting in classification.hpp. // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks,clang-analyzer-cplusplus.NewDelete) create_driver = plugin_loader(spec, "create_driver"); driver_bundle result; - create_driver(driver_proxy{}, config, &result); - return result; + create_driver(g.driver_proxy(required_sources), config, &result); + g.add_driver(result); } } diff --git a/phlex/app/load_module.hpp b/phlex/app/load_module.hpp index 6ebab477f..110635bbd 100644 --- a/phlex/app/load_module.hpp +++ b/phlex/app/load_module.hpp @@ -22,7 +22,7 @@ namespace phlex::experimental { RUN_PHLEX_EXPORT void load_source(framework_graph& g, std::string const& label, boost::json::object config); - RUN_PHLEX_EXPORT driver_bundle load_driver(boost::json::object const& config); + RUN_PHLEX_EXPORT void load_driver(framework_graph& g, boost::json::object const& config); } #endif // PHLEX_APP_LOAD_MODULE_HPP diff --git a/phlex/app/run.cpp b/phlex/app/run.cpp index 04d24442f..7f58d8b4d 100644 --- a/phlex/app/run.cpp +++ b/phlex/app/run.cpp @@ -17,8 +17,7 @@ namespace { namespace phlex::experimental { void run(boost::json::object const& configurations, int const max_parallelism) { - auto const driver_config = object_decorate_exception(configurations, "driver"); - framework_graph g{load_driver(driver_config), max_parallelism}; + auto g = framework_graph::without_driver(max_parallelism); // It is allowed for users to not specify any modules boost::json::object module_configs; @@ -38,6 +37,10 @@ namespace phlex::experimental { for (auto const& [key, value] : source_configs) { load_source(g, key, value.as_object()); } + + auto const driver_config = object_decorate_exception(configurations, "driver"); + load_driver(g, driver_config); + g.execute(); } } diff --git a/phlex/core/framework_graph.cpp b/phlex/core/framework_graph.cpp index aa58af777..9eb9bcfdb 100644 --- a/phlex/core/framework_graph.cpp +++ b/phlex/core/framework_graph.cpp @@ -15,24 +15,23 @@ #include namespace phlex::experimental { - framework_graph::framework_graph(int const max_parallelism) : - framework_graph{[](framework_driver& driver) { driver.yield(data_cell_index::job()); }, - max_parallelism} + framework_graph framework_graph::with_default_driver(int const max_parallelism) { + return framework_graph{driver_mode::default_driver, max_parallelism}; } - framework_graph::framework_graph(detail::next_index_t next_index, int const max_parallelism) : - framework_graph{driver_bundle{std::move(next_index), {}}, max_parallelism} + framework_graph framework_graph::without_driver(int const max_parallelism) { + return framework_graph{driver_mode::deferred_driver, max_parallelism}; } - framework_graph::framework_graph(driver_bundle bundle, int const max_parallelism) : + // NOLINTNEXTLINE(bugprone-easily-swappable-parameters) + framework_graph::framework_graph(driver_mode const mode, int const max_parallelism) : parallelism_limit_{static_cast(max_parallelism)}, - fixed_hierarchy_{std::move(bundle.hierarchy)}, - driver_{std::move(bundle.driver)}, src_{graph_, [this](tbb::flow_control& fc) mutable -> ready_flushes_then_emit { - if (auto item = driver_()) { + assert(driver_); + if (auto item = (*driver_)()) { return {.ready_flushes = cell_tracker_.report_and_evict_ready_flushes(*item), .index_to_emit = *item}; } @@ -51,12 +50,33 @@ namespace phlex::experimental { [this](data_cell_index_ptr const& index) -> tbb::flow::continue_msg { hierarchy_.increment_count(index); return {}; - }} + }}, + driver_mode_{mode} { + if (driver_mode_ == driver_mode::default_driver) { + driver_.emplace([](framework_driver& driver) { driver.yield(data_cell_index::job()); }); + } + spdlog::cfg::load_env_levels(); spdlog::info("Number of worker threads: {}", max_allowed_parallelism::active_value()); } + void framework_graph::add_driver(driver_bundle bundle) + { + if (driver_mode_ != driver_mode::deferred_driver) { + throw std::runtime_error( + "Cannot configure framework_graph with a driver when not in deferred mode."); + } + if (driver_) { + throw std::runtime_error("Driver has already been configured for framework_graph."); + } + if (!bundle.driver) { + throw std::runtime_error("Cannot configure framework_graph with an empty driver."); + } + fixed_hierarchy_ = std::move(bundle.hierarchy); + driver_.emplace(std::move(bundle.driver)); + } + framework_graph::~framework_graph() { if (shutdown_on_error_) { @@ -79,19 +99,26 @@ namespace phlex::experimental { } void framework_graph::execute() - try { - finalize(); - run(); - } catch (std::exception const& e) { - driver_.stop(); - spdlog::error(e.what()); - shutdown_on_error_ = true; - throw; - } catch (...) { - driver_.stop(); - spdlog::error("Unknown exception during graph execution"); - shutdown_on_error_ = true; - throw; + { + if (!driver_) { + throw std::runtime_error("No driver configured for framework_graph."); + } + + try { + finalize(); + run(); + } catch (std::exception const& e) { + driver_->stop(); + + spdlog::error(e.what()); + shutdown_on_error_ = true; + throw; + } catch (...) { + driver_->stop(); + spdlog::error("Unknown exception during graph execution"); + shutdown_on_error_ = true; + throw; + } } void framework_graph::run() diff --git a/phlex/core/framework_graph.hpp b/phlex/core/framework_graph.hpp index 5a359ab16..159ba8d99 100644 --- a/phlex/core/framework_graph.hpp +++ b/phlex/core/framework_graph.hpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -38,17 +39,28 @@ namespace phlex { namespace phlex::experimental { class PHLEX_CORE_EXPORT framework_graph { public: - explicit framework_graph(int max_parallelism = oneapi::tbb::info::default_concurrency()); - explicit framework_graph(detail::next_index_t next_index, - int max_parallelism = oneapi::tbb::info::default_concurrency()); - explicit framework_graph(driver_bundle bundle, - int max_parallelism = oneapi::tbb::info::default_concurrency()); + [[nodiscard]] static framework_graph with_default_driver( + int max_parallelism = oneapi::tbb::info::default_concurrency()); + [[nodiscard]] static framework_graph without_driver( + int max_parallelism = oneapi::tbb::info::default_concurrency()); + ~framework_graph(); framework_graph(framework_graph const&) = delete; framework_graph& operator=(framework_graph const&) = delete; framework_graph(framework_graph&&) = delete; framework_graph& operator=(framework_graph&&) = delete; + void add_driver(driver_bundle bundle); + + template + requires requires(std::shared_ptr generator, std::vector sources) { + { experimental::driver_proxy{sources}.driver(generator) } -> std::same_as; + } + void add_driver(std::shared_ptr generator) + { + add_driver(driver_proxy().driver(std::move(generator))); + } + void execute(); std::size_t seen_cell_count(std::string const& layer_name, bool missing_ok = false) const; @@ -64,6 +76,11 @@ namespace phlex::experimental { return {config, graph_, nodes_, registration_errors_}; } + experimental::driver_proxy driver_proxy(std::vector strings = {}) + { + return experimental::driver_proxy(nodes_.sources_for(strings)); + } + // Framework function registrations // N.B. declare_output() is not directly accessible through framework_graph. Is this @@ -115,9 +132,9 @@ namespace phlex::experimental { } template Source, typename... Args> - void source(std::string name, Args&&... args) + void add_source(std::string name, Args&&... args) { - return make_glue().template source(std::move(name), std::forward(args)...); + return make_glue().template add_source(std::move(name), std::forward(args)...); } template @@ -170,6 +187,9 @@ namespace phlex::experimental { void finalize_router(index_router::provider_input_ports_t provider_input_ports, std::map multilayer_join_index_ports); + enum class driver_mode { default_driver, deferred_driver }; + explicit framework_graph(driver_mode mode, int max_parallelism); + resource_usage graph_resource_usage_{}; max_allowed_parallelism parallelism_limit_; fixed_hierarchy fixed_hierarchy_; @@ -178,7 +198,7 @@ namespace phlex::experimental { std::map filters_{}; // The graph_ object uses the filters_, nodes_, and hierarchy_ objects implicitly. tbb::flow::graph graph_{}; - framework_driver driver_; + std::optional driver_{}; std::vector registration_errors_{}; data_cell_tracker cell_tracker_{}; tbb::flow::input_node src_; @@ -187,6 +207,7 @@ namespace phlex::experimental { index_receiver_; tbb::flow::function_node hierarchy_node_; + driver_mode driver_mode_{driver_mode::default_driver}; bool shutdown_on_error_{false}; }; } diff --git a/phlex/core/glue.hpp b/phlex/core/glue.hpp index 75c54412e..9b22314be 100644 --- a/phlex/core/glue.hpp +++ b/phlex/core/glue.hpp @@ -152,7 +152,7 @@ namespace phlex::experimental { } template Source, typename... Args> - void source(std::string name, Args&&... args) + void add_source(std::string name, Args&&... args) { auto [_, inserted] = nodes_.sources.try_emplace(name, std::make_unique(std::forward(args)...)); diff --git a/phlex/core/graph_proxy.hpp b/phlex/core/graph_proxy.hpp index 8e3821bbb..866a8fef1 100644 --- a/phlex/core/graph_proxy.hpp +++ b/phlex/core/graph_proxy.hpp @@ -109,13 +109,13 @@ namespace phlex::experimental { /// @brief Registers a source (used by the framework to create provider nodes) template Source, typename... Args> - void source(std::string name, Args&&... args) + void add_source(std::string name, Args&&... args) requires(not is_bound_object) { // The bound object is created when invoking source(...), so we explicitly indicate that // no bound object should be used in the create_glue(...) call. - return create_glue(false).template source(std::move(name), - std::forward(args)...); + return create_glue(false).template add_source(std::move(name), + std::forward(args)...); } /// @brief Registers an output node. diff --git a/phlex/core/node_catalog.cpp b/phlex/core/node_catalog.cpp index 577df3202..355ba0c30 100644 --- a/phlex/core/node_catalog.cpp +++ b/phlex/core/node_catalog.cpp @@ -1,6 +1,9 @@ #include "phlex/core/node_catalog.hpp" +#include "fmt/format.h" + #include +#include using namespace std::string_literals; @@ -53,4 +56,18 @@ namespace phlex::experimental { { return producer_catalog{transforms, folds, unfolds}; } + + source_vector node_catalog::sources_for(std::vector const& keys) const + { + source_vector result; + result.reserve(keys.size()); + for (auto const& key : keys) { + if (auto src = sources.get(key)) { + result.push_back(src); + } else { + throw std::runtime_error(fmt::format("Unknown source with name: {}", key)); + } + } + return result; + } } diff --git a/phlex/core/node_catalog.hpp b/phlex/core/node_catalog.hpp index 60fdc57f2..bdeef5a2b 100644 --- a/phlex/core/node_catalog.hpp +++ b/phlex/core/node_catalog.hpp @@ -28,6 +28,8 @@ namespace phlex::experimental { return registrar{ptr_map_for(), errors}; } + source_vector sources_for(std::vector const& keys) const; + std::size_t execution_count(std::string const& node_name) const; std::vector consumers() const; producer_catalog producers() const; diff --git a/phlex/core/source.hpp b/phlex/core/source.hpp index 2ed7c93a1..033d7d16d 100644 --- a/phlex/core/source.hpp +++ b/phlex/core/source.hpp @@ -43,6 +43,7 @@ namespace phlex::experimental { using source_ptr = std::unique_ptr; using source_map = simple_ptr_map; + using source_vector = std::vector; } #endif // PHLEX_CORE_SOURCE_HPP diff --git a/phlex/driver.hpp b/phlex/driver.hpp index 77156e6d6..20c3be35b 100644 --- a/phlex/driver.hpp +++ b/phlex/driver.hpp @@ -2,16 +2,27 @@ #define PHLEX_DRIVER_HPP #include "phlex/configuration.hpp" -#include "phlex/core/fwd.hpp" +#include "phlex/core/source.hpp" #include "phlex/detail/plugin_macros.hpp" -#include "phlex/model/data_cell_index.hpp" +#include "phlex/metaprogramming/type_deduction.hpp" #include "phlex/model/fixed_hierarchy.hpp" -#include "phlex/model/product_store.hpp" #include "phlex/utilities/resumable_driver.hpp" +#include "boost/core/demangle.hpp" +#include "boost/mp11/algorithm.hpp" +#include "fmt/format.h" + +#include #include +#include #include +#include +#include +#include +#include +#include #include +#include namespace phlex::experimental { class driver_proxy; @@ -22,6 +33,35 @@ namespace phlex::experimental { // Shim type for the extern "C" entry-point: out-parameter avoids returning a C++ type // across a C-linkage boundary. using driver_shim_t = void(driver_proxy, configuration const&, driver_bundle*); + + template + std::remove_cvref_t const& as_driver_source(source const* src, std::size_t index) + { + assert(src != nullptr); + + using expected_source_t = std::remove_cvref_t; + + if (auto const* casted = dynamic_cast(src)) { + return *casted; + } + + throw std::runtime_error( + fmt::format("Driver source type mismatch at source index {}: expected '{}' but got '{}'.", + index, + boost::core::demangle(typeid(expected_source_t).name()), + boost::core::demangle(typeid(*src).name()))); + } + + template + void invoke_driver_with_sources(F& f, + FirstArg&& first_arg, + std::vector const& sources) + { + [&](std::index_sequence) { + f(std::forward(first_arg), + as_driver_source>(sources[Is], Is)...); + }(std::make_index_sequence::value>{}); + } }; /// @brief Bundles the driver function and data hierarchy for the framework. @@ -29,14 +69,35 @@ namespace phlex::experimental { detail::next_index_t driver; ///< Driver function that advances data cells. fixed_hierarchy hierarchy; ///< Data hierarchy traversed by the driver. }; + + template + using is_derived_from_source = std::is_base_of>; + + template + concept is_driver_like_with_sources = + check_parameters::value && + mp11::mp_all_of>, is_derived_from_source>::value; + + template + using source_parameter_types = skip_first_type; } namespace phlex::experimental { template - concept is_driver_like_with_cursor = std::invocable; + concept is_driver_like_with_cursor = is_driver_like_with_sources; template - concept is_driver_like_with_yielder = std::invocable; + concept is_driver_like_with_yielder = is_driver_like_with_sources; + + template + concept is_driver_like = is_driver_like_with_sources || + is_driver_like_with_sources; + + template + concept is_driver_builder_like = requires(T& driver_builder) { + { driver_builder.hierarchy() } -> std::same_as; + { driver_builder.driver_function() } -> is_driver_like; + }; /// @brief Proxy for constructing a driver bundle from a user-supplied driver function. /// @@ -44,6 +105,8 @@ namespace phlex::experimental { /// construct this type directly. class driver_proxy { public: + explicit driver_proxy(std::vector sources) : sources_(std::move(sources)) {} + /// @brief Creates a driver_bundle from a hierarchy and a user-supplied driver function. /// /// @param hierarchy The data hierarchy the driver will traverse. @@ -52,11 +115,10 @@ namespace phlex::experimental { driver_bundle driver(fixed_hierarchy hierarchy, is_driver_like_with_cursor auto driver_function) const { - auto h = hierarchy; - return {[f = std::move(driver_function), h = std::move(h)](framework_driver& d) mutable { - f(h.yield_job(d)); - }, - std::move(hierarchy)}; + return make_driver_bundle( + std::move(hierarchy), + std::move(driver_function), + [](fixed_hierarchy const& h, framework_driver& d) { return h.yield_job(d); }); } /// @brief Creates a driver_bundle from a hierarchy and a user-supplied driver function. @@ -67,12 +129,70 @@ namespace phlex::experimental { driver_bundle driver(fixed_hierarchy hierarchy, is_driver_like_with_yielder auto driver_function) const { + return make_driver_bundle( + std::move(hierarchy), + std::move(driver_function), + [](fixed_hierarchy const& h, framework_driver& d) { return h.yielder(d); }); + } + + template + requires is_driver_builder_like + driver_bundle driver(std::shared_ptr driver_builder) const + { + if (!driver_builder) { + throw std::invalid_argument("Cannot configure driver with an empty driver builder."); + } + + auto hierarchy = driver_builder->hierarchy(); + auto driver_function = driver_builder->driver_function(); + + using first_argument_type = std::remove_cvref_t; + auto first_arg_factory = [hierarchy = hierarchy](fixed_hierarchy const& h, + framework_driver& d) { + if constexpr (std::is_same_v) { + return h.yield_job(d); + } else { + return h.yielder(d); + } + }; + + return make_driver_bundle( + std::move(hierarchy), std::move(driver_function), std::move(first_arg_factory)); + } + + private: + template + void verify_source_parameter_count() const + { + if (mp11::mp_size::value != sources_.size()) { + throw std::invalid_argument("Number of source parameters of driver function does not match " + "the number of sources specified in the configuration."); + } + } + + template + driver_bundle make_driver_bundle(fixed_hierarchy hierarchy, + DriverFunction driver_function, + FirstArgFactory first_arg_factory) const + { + using driver_function_t = std::remove_cvref_t; + using source_parameters_t = + source_parameter_types>; + + verify_source_parameter_count(); + auto h = hierarchy; - return {[f = std::move(driver_function), h = std::move(h)](framework_driver& d) mutable { - f(h.yielder(d)); + return {[f = std::move(driver_function), + h = std::move(h), + srcs = std::move(sources_), + first_arg_factory = std::move(first_arg_factory)](framework_driver& d) mutable { + detail::invoke_driver_with_sources( + f, first_arg_factory(h, d), srcs); }, std::move(hierarchy)}; } + + std::vector sources_; }; } diff --git a/phlex/model/fixed_hierarchy.cpp b/phlex/model/fixed_hierarchy.cpp index 48e1f7a63..2ef1f2f38 100644 --- a/phlex/model/fixed_hierarchy.cpp +++ b/phlex/model/fixed_hierarchy.cpp @@ -13,7 +13,16 @@ #include #include +using phlex::experimental::layer_path; + namespace { + // Removes duplicate paths from the provided list. + std::vector unique_paths(std::vector paths) + { + std::set const unique{std::from_range, std::move(paths)}; + return {std::from_range, std::move(unique)}; + } + // Builds the set of cumulative layer hashes that define the fixed hierarchy. // For example, if the layer paths are ["job", "run", "subrun"] and ["job", "spill"], // the hashes included will correspond to: @@ -27,8 +36,7 @@ namespace { // - "job/spill" // // Each path must be non-empty and may only contain "job" as the first element. - std::set build_hashes( - std::vector const& layer_paths) + std::set build_hashes(std::vector const& layer_paths) { using namespace phlex::experimental; using namespace phlex::experimental::literals; @@ -40,17 +48,19 @@ namespace { return hashes; } - std::vector convert_vector_vector_string( - std::vector>&& layer_paths) + std::vector convert_vector_vector_string( + std::vector>&& layer_path_strings) { using namespace phlex::experimental; - return std::move(layer_paths) | std::views::transform([](std::vector& lp) { - auto lp_as_ids = - lp | std::views::transform([](auto& str) { return identifier(std::move(str)); }) | - std::ranges::to(); - return layer_path(std::move(lp_as_ids)); - }) | - std::ranges::to>(); + auto layer_paths = + std::move(layer_path_strings) | std::views::transform([](std::vector& lp) { + auto lp_as_ids = + lp | std::views::transform([](auto& str) { return identifier(std::move(str)); }) | + std::ranges::to(); + return layer_path(std::move(lp_as_ids)); + }) | + std::ranges::to>(); + return unique_paths(std::move(layer_paths)); } } @@ -102,6 +112,15 @@ namespace phlex { { } + void fixed_hierarchy::update(std::vector layer_paths) + { + std::set merged{std::from_range, std::move(layer_paths_)}; + merged.insert_range(std::move(layer_paths)); + + layer_paths_.assign_range(merged); + layer_hashes_.assign_range(build_hashes(layer_paths_)); + } + void fixed_hierarchy::validate(data_cell_index_ptr const& index) const { if (layer_hashes_.empty()) { diff --git a/phlex/model/fixed_hierarchy.hpp b/phlex/model/fixed_hierarchy.hpp index 4d419b05f..2ea20ff55 100644 --- a/phlex/model/fixed_hierarchy.hpp +++ b/phlex/model/fixed_hierarchy.hpp @@ -87,6 +87,9 @@ namespace phlex { // Returns a callable data-cell yielder bound to the provided driver. data_cell_yielder yielder(experimental::framework_driver& d) const; + // Merges additional layer paths into the hierarchy, ignoring duplicates. + void update(std::vector layer_paths); + private: std::vector layer_paths_; std::vector layer_hashes_; diff --git a/phlex/source.hpp b/phlex/source.hpp index a79870a32..db3b4acf8 100644 --- a/phlex/source.hpp +++ b/phlex/source.hpp @@ -62,7 +62,7 @@ namespace phlex::experimental { } // Only source(...) should be accessible - using base::source; + using base::add_source; }; namespace detail { diff --git a/plugins/generate_layers.cpp b/plugins/generate_layers.cpp index 3ff7517fb..39ca48ed6 100644 --- a/plugins/generate_layers.cpp +++ b/plugins/generate_layers.cpp @@ -19,13 +19,15 @@ #include "phlex/driver.hpp" #include "plugins/layer_generator.hpp" +#include #include +#include PHLEX_REGISTER_DRIVER(d, config) { using namespace phlex; - auto gen = std::make_shared(); + auto gen = experimental::layer_generator::make(); auto const layers = config.get("layers", {}); for (auto const& key : layers.keys()) { @@ -36,9 +38,5 @@ PHLEX_REGISTER_DRIVER(d, config) .starting_value = layer_config.get("starting_number", 0)}); } - return d.driver(gen->hierarchy(), [gen](data_cell_yielder const yield) { - for (data_cell_index_ptr const& index : gen->indices()) { - yield(index); - } - }); + return d.driver(std::move(gen)); } diff --git a/plugins/layer_generator.cpp b/plugins/layer_generator.cpp index 2b9fd3abd..bfb4b7dee 100644 --- a/plugins/layer_generator.cpp +++ b/plugins/layer_generator.cpp @@ -11,6 +11,11 @@ namespace phlex::experimental { + std::shared_ptr layer_generator::make() + { + return std::shared_ptr(new layer_generator{}); + } + layer_generator::layer_generator() { // Always seed the "job" in case only the job is desired @@ -131,6 +136,15 @@ namespace phlex::experimental { } } + std::function layer_generator::driver_function() + { + return [gen = shared_from_this()](data_cell_yielder const yield) { + for (data_cell_index_ptr const& index : gen->indices()) { + yield(index); + } + }; + } + index_generator layer_generator::execute(data_cell_index_ptr const cell) { // Used in drivers which are close to public API --> easier to stick to strings diff --git a/plugins/layer_generator.hpp b/plugins/layer_generator.hpp index e2aa4e77b..48943b891 100644 --- a/plugins/layer_generator.hpp +++ b/plugins/layer_generator.hpp @@ -16,12 +16,12 @@ // // To create the above tree of layers, the following function calls could be made: // -// layer_generator gen; -// gen.add_layer("spill", {"job", 16}); // 16 spill data cells with job as parent -// gen.add_layer("CRU", {"spill", 256}); // 256 CRU data cells per spill parent -// gen.add_layer("run", {"job", 16}); // 16 run data cells with job as parent -// gen.add_layer("APA", {"run", 150, 1}); // 150 APA data cells per run parent -// // with first APA data cell number starting at 1 +// auto gen = layer_generator::make(); +// gen->add_layer("spill", {"job", 16}); // 16 spill data cells with job as parent +// gen->add_layer("CRU", {"spill", 256}); // 256 CRU data cells per spill parent +// gen->add_layer("run", {"job", 16}); // 16 run data cells with job as parent +// gen->add_layer("APA", {"run", 150, 1}); // 150 APA data cells per run parent +// // with first APA data cell number starting at 1 // // ---------------------------------------------------------------------------------------------- // N.B. The layer generator can create data-layer hierarchies that are trees, and not @@ -34,6 +34,7 @@ #include #include +#include #include #include @@ -44,9 +45,11 @@ namespace phlex::experimental { std::size_t starting_value = 0; }; - class layer_generator { + // Inherit enable_shared_from_this so driver_function can capture a shared_ptr to this, + // ensuring layer_generator remains alive while the returned driver callable is in use. + class layer_generator : public std::enable_shared_from_this { public: - layer_generator(); + [[nodiscard]] static std::shared_ptr make(); ~layer_generator() = default; layer_generator(layer_generator const&) = delete; @@ -57,11 +60,14 @@ namespace phlex::experimental { void add_layer(std::string layer_name, layer_spec lspec); index_generator indices(); + std::function driver_function(); fixed_hierarchy hierarchy() const; std::size_t emitted_cell_count(std::string layer_path = {}) const; private: + layer_generator(); + index_generator execute(data_cell_index_ptr const cell); std::string parent_path(std::string const& layer_name, std::string const& parent_layer_spec) const; @@ -75,17 +81,6 @@ namespace phlex::experimental { using reverse_map_t = std::map>; reverse_map_t parent_to_children_; }; - - // N.B. The layer_generator object must outlive whatever uses it. - inline driver_bundle driver_for_test(layer_generator& generator) - { - driver_proxy const proxy{}; - return proxy.driver(generator.hierarchy(), [&generator](data_cell_yielder const yield) { - for (data_cell_index_ptr const& index : generator.indices()) { - yield(index); - } - }); - } } #endif // PLUGINS_LAYER_GENERATOR_HPP diff --git a/test/allowed_families.cpp b/test/allowed_families.cpp index 9d68d9b56..6e2c0bf5f 100644 --- a/test/allowed_families.cpp +++ b/test/allowed_families.cpp @@ -34,12 +34,13 @@ namespace { TEST_CASE("Testing families", "[data model]") { - experimental::layer_generator gen; - gen.add_layer("run", {"job", 1}); - gen.add_layer("subrun", {"run", 1}); - gen.add_layer("event", {"subrun", 1}); + auto gen = experimental::layer_generator::make(); + gen->add_layer("run", {"job", 1}); + gen->add_layer("subrun", {"run", 1}); + gen->add_layer("event", {"subrun", 1}); - experimental::framework_graph g{driver_for_test(gen)}; + auto g = experimental::framework_graph::without_driver(); + g.add_driver(gen); // Wire up providers for each level g.provide("run_id_provider", provide_index, concurrency::unlimited) diff --git a/test/cached_execution.cpp b/test/cached_execution.cpp index bf6768704..9b92d55a0 100644 --- a/test/cached_execution.cpp +++ b/test/cached_execution.cpp @@ -52,12 +52,13 @@ TEST_CASE("Cached function calls", "[data model]") constexpr unsigned int n_subruns{2u}; constexpr unsigned int n_events{5000u}; - experimental::layer_generator gen; - gen.add_layer("run", {"job", n_runs}); - gen.add_layer("subrun", {"run", n_subruns}); - gen.add_layer("event", {"subrun", n_events}); + auto gen = experimental::layer_generator::make(); + gen->add_layer("run", {"job", n_runs}); + gen->add_layer("subrun", {"run", n_subruns}); + gen->add_layer("event", {"subrun", n_events}); - experimental::framework_graph g{driver_for_test(gen)}; + auto g = experimental::framework_graph::without_driver(); + g.add_driver(gen); // Register providers g.provide("provide_number", provide_number, concurrency::unlimited) diff --git a/test/class_registration.cpp b/test/class_registration.cpp index cd45f71cc..fee42b15d 100644 --- a/test/class_registration.cpp +++ b/test/class_registration.cpp @@ -66,7 +66,7 @@ TEST_CASE("Call non-framework functions", "[programming model]") product_selector{.creator = "input", .layer = "job", .suffix = "name"}}; std::array const product_suffixes{"onumber"s, "otemperature"s, "oname"s}; - experimental::framework_graph g; + auto g = experimental::framework_graph::with_default_driver(); // Register providers for the input products g.provide("provide_number", provide_number, concurrency::unlimited) diff --git a/test/demo-giantdata/unfold_transform_fold.cpp b/test/demo-giantdata/unfold_transform_fold.cpp index 1d29c4db3..f47548383 100644 --- a/test/demo-giantdata/unfold_transform_fold.cpp +++ b/test/demo-giantdata/unfold_transform_fold.cpp @@ -45,12 +45,13 @@ TEST_CASE("Unfold-transform-fold pipeline", "[concurrency][unfold][fold]") tracker.total_expected = n_spills * apas_per_spill; // Create data layers using layer generator - experimental::layer_generator gen; - gen.add_layer("run", {"job", n_runs}); - gen.add_layer("subrun", {"run", n_subruns}); - gen.add_layer("spill", {"subrun", n_spills}); + auto gen = experimental::layer_generator::make(); + gen->add_layer("run", {"job", n_runs}); + gen->add_layer("subrun", {"run", n_subruns}); + gen->add_layer("spill", {"subrun", n_spills}); - experimental::framework_graph g{driver_for_test(gen)}; + auto g = experimental::framework_graph::without_driver(); + g.add_driver(gen); g.provide("provide_wgen", [](data_cell_index const& spill_index) { diff --git a/test/filter.cpp b/test/filter.cpp index f57f918b5..da2d5df76 100644 --- a/test/filter.cpp +++ b/test/filter.cpp @@ -102,9 +102,10 @@ namespace { TEST_CASE("Two predicates", "[filtering]") { - experimental::layer_generator gen; - gen.add_layer("event", {"job", 10, 1}); - experimental::framework_graph g{driver_for_test(gen)}; + auto gen = experimental::layer_generator::make(); + gen->add_layer("event", {"job", 10, 1}); + auto g = experimental::framework_graph::without_driver(); + g.add_driver(gen); g.provide("provide_num", give_me_nums, concurrency::unlimited) .output_product("input", "num", "event"); g.predicate("evens_only", evens_only, concurrency::unlimited) @@ -128,9 +129,10 @@ TEST_CASE("Two predicates", "[filtering]") TEST_CASE("Two predicates in series", "[filtering]") { - experimental::layer_generator gen; - gen.add_layer("event", {"job", 10, 1}); - experimental::framework_graph g{driver_for_test(gen)}; + auto gen = experimental::layer_generator::make(); + gen->add_layer("event", {"job", 10, 1}); + auto g = experimental::framework_graph::without_driver(); + g.add_driver(gen); g.provide("provide_num", give_me_nums, concurrency::unlimited) .output_product("input", "num", "event"); g.predicate("evens_only", evens_only, concurrency::unlimited) @@ -150,9 +152,10 @@ TEST_CASE("Two predicates in series", "[filtering]") TEST_CASE("Two predicates in parallel", "[filtering]") { - experimental::layer_generator gen; - gen.add_layer("event", {"job", 10, 1}); - experimental::framework_graph g{driver_for_test(gen)}; + auto gen = experimental::layer_generator::make(); + gen->add_layer("event", {"job", 10, 1}); + auto g = experimental::framework_graph::without_driver(); + g.add_driver(gen); g.provide("provide_num", give_me_nums, concurrency::unlimited) .output_product("input", "num", "event"); g.predicate("evens_only", evens_only, concurrency::unlimited) @@ -180,9 +183,10 @@ TEST_CASE("Three predicates in parallel", "[filtering]") {.name = "exclude_6_to_7", .begin = 6, .end = 7}, {.name = "exclude_gt_8", .begin = 8, .end = -1u}}; - experimental::layer_generator gen; - gen.add_layer("event", {"job", 10, 1}); - experimental::framework_graph g{driver_for_test(gen)}; + auto gen = experimental::layer_generator::make(); + gen->add_layer("event", {"job", 10, 1}); + auto g = experimental::framework_graph::without_driver(); + g.add_driver(gen); g.provide("provide_num", give_me_nums, concurrency::unlimited) .output_product("input", "num", "event"); for (auto const& [name, b, e] : configs) { @@ -206,9 +210,10 @@ TEST_CASE("Three predicates in parallel", "[filtering]") TEST_CASE("Two predicates in parallel (each with multiple arguments)", "[filtering]") { - experimental::layer_generator gen; - gen.add_layer("event", {"job", 10, 1}); - experimental::framework_graph g{driver_for_test(gen)}; + auto gen = experimental::layer_generator::make(); + gen->add_layer("event", {"job", 10, 1}); + auto g = experimental::framework_graph::without_driver(); + g.add_driver(gen); g.provide("provide_num", give_me_nums, concurrency::unlimited) .output_product("input", "num", "event"); g.provide("provide_other_num", give_me_other_nums, concurrency::unlimited) diff --git a/test/fixed_hierarchy_test.cpp b/test/fixed_hierarchy_test.cpp index 4f397e0c4..7b18a4f3d 100644 --- a/test/fixed_hierarchy_test.cpp +++ b/test/fixed_hierarchy_test.cpp @@ -71,6 +71,39 @@ TEST_CASE("Fixed hierarchy rejects indices not present in the hierarchy", "[fixe ContainsSubstring("Layer /job/subrun is not part of the fixed hierarchy")); } +TEST_CASE("Duplicate paths in constructor are deduplicated", "[fixed_hierarchy]") +{ + fixed_hierarchy const h{{"run", "subrun"}, {"run", "subrun"}, {"run", "calibration_period"}}; + // fixed_hierarchy stores layer-paths are stored lexicographically. + std::vector expected{"run/calibration_period", "run/subrun"}; + CHECK(h.layer_paths() == expected); +} + +TEST_CASE("update() adds new paths and ignores duplicates", "[fixed_hierarchy]") +{ + fixed_hierarchy h{{"run", "subrun"}}; + h.update({"run/calibration_period"}); + + std::vector expected = {"run/calibration_period", "run/subrun"}; + CHECK(h.layer_paths() == expected); + + // Adding the same path again is a no-op + h.update({"run/calibration_period", "run/subrun"}); + CHECK(h.layer_paths() == expected); +} + +TEST_CASE("update() extends the validated set of indices", "[fixed_hierarchy]") +{ + fixed_hierarchy h{{"run", "subrun"}}; + auto const job = data_cell_index::job(); + auto const run = job->make_child("run", 0); + + CHECK_THROWS(h.validate(run->make_child("calibration_period", 0))); + + h.update({"run/calibration_period"}); + CHECK_NOTHROW(h.validate(run->make_child("calibration_period", 0))); +} + TEST_CASE("Paths with and without 'job' prefix produce the same hierarchy", "[fixed_hierarchy]") { fixed_hierarchy const without_prefix{{"run", "subrun"}, {"run", "calibration_period"}}; diff --git a/test/fold.cpp b/test/fold.cpp index 64ed00aca..ed1a308f6 100644 --- a/test/fold.cpp +++ b/test/fold.cpp @@ -60,11 +60,12 @@ TEST_CASE("Different data layers of fold", "[graph]") constexpr auto index_limit = 2u; constexpr auto number_limit = 5u; - experimental::layer_generator gen; - gen.add_layer("run", {"job", index_limit}); - gen.add_layer("event", {"run", number_limit}); + auto gen = experimental::layer_generator::make(); + gen->add_layer("run", {"job", index_limit}); + gen->add_layer("event", {"run", number_limit}); - experimental::framework_graph g{driver_for_test(gen)}; + auto g = experimental::framework_graph::without_driver(); + g.add_driver(gen); g.provide("provide_number", provide_number, concurrency::unlimited) .output_product("input", "number", "event"); @@ -103,11 +104,12 @@ TEST_CASE("Fold output without send consumed downstream", "[graph]") constexpr auto index_limit = 2u; constexpr auto number_limit = 5u; - experimental::layer_generator gen; - gen.add_layer("run", {"job", index_limit}); - gen.add_layer("event", {"run", number_limit}); + auto gen = experimental::layer_generator::make(); + gen->add_layer("run", {"job", index_limit}); + gen->add_layer("event", {"run", number_limit}); - experimental::framework_graph g{driver_for_test(gen)}; + auto g = experimental::framework_graph::without_driver(); + g.add_driver(gen); g.provide("provide_number", provide_number, concurrency::unlimited) .output_product("input", "number", "event"); diff --git a/test/fold_duplicate_layer_name_test.cpp b/test/fold_duplicate_layer_name_test.cpp index d7fa81adb..664c1a9a3 100644 --- a/test/fold_duplicate_layer_name_test.cpp +++ b/test/fold_duplicate_layer_name_test.cpp @@ -57,12 +57,13 @@ TEST_CASE("Fold different layer paths with same trailing name", "[graph]") // job -> event layers constexpr auto top_level_event_limit = 10u; - experimental::layer_generator gen; - gen.add_layer("run", {"job", index_limit}); - gen.add_layer("event", {"run", number_limit}); - gen.add_layer("event", {"job", top_level_event_limit}); + auto gen = experimental::layer_generator::make(); + gen->add_layer("run", {"job", index_limit}); + gen->add_layer("event", {"run", number_limit}); + gen->add_layer("event", {"job", top_level_event_limit}); - experimental::framework_graph g{driver_for_test(gen)}; + auto g = experimental::framework_graph::without_driver(); + g.add_driver(gen); // Register provider g.provide("provide_number", provide_number, concurrency::unlimited) diff --git a/test/framework_graph.cpp b/test/framework_graph.cpp index a9ba1a410..045b97424 100644 --- a/test/framework_graph.cpp +++ b/test/framework_graph.cpp @@ -1,33 +1,72 @@ #include "phlex/core/framework_graph.hpp" +#include "phlex/driver.hpp" +#include "phlex/model/data_cell_index.hpp" #include "phlex/utilities/max_allowed_parallelism.hpp" #include "plugins/layer_generator.hpp" #include "catch2/catch_test_macros.hpp" #include "catch2/matchers/catch_matchers_string.hpp" +#include "boost/core/demangle.hpp" +#include "fmt/format.h" + +#include #include +#include +#include using namespace phlex; +using phlex::experimental::driver_bundle; using phlex::experimental::framework_driver; +namespace { + struct test_source final : phlex::experimental::source { + phlex::experimental::provider_bundles create_providers(product_selector const&) override + { + return {}; + } + index_generator indices() override { co_return; } + }; + + struct other_source final : phlex::experimental::source { + phlex::experimental::provider_bundles create_providers(product_selector const&) override + { + return {}; + } + index_generator indices() override { co_return; } + }; + + struct test_driver_builder { + [[nodiscard]] fixed_hierarchy hierarchy() const { return {}; } + + [[nodiscard]] std::function driver_function() const + { + return [](data_cell_yielder const /*yielder*/) {}; + } + }; +} + TEST_CASE("Catch STL exceptions", "[graph]") { - experimental::framework_graph g{[](framework_driver&) { throw std::runtime_error("STL error"); }}; + auto g = experimental::framework_graph::without_driver(); + g.add_driver(driver_bundle{[](framework_driver&) { throw std::runtime_error("STL error"); }, {}}); CHECK_THROWS_AS(g.execute(), std::exception); } TEST_CASE("Catch other exceptions", "[graph]") { - experimental::framework_graph g{[](framework_driver&) { throw 2.5; }}; + auto g = experimental::framework_graph::without_driver(); + g.add_driver(driver_bundle{[](framework_driver&) { throw 2.5; }, {}}); CHECK_THROWS_AS(g.execute(), double); } TEST_CASE("Make progress with one thread", "[graph]") { - experimental::layer_generator gen; - gen.add_layer("spill", {"job", 1000}); + auto gen = experimental::layer_generator::make(); + gen->add_layer("spill", {"job", 1000}); - experimental::framework_graph g{driver_for_test(gen), 1}; + auto g = experimental::framework_graph::without_driver(1); + g.add_driver(gen); g.provide( "provide_number", [](data_cell_index const& index) -> unsigned int { return index.number(); }, @@ -38,17 +77,18 @@ TEST_CASE("Make progress with one thread", "[graph]") .input_family(product_selector{.creator = "input", .layer = "spill", .suffix = "number"}); g.execute(); - CHECK(gen.emitted_cell_count("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/job/spill") == 1000); + CHECK(gen->emitted_cell_count("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/job/spill") == 1000); CHECK(g.execution_count("provide_number") == 1000); CHECK(g.execution_count("observe_number") == 1000); } TEST_CASE("Stop driver when workflow throws exception", "[graph]") { - experimental::layer_generator gen; - gen.add_layer("spill", {"job", 1000}); + auto gen = experimental::layer_generator::make(); + gen->add_layer("spill", {"job", 1000}); - experimental::framework_graph g{driver_for_test(gen)}; + auto g = experimental::framework_graph::without_driver(); + g.add_driver(gen); g.provide( "throw_exception", [](data_cell_index const&) -> unsigned int { @@ -71,7 +111,7 @@ TEST_CASE("Stop driver when workflow throws exception", "[graph]") // "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/job/spill" data layer before the job ends. In that case, the "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/job/spill" layer // will not have been recorded, and we therefore allow it to be "missing", which is what // the 'true' argument allows for. - CHECK(gen.emitted_cell_count("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/job/spill") >= g.seen_cell_count("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/job/spill", true)); + CHECK(gen->emitted_cell_count("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/job/spill") >= g.seen_cell_count("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/job/spill", true)); // A node has not "executed" until it has returned successfully. For that reason, // neither the "throw_exception" provider nor the "downstream_of_exception" observer @@ -82,10 +122,11 @@ TEST_CASE("Stop driver when workflow throws exception", "[graph]") TEST_CASE("Throw when predicate specified by consumer does not exist", "[graph]") { - experimental::layer_generator gen; - gen.add_layer("event", {"job", 1, 1}); + auto gen = experimental::layer_generator::make(); + gen->add_layer("event", {"job", 1, 1}); - experimental::framework_graph g{driver_for_test(gen)}; + auto g = experimental::framework_graph::without_driver(); + g.add_driver(gen); g.provide( "provide_num", [](data_cell_index const& id) -> unsigned int { return id.number(); }, @@ -103,9 +144,64 @@ TEST_CASE("Throw when predicate specified by consumer does not exist", "[graph]" "A non-existent filter with the name 'missing_predicate' was specified for observe_num")); } +TEST_CASE("Throw for invalid deferred driver setup", "[graph]") +{ + auto g = experimental::framework_graph::without_driver(); + + SECTION("Throw when source specified for driver does not exist") + { + CHECK_THROWS_WITH( + g.driver_proxy({"missing_source"}), + Catch::Matchers::ContainsSubstring("Unknown source with name: missing_source")); + } + + SECTION("Throw when no driver configured") + { + CHECK_THROWS_WITH( + g.execute(), Catch::Matchers::ContainsSubstring("No driver configured for framework_graph")); + } + + SECTION("Throw when configuring empty driver bundle") + { + CHECK_THROWS_WITH( + g.add_driver(driver_bundle{}), + Catch::Matchers::ContainsSubstring("Cannot configure framework_graph with an empty driver.")); + } +} + +TEST_CASE("Use default driver", "[graph]") +{ + auto g = experimental::framework_graph::with_default_driver(); + + SECTION("Throw when attempting to add a driver in default mode") + { + CHECK_THROWS_WITH( + g.add_driver(driver_bundle{}), + Catch::Matchers::ContainsSubstring( + "Cannot configure framework_graph with a driver when not in deferred mode.")); + } + + SECTION("Ensure default driver executes one job cell") + { + g.provide( + "provide_number", + [](data_cell_index const&) -> unsigned int { return 42u; }, + concurrency::unlimited) + .output_product("input", "number", "job"); + g.observe( + "observe_number", [](unsigned int const) {}, concurrency::unlimited) + .input_family(product_selector{.creator = "input", .layer = "job", .suffix = "number"}); + + g.execute(); + + CHECK(g.execution_count("provide_number") == 1); + CHECK(g.execution_count("observe_number") == 1); + } +} + TEST_CASE("Throw on duplicate node registration", "[graph]") { - experimental::framework_graph g; + auto g = experimental::framework_graph::with_default_driver(); g.observe( "duplicate_name", [](unsigned int const) {}, concurrency::unlimited) @@ -118,3 +214,104 @@ TEST_CASE("Throw on duplicate node registration", "[graph]") Catch::Matchers::ContainsSubstring("Configuration errors") && Catch::Matchers::ContainsSubstring("duplicate_name")); } + +TEST_CASE("Allow late driver configuration", "[graph]") +{ + auto gen = experimental::layer_generator::make(); + gen->add_layer("spill", {"job", 3}); + + auto g = experimental::framework_graph::without_driver(); + + g.provide( + "provide_number", + [](data_cell_index const& index) -> unsigned int { return index.number(); }, + concurrency::unlimited) + .output_product("input", "number", "spill"); + g.observe( + "observe_number", [](unsigned int const /*number*/) {}, concurrency::unlimited) + .input_family(product_selector{.creator = "input", .layer = "spill", .suffix = "number"}); + + g.add_driver(gen); + g.execute(); + + CHECK(g.execution_count("provide_number") == 3); + CHECK(g.execution_count("observe_number") == 3); +} + +TEST_CASE("driver_proxy validates sources and generator", "[graph]") +{ + std::vector sources{}; + auto src = std::make_unique(); + sources.push_back(src.get()); + experimental::driver_proxy proxy{sources}; + + SECTION("Throw when source parameter count mismatches") + { + CHECK_THROWS_WITH( + proxy.driver(fixed_hierarchy{}, [](data_cell_cursor) {}), + Catch::Matchers::ContainsSubstring( + "Number of source parameters of driver function does not match the number of sources ") && + Catch::Matchers::ContainsSubstring("specified in the configuration")); + } + + SECTION("Throw when driver_builder is empty") + { + std::shared_ptr const null_generator{nullptr}; + CHECK_THROWS_WITH( + proxy.driver(null_generator), + Catch::Matchers::ContainsSubstring("Cannot configure driver with an empty driver builder")); + } +} + +TEST_CASE("driver_proxy creates bundle from driver builder", "[graph]") +{ + experimental::driver_proxy proxy{{}}; + auto const bundle = proxy.driver(std::make_shared()); + + CHECK(static_cast(bundle.driver)); +} + +TEST_CASE("Driver function receives registered source", "[graph]") +{ + auto g = experimental::framework_graph::without_driver(); + g.add_source("src"); + + test_source const* received_src{nullptr}; + auto bundle = g.driver_proxy({"src"}).driver( + fixed_hierarchy{}, + [&received_src](data_cell_cursor, test_source const& src) { received_src = &src; }); + g.add_driver(std::move(bundle)); + g.execute(); + + CHECK(received_src != nullptr); +} + +TEST_CASE("Driver function throws on source type mismatch", "[graph]") +{ + // Register other_source but declare test_source const& in the driver function. + // The source downcast inside invoke_driver_with_sources throws with context. + auto g = experimental::framework_graph::without_driver(); + g.add_source("src"); + + auto bundle = g.driver_proxy({"src"}).driver(fixed_hierarchy{}, + [](data_cell_cursor, test_source const& /*src*/) {}); + g.add_driver(std::move(bundle)); + + auto const expected_msg = + fmt::format("Driver source type mismatch at source index 0: expected '{}' but got '{}'.", + boost::core::demangle(typeid(test_source).name()), + boost::core::demangle(typeid(other_source).name())); + + CHECK_THROWS_WITH(g.execute(), expected_msg); +} + +TEST_CASE("Throw when configuring driver twice", "[graph]") +{ + auto g = experimental::framework_graph::without_driver(); + + auto gen = experimental::layer_generator::make(); + CHECK_NOTHROW(g.add_driver(gen)); + CHECK_THROWS_WITH( + g.add_driver(gen), + Catch::Matchers::ContainsSubstring("Driver has already been configured for framework_graph")); +} diff --git a/test/function_registration.cpp b/test/function_registration.cpp index 6d1e2f042..ddac33097 100644 --- a/test/function_registration.cpp +++ b/test/function_registration.cpp @@ -62,7 +62,7 @@ TEST_CASE("Call non-framework functions", "[programming model]") std::array const product_suffixes = {"onumber"s, "otemperature"s, "oname"s}; std::array const result{"result"s}; - experimental::framework_graph g; + auto g = experimental::framework_graph::with_default_driver(); // Register providers g.provide("provide_number", provide_number, concurrency::unlimited) diff --git a/test/hierarchical_nodes.cpp b/test/hierarchical_nodes.cpp index dd9d08518..eb3f94cad 100644 --- a/test/hierarchical_nodes.cpp +++ b/test/hierarchical_nodes.cpp @@ -78,11 +78,12 @@ namespace { TEST_CASE("Hierarchical nodes", "[graph]") { - experimental::layer_generator gen; - gen.add_layer("run", {"job", index_limit}); - gen.add_layer("event", {"run", number_limit}); + auto gen = experimental::layer_generator::make(); + gen->add_layer("run", {"job", index_limit}); + gen->add_layer("event", {"run", number_limit}); - experimental::framework_graph g{driver_for_test(gen)}; + auto g = experimental::framework_graph::without_driver(); + g.add_driver(gen); g.provide("provide_time", [](data_cell_index const& index) { diff --git a/test/layer_generator.cpp b/test/layer_generator.cpp index 3289dc7d4..5661b72cf 100644 --- a/test/layer_generator.cpp +++ b/test/layer_generator.cpp @@ -9,66 +9,70 @@ using namespace Catch::Matchers; TEST_CASE("Only job layer", "[layer-generation]") { - layer_generator gen; + auto gen = layer_generator::make(); - framework_graph g{driver_for_test(gen)}; + auto g = framework_graph::without_driver(); + g.add_driver(gen); g.execute(); - CHECK(gen.emitted_cell_count("/job") == 1); - CHECK(gen.emitted_cell_count() == 1); + CHECK(gen->emitted_cell_count("/job") == 1); + CHECK(gen->emitted_cell_count() == 1); } TEST_CASE("One non-job layer", "[layer-generation]") { - layer_generator gen; - gen.add_layer("spill", {"job", 16}); + auto gen = layer_generator::make(); + gen->add_layer("spill", {"job", 16}); - framework_graph g{driver_for_test(gen)}; + auto g = framework_graph::without_driver(); + g.add_driver(gen); g.execute(); - CHECK(gen.emitted_cell_count("/job") == 1); - CHECK(gen.emitted_cell_count("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/job/spill") == 16); - CHECK(gen.emitted_cell_count() == 1 + 16); + CHECK(gen->emitted_cell_count("/job") == 1); + CHECK(gen->emitted_cell_count("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/job/spill") == 16); + CHECK(gen->emitted_cell_count() == 1 + 16); } TEST_CASE("Two non-job layers", "[layer-generation]") { - layer_generator gen; - gen.add_layer("spill", {"job", 16}); - gen.add_layer("APA", {"spill", 16}); + auto gen = layer_generator::make(); + gen->add_layer("spill", {"job", 16}); + gen->add_layer("APA", {"spill", 16}); - framework_graph g{driver_for_test(gen)}; + auto g = framework_graph::without_driver(); + g.add_driver(gen); g.execute(); - CHECK(gen.emitted_cell_count("/job") == 1); - CHECK(gen.emitted_cell_count("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/job/spill") == 16); - CHECK(gen.emitted_cell_count("/job/spill/APA") == 256); - CHECK(gen.emitted_cell_count() == 1 + 16 + 256); + CHECK(gen->emitted_cell_count("/job") == 1); + CHECK(gen->emitted_cell_count("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/job/spill") == 16); + CHECK(gen->emitted_cell_count("/job/spill/APA") == 256); + CHECK(gen->emitted_cell_count() == 1 + 16 + 256); } TEST_CASE("Test rebasing layers", "[layer-generation]") { - layer_generator gen; - gen.add_layer("APA", {"spill", 16}); - gen.add_layer("spill", {"job", 16}); + auto gen = layer_generator::make(); + gen->add_layer("APA", {"spill", 16}); + gen->add_layer("spill", {"job", 16}); - framework_graph g{driver_for_test(gen)}; + auto g = framework_graph::without_driver(); + g.add_driver(gen); g.execute(); - CHECK(gen.emitted_cell_count("/job") == 1); - CHECK(gen.emitted_cell_count("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/job/spill") == 16); - CHECK(gen.emitted_cell_count("/job/spill/APA") == 256); - CHECK(gen.emitted_cell_count() == 1 + 16 + 256); + CHECK(gen->emitted_cell_count("/job") == 1); + CHECK(gen->emitted_cell_count("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/job/spill") == 16); + CHECK(gen->emitted_cell_count("/job/spill/APA") == 256); + CHECK(gen->emitted_cell_count() == 1 + 16 + 256); } TEST_CASE("Ambiguous layers", "[layer-generation]") { - layer_generator gen; - gen.add_layer("run", {"job", 16}); - gen.add_layer("spill", {"run", 16}); - gen.add_layer("spill", {"job", 16}); + auto gen = layer_generator::make(); + gen->add_layer("run", {"job", 16}); + gen->add_layer("spill", {"run", 16}); + gen->add_layer("spill", {"job", 16}); - CHECK_THROWS_WITH(gen.add_layer("APA", {"spill", 16}), + CHECK_THROWS_WITH(gen->add_layer("APA", {"spill", 16}), ContainsSubstring("Ambiguous: two parent layers found for data layer 'APA'") && ContainsSubstring("/job/run/spill") && ContainsSubstring("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/job/spill")); @@ -77,23 +81,24 @@ TEST_CASE("Ambiguous layers", "[layer-generation]") TEST_CASE("Avoid ambiguous layers", "[layer-generation]") { - layer_generator gen; - gen.add_layer("run", {"job", 16}); - gen.add_layer("spill", {"run", 16}); - gen.add_layer("spill", {"job", 16}); - gen.add_layer("APA", {"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/run/spill", 16}); // More complete parent path used to disambiguate - - framework_graph g{driver_for_test(gen)}; + auto gen = layer_generator::make(); + gen->add_layer("run", {"job", 16}); + gen->add_layer("spill", {"run", 16}); + gen->add_layer("spill", {"job", 16}); + gen->add_layer("APA", {"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/run/spill", 16}); // More complete parent path used to disambiguate + + auto g = framework_graph::without_driver(); + g.add_driver(gen); g.execute(); - CHECK(gen.emitted_cell_count("/job") == 1); - CHECK(gen.emitted_cell_count("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/job/run") == 16); - CHECK(gen.emitted_cell_count("/job/run/spill") == 256); - CHECK(gen.emitted_cell_count("/job/run/spill/APA") == 4096); - CHECK(gen.emitted_cell_count("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/job/spill") == 16); - CHECK(gen.emitted_cell_count() == 1 + 16 + 256 + 4096 + 16); + CHECK(gen->emitted_cell_count("/job") == 1); + CHECK(gen->emitted_cell_count("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/job/run") == 16); + CHECK(gen->emitted_cell_count("/job/run/spill") == 256); + CHECK(gen->emitted_cell_count("/job/run/spill/APA") == 4096); + CHECK(gen->emitted_cell_count("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/job/spill") == 16); + CHECK(gen->emitted_cell_count() == 1 + 16 + 256 + 4096 + 16); CHECK_THROWS_WITH( - gen.emitted_cell_count("/job/spill/APA"), + gen->emitted_cell_count("/job/spill/APA"), ContainsSubstring("No emitted cells corresponding to layer path '/job/spill/APA'")); } diff --git a/test/max-parallelism/provide_parallelism.cpp b/test/max-parallelism/provide_parallelism.cpp index 0d9cdd995..8802826bc 100644 --- a/test/max-parallelism/provide_parallelism.cpp +++ b/test/max-parallelism/provide_parallelism.cpp @@ -38,5 +38,5 @@ namespace { PHLEX_REGISTER_SOURCE(s, config) { - s.source(config.get("module_label")); + s.add_source(config.get("module_label")); } diff --git a/test/memory-checks/many_events.cpp b/test/memory-checks/many_events.cpp index af2fb186b..3eee069ad 100644 --- a/test/memory-checks/many_events.cpp +++ b/test/memory-checks/many_events.cpp @@ -14,10 +14,11 @@ try { constexpr auto max_events{100'000u}; - experimental::layer_generator gen; - gen.add_layer("event", {"job", max_events, 1u}); + auto gen = experimental::layer_generator::make(); + gen->add_layer("event", {"job", max_events, 1u}); - experimental::framework_graph g{driver_for_test(gen)}; + auto g = experimental::framework_graph::without_driver(); + g.add_driver(gen); g.provide("provide_number", [](data_cell_index const& id) -> unsigned { return id.number(); }) .output_product("input", "number", "event"); diff --git a/test/multiple_function_registration.cpp b/test/multiple_function_registration.cpp index 159b1c7c3..d48accd4f 100644 --- a/test/multiple_function_registration.cpp +++ b/test/multiple_function_registration.cpp @@ -41,7 +41,7 @@ namespace { TEST_CASE("Call multiple functions", "[programming model]") { - experimental::framework_graph g; + auto g = experimental::framework_graph::with_default_driver(); g.provide("provide_numbers", [](data_cell_index const&) -> std::vector { return {0, 1, 2, 3, 4}; }) diff --git a/test/output_products.cpp b/test/output_products.cpp index cebce723b..6984bb848 100644 --- a/test/output_products.cpp +++ b/test/output_products.cpp @@ -36,10 +36,11 @@ namespace { TEST_CASE("Output data products", "[graph]") { - experimental::layer_generator gen; - gen.add_layer("spill", {"job", 1u}); + auto gen = experimental::layer_generator::make(); + gen->add_layer("spill", {"job", 1u}); - experimental::framework_graph g{driver_for_test(gen)}; + auto g = experimental::framework_graph::without_driver(); + g.add_driver(gen); g.provide("provide_number", [](data_cell_index const&) -> int { return 17; }) .output_product("input", "number_from_provider", "spill"); diff --git a/test/product_selecting.cpp b/test/product_selecting.cpp index 060772416..e816e8d7a 100644 --- a/test/product_selecting.cpp +++ b/test/product_selecting.cpp @@ -28,9 +28,10 @@ namespace { TEST_CASE("Querying products in different ways", "[graph]") { constexpr int num_events = 25; - experimental::layer_generator gen; - gen.add_layer("event", {.parent_layer_name = "job", .total_per_parent_data_cell = num_events}); - experimental::framework_graph g{driver_for_test(gen)}; + auto gen = experimental::layer_generator::make(); + gen->add_layer("event", {.parent_layer_name = "job", .total_per_parent_data_cell = num_events}); + auto g = experimental::framework_graph::without_driver(); + g.add_driver(gen); // Register providers g.provide("provide_number_in_job", provide_number, concurrency::unlimited) diff --git a/test/provider_test.cpp b/test/provider_test.cpp index 62b95c672..34646ac32 100644 --- a/test/provider_test.cpp +++ b/test/provider_test.cpp @@ -74,10 +74,11 @@ TEST_CASE("Explicit providers") { constexpr auto num_spills{3u}; - experimental::layer_generator gen; - gen.add_layer("spill", {"job", num_spills, 1u}); + auto gen = experimental::layer_generator::make(); + gen->add_layer("spill", {"job", num_spills, 1u}); - experimental::framework_graph g{driver_for_test(gen)}; + auto g = experimental::framework_graph::without_driver(); + g.add_driver(gen); g.provide("my_name_here", give_me_vertices, concurrency::unlimited) .output_product("vertices_maker", "happy_vertices", "spill"); @@ -102,12 +103,12 @@ TEST_CASE("Implicit providers") { constexpr auto num_spills{3u}; - experimental::layer_generator gen; - gen.add_layer("spill", {"job", num_spills, 1u}); + auto gen = experimental::layer_generator::make(); + gen->add_layer("spill", {"job", num_spills, 1u}); - experimental::framework_graph g{driver_for_test(gen)}; - - g.source("vertices_source"); + auto g = experimental::framework_graph::without_driver(); + g.add_driver(gen); + g.add_source("vertices_source"); g.transform("passer", pass_on, concurrency::unlimited) .input_family( @@ -128,21 +129,37 @@ TEST_CASE("Implicit providers") TEST_CASE("Throw when two sources with the same name are registered") { - experimental::framework_graph g; - g.source("vertices_source"); - g.source("vertices_source"); + auto g = experimental::framework_graph::with_default_driver(); + g.add_source("vertices_source"); + g.add_source("vertices_source"); CHECK_THROWS_WITH(g.execute(), ContainsSubstring("Source with name 'vertices_source' already exists")); } +TEST_CASE("Throw when no provider found for required product") +{ + auto g = experimental::framework_graph::with_default_driver(); + + // Register an observer that needs a product from a creator that does not exist in the graph. + // Since there is no matching provider, make_computational_edges should throw listing all + // unmatched products. + g.observe( + "observer", [](unsigned int const) {}, concurrency::unlimited) + .input_family(product_selector{.creator = "nonexistent_creator", .layer = "job"}); + + CHECK_THROWS_WITH(g.execute(), + ContainsSubstring("No provider found for the following required products:") && + ContainsSubstring("nonexistent_creator") && ContainsSubstring("job")); +} + TEST_CASE("Throw when two implicit providers are found for the same product") { - experimental::framework_graph g; + auto g = experimental::framework_graph::with_default_driver(); // Register two sources that can provide the same product - g.source("vertices_source_1"); - g.source("vertices_source_2"); + g.add_source("vertices_source_1"); + g.add_source("vertices_source_2"); g.transform("passer", pass_on, concurrency::unlimited) .input_family( @@ -155,26 +172,14 @@ TEST_CASE("Throw when two implicit providers are found for the same product") ContainsSubstring("spill") && ContainsSubstring("passer")); } -TEST_CASE("Throw when no provider found for required product") -{ - experimental::framework_graph g; - - // Register an observer that needs a product from a creator that does not exist in the graph. - // Since there is no matching provider, make_computational_edges should throw listing all - // unmatched products. - g.observe( - "observer", [](unsigned int const) {}, concurrency::unlimited) - .input_family(product_selector{.creator = "nonexistent_creator", .layer = "job"}); - - CHECK_THROWS_WITH(g.execute(), - ContainsSubstring("No provider found for the following required products:") && - ContainsSubstring("nonexistent_creator") && ContainsSubstring("job")); -} - TEST_CASE("Throw when implicit provider insertion fails") { - experimental::framework_graph g; - g.source("duplicate_vertices_source"); + auto gen = experimental::layer_generator::make(); + gen->add_layer("spill", {"job", 1u}); + + auto g = experimental::framework_graph::without_driver(); + g.add_driver(std::move(gen)); + g.add_source("duplicate_vertices_source"); g.transform("passer", pass_on, concurrency::unlimited) .input_family( diff --git a/test/type_distinction.cpp b/test/type_distinction.cpp index 8d2035b22..449e5e23e 100644 --- a/test/type_distinction.cpp +++ b/test/type_distinction.cpp @@ -43,10 +43,11 @@ namespace { TEST_CASE("Distinguish products with same name and different types", "[programming model]") { - experimental::layer_generator gen; - gen.add_layer("event", {"job", 10, 1}); + auto gen = experimental::layer_generator::make(); + gen->add_layer("event", {"job", 10, 1}); - experimental::framework_graph g{driver_for_test(gen)}; + auto g = experimental::framework_graph::without_driver(); + g.add_driver(gen); // Register providers g.provide("provide_numbers", provide_numbers, concurrency::unlimited) diff --git a/test/unfold.cpp b/test/unfold.cpp index a7a03d0fc..d7be4974e 100644 --- a/test/unfold.cpp +++ b/test/unfold.cpp @@ -91,10 +91,11 @@ TEST_CASE("Splitting the processing", "[graph]") { constexpr auto index_limit = 2u; - experimental::layer_generator gen; - gen.add_layer("event", {"job", index_limit}); + auto gen = experimental::layer_generator::make(); + gen->add_layer("event", {"job", index_limit}); - experimental::framework_graph g{driver_for_test(gen)}; + auto g = experimental::framework_graph::without_driver(); + g.add_driver(gen); g.provide("provide_max_number", provide_max_number, concurrency::unlimited) .output_product("input", "max_number", "event"); @@ -158,10 +159,11 @@ TEST_CASE("Multi-layer transform with one input from an unfold", "[graph]") { constexpr auto index_limit = 2u; - experimental::layer_generator gen; - gen.add_layer("event", {"job", index_limit}); + auto gen = experimental::layer_generator::make(); + gen->add_layer("event", {"job", index_limit}); - experimental::framework_graph g{driver_for_test(gen)}; + auto g = experimental::framework_graph::without_driver(); + g.add_driver(gen); g.provide("provide_max_number", provide_max_number, concurrency::unlimited) .output_product("input", "max_number", "event"); diff --git a/test/vector_of_abstract_types.cpp b/test/vector_of_abstract_types.cpp index cdd2dd817..f2d047f2f 100644 --- a/test/vector_of_abstract_types.cpp +++ b/test/vector_of_abstract_types.cpp @@ -39,10 +39,11 @@ namespace { TEST_CASE("Test vector of abstract types") { - experimental::layer_generator gen; - gen.add_layer("event", {"job", 1u, 1u}); + auto gen = experimental::layer_generator::make(); + gen->add_layer("event", {"job", 1u, 1u}); - experimental::framework_graph g{driver_for_test(gen)}; + auto g = experimental::framework_graph::without_driver(); + g.add_driver(gen); g.provide("provide_thing", [](data_cell_index const&) { return make_derived_as_abstract(); }) .output_product("dummy", "thing", "event"); g.transform("read_thing", read_abstract)