SimpleMPIScheduler.h

// Copyright 2024, UChicago Argonne, LLC
// All Rights Reserved
// Software Name: NEML2 -- the New Engineering material Model Library, version 2
// By: Argonne National Laboratory
// OPEN SOURCE LICENSE (MIT)
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
 
#pragma once
 
#include <mpi.h>
 
#include "neml2/dispatchers/WorkScheduler.h"
#include "neml2/base/Registry.h"
#include "neml2/base/NEML2Object.h"
#include "neml2/base/Factory.h"
 
namespace neml2
{
class SimpleMPIScheduler : public WorkScheduler
{
public:
  static OptionSet expected_options();

  SimpleMPIScheduler(const OptionSet & options);

  void setup() override;
 
  std::vector<Device> devices() const override { return {_available_devices[_device_index]}; }
 
  virtual MPI_Comm & comm() { return _comm; }
 
  virtual void set_comm(MPI_Comm comm) { _comm = comm; }
 
protected:
  bool schedule_work_impl(Device &, std::size_t &) const override;
  void dispatched_work_impl(Device, std::size_t) override;
  void completed_work_impl(Device, std::size_t) override;
  bool all_work_completed() const override;
 
private:
  void determine_my_device();

  std::vector<Device> _available_devices;

  std::vector<std::size_t> _batch_sizes;

  std::vector<std::size_t> _capacities;

  MPI_Comm _comm;

  std::size_t _device_index = 0;

  std::size_t _load = 0;
};
 
#define neml2_call_mpi(call)                                                                       \
  do                                                                                               \
  {                                                                                                \
    int err = (call);                                                                              \
    if (err != MPI_SUCCESS)                                                                        \
    {                                                                                              \
      char err_string[MPI_MAX_ERROR_STRING];                                                       \
      int len = 0;                                                                                 \
      MPI_Error_string(err, err_string, &len);                                                     \
      throw NEMLException(std::string("MPI error: ") + err_string);                                \
    }                                                                                              \
  } while (0)
 
} // namespace neml2