[WIP] Enabling multi-passage duplication for turbomachinery cases.

Common/include/CConfig.hpp

@@ -996,6 +996,8 @@ class CConfig {
   bool DeadLoad;                 /*!< \brief Application of dead loads to the FE analysis */
   bool PseudoStatic;             /*!< \brief Application of dead loads to the FE analysis */
   bool SteadyRestart;            /*!< \brief Restart from a steady state for FSI problems. */
+  bool Turbo_MultiPsgs;        /*!< \brief Restart multi passage simulation from a single passage solution for turbomachinery application. */
+  unsigned short nPassages;    /*!< \brief Number of user defined passages. */
   su2double Newmark_beta,        /*!< \brief Parameter alpha for Newmark method. */
   Newmark_gamma;                 /*!< \brief Parameter delta for Newmark method. */
   unsigned short nIntCoeffs;     /*!< \brief Number of integration coeffs for structural calculations. */
@@ -5219,6 +5221,18 @@ class CConfig {
    */
   void SetnBlades(unsigned short val_iZone, su2double nblades) { nBlades[val_iZone] = nblades;}
 
+  /*!
+   * \brief Get the number of user defined passages to be modelled in computation.
+   * \param[in] nPassages - user defined number of passages
+   */
+  unsigned short GetnPassages(void) const { return nPassages; }
+
+  /*!
+   * \brief Identifies if we want to restart multi passage simulation from a single passage solution for turbomachinery case.
+   * \return <code>TRUE</code> if we restart from multi passage simulation, <code>FALSE</code> otherwise.
+   */
+  bool GetTurbo_MultiPsgs(void) const { return Turbo_MultiPsgs; }
+
   /*!
    * \brief Verify if there is any Giles Boundary Condition option specified from config file.
    * \return boolean.

Common/include/geometry/CGeometry.hpp

@@ -317,6 +317,54 @@
   SU2_MPI::Request* req_PeriodicSend{nullptr}; /*!< \brief Data structure for periodic send requests. */
   SU2_MPI::Request* req_PeriodicRecv{nullptr}; /*!< \brief Data structure for periodic recv requests. */
 
+  /*--- Data structures for mesh Full annulus duplication. ---*/
+
+  int nPassages{1};       /*!< \brief the user defined number of passages. Used only for mesh duplication.*/
+  int nPsgs_FullAnnu{0};  /*!< \brief the number of passages for full annulus. Used only for mesh duplication.*/
+  bool Fullannus = false; /*!< \brief whether it's full annulus. Used only for mesh duplication.*/
+  unsigned long Global_nPoint_OldPsg{
+      0};               /*!< \brief the number of points in the original one passage. Used only for mesh duplication.*/
+  su2double PitchAngle; /*!< \brief the pitch angle for one passage. Used only for mesh duplication.*/
+  int* nPerPointOnRank_donor = new int[size];  /*!< \brief vector contains the number of periodic points on each node.
+                                                  Used only for mesh duplication. */
+  int* nPerPointOnRank_target = new int[size]; /*!< \brief vector contains the number of periodic points on each node.
+                                                  Used only for mesh duplication. */
+
+  unsigned long nPerPointAll_donor = 0;
+  unsigned long nPerPointAll_target = 0;
+  unsigned long nPsgPoint_RemPer =
+      0; /*!< \brief the number of points in one passage after removing donor points. Used only for mesh duplication.*/
+  unsigned long* PerPointsMatched_donor{nullptr}; /*!< \brief Global index of the matched points on the target of one
+                                                     pair periodic BC. Used only for mesh duplication.*/
+  unsigned long* PerPointsGlbIndx_donor{nullptr}; /*!< \brief Global index of the points on the donor of one pair
+                                                     periodic BC. Used only for mesh duplication.*/
+  su2double* PerPointsCoord_donor{nullptr}; /*!< \brief Coordinates of the points on the donor of one pair periodic BC.
+                                               Used only for mesh duplication.*/
+  unsigned long* PsgPointsOldGlbIndx_RemPer{nullptr}; /*!< \brief new Global index of the points in one passage after
+                                                         removing donor points. Used only for mesh duplication.*/
+  su2double* PsgPointsCoord_RemPer{nullptr}; /*!< \brief coordinates of the points in one passage after removing donor
+                                                points. Used only for mesh duplication.*/
+  unordered_map<unsigned long, unsigned long>
+      PerPoint_Match_Map; /*!< \brief Global index map for matched points on periodic boundaries. Used only for mesh
+                             duplication.*/
+  unordered_map<unsigned long, unsigned long>
+      Donor_Indx_Map; /*!< \brief Global to donor periodic boudnry index map. Used only for mesh duplication.*/
+  unordered_map<unsigned long, unsigned long> Psg_Old2New_Map; /*!< \brief Old to new global index map for points in one
+                                                                  passage. Used only for mesh duplication.*/
+  unordered_map<unsigned long, long> Psg_Old2New_Map2;         /*!< \brief Old to new global index map for points in one
+                                                                  passage. Used only for mesh duplication.*/
+
+  // to be removed
+  unsigned long* nElem_Bound_FullAnnu{nullptr};
+  unsigned long Global_nPoint_Fullannulus = 0; /*!< \brief the new global number of points for full annulus*/
+  unsigned long nPoint_FullAnnu = 0;           /*!< \brief the new local number of points for full annulus*/
+  unsigned long* LocalPointsGlbIndx_FullAnnu{nullptr};
+  su2double* LocalPointsCoord_FullAnnu{nullptr};
+  unsigned long numberOfLocalElements_FullAnnu =
+      0; /*!< \brief Number of local elements on this rank for full annulus. */
+  vector<unsigned long> localVolumeElementConnectivity_FullAnnu; /*!< \brief Vector containing the element connectivity
+                                                                    from the mesh file for the local elements. */
+
   /*--- Mesh quality metrics. ---*/
 
   vector<su2double> Orthogonality; /*!< \brief Measure of dual CV orthogonality angle (0 to 90 deg., 90 being best). */
@@ -1871,4 +1919,86 @@
    * \return A pointer to the reference node coordinate vector.
    */
   inline virtual const su2double* GetStreamwise_Periodic_RefNode() const { return nullptr; }
+
+  /*!
+   * \brief Set the number of passages.
+   * \return The number of passages.
+   */
+  // inline void SetGlobal_nPoint_OldPsg(unsigned long val_nPoint) { Global_nPoint_OldPsg = val_nPoint; }
+
+  /*!
+   * \brief Retrieve the bool Fullannus.
+   * \return bool Fullannus.
+   */
+  inline bool GetFullannus() const { return Fullannus; }
+
+  /*!
+   * \brief Retrieve the old number of points before full annulus transformation.
+   * \return The old number of points before full annulus transformation.
+   */
+  inline unsigned long GetGlobal_nPoint_OldPsg() const { return Global_nPoint_OldPsg; }
+
+  /*!
+   * \brief Retrieve the new number of points in one passage after removing one periodic boundary.
+   * \return The new number of points in one passage after removing one periodic boundary.
+   */
+  inline unsigned long GetGlobal_nPsgPoint_RemPer() const { return nPsgPoint_RemPer; }
+
+  /*!
+   * \brief Retrieve the number of passages.
+   * \return The number of passages.
+   */
+  inline int GetnPassages() const { return nPassages; }
+
+  /*!
+   * \brief Retrieve the number of passages for full annulus.
+   * \return The number of passages.
+   */
+  inline int GetnPassages_FullAnnu() const { return nPsgs_FullAnnu; }
+
+  /*!
+   * \brief Retrieve the pitch angle for duplicating passages.
+   * \return The pitch angle for duplicating passages.
+   */
+  inline su2double GetPitchangle() const { return PitchAngle; }
+
+  /*!
+   * \brief Get the matched point's global index for periodic BC.
+   * \param[in] val_ipoint - Global point.
+   * \return Local index that correspond with the global index, -1 if not found on the current rank (process).
+   */
+  inline void Set_PsgOld2New_Indx_2(unsigned long val_old_indx, unsigned long val_new_indx) {
+    Psg_Old2New_Map2[val_old_indx] = val_new_indx;
+  }
+
+  /*!
+   * \brief Get the matched point's global index for periodic BC.
+   * \param[in] val_ipoint - Global point.
+   * \return Local index that correspond with the global index, -1 if not found on the current rank (process).
+   */
+  inline void Set_Donor_Indx(unsigned long val_donor_indx, unsigned long val_new_indx) {
+    Donor_Indx_Map[val_donor_indx] = val_new_indx;
+  }
+
+  /*!
+   * \brief Get the matched point's global index for periodic BC.
+   * \param[in] val_ipoint - Global point.
+   * \return Local index that correspond with the global index, -1 if not found on the current rank (process).
+   */
+  inline unsigned long Get_PsgOld2New_Indx_2(unsigned long val_old_indx) {
+    unsigned long val_new_indx = Psg_Old2New_Map2[val_old_indx];
+    // return Psg_Old2New_Map[val_old_indx];
+    return val_new_indx;
+  }
+
+  /*!
+   * \brief Get the matched point's global index for periodic BC.
+   * \param[in] val_ipoint - Global point.
+   * \return Local index that correspond with the global index, -1 if not found on the current rank (process).
+   */
+  inline unsigned long Get_Donor_Indx(unsigned long val_donor_indx) {
+    unsigned long val_new_indx = Donor_Indx_Map[val_donor_indx];
+    // return Psg_Old2New_Map[val_old_indx];
+    return val_new_indx;
+  }
 };

Common/include/geometry/CPhysicalGeometry.hpp

@@ -328,6 +328,41 @@ class CPhysicalGeometry final : public CGeometry {
    */
   void LoadUnpartitionedSurfaceElements(CConfig* config, CMeshReaderFVM* mesh);
 
+  /*!
+   * \brief Match the periodic boundary, useful for full annulus mesh duplication.
+   * \param[in] config - definition of the particular problem.
+   * \param[in] val_periodic  - index of the periodic pair that is going to be treated.
+   */
+  void MatchPeriodicPoints(CConfig* config, unsigned short val_periodic);
+
+  /*!
+   * \brief Reset the rotating angle for periodic boudary conditions, useful for full annulus mesh duplication.
+   * \param[in] config - definition of the particular problem.
+   * \param[in] val_periodic  - index of the periodic pair that is going to be treated.
+   */
+  void ReSetPeriodicAngle_Z(CConfig* config);
+
+  /*!
+   * \brief duplicate the points, and redistribute points on each node.
+   * \param[in] config - definition of the particular problem.
+   * \param[in] mesh   - mesh reader object containing the current zone data.
+   */
+  void DuplicatePoints(CConfig* config, CMeshReaderFVM* mesh);
+
+  /*!
+   * \brief duplicate the volume elems, and redistribute volume elems on each node.
+   * \param[in] config - definition of the particular problem.
+   * \param[in] mesh   - mesh reader object containing the current zone data.
+   */
+  void DuplicateVolumeElems(CConfig* config, CMeshReaderFVM* mesh);
+
+  /*!
+   * \brief duplicate the surface elems, and redistribute surface elems on each node.
+   * \param[in] config - definition of the particular problem.
+   * \param[in] mesh   - mesh reader object containing the current zone data.
+   */
+  void DuplicateSurfaceElems(CConfig* config, CMeshReaderFVM* mesh);
+
   /*!
    * \brief Prepares the grid point adjacency based on a linearly partitioned mesh object needed by ParMETIS for graph
    * partitioning in parallel. \param[in] config - Definition of the particular problem.

Common/include/parallelization/mpi_structure.hpp

@@ -218,6 +218,16 @@ class CBaseMPIWrapper {
     MPI_Scatter(sendbuf, sendcnt, sendtype, recvbuf, recvcnt, recvtype, root, comm);
   }
 
+  static inline void Scatterv(const void* sendbuf, const int* sendcnts, const int* displs, Datatype sendtype,
+                              void* recvbuf, int recvcnt, Datatype recvtype, int root, Comm comm) {
+    MPI_Scatterv(sendbuf, sendcnts, displs, sendtype, recvbuf, recvcnt, recvtype, root, comm);
+  }
+
+  static inline void Gatherv(const void* sendbuf, int sendcnt, Datatype sendtype, void* recvbuf, const int* recvcounts,
+                             const int* displs, Datatype recvtype, int root, Comm comm) {
+    MPI_Gatherv(sendbuf, sendcnt, sendtype, recvbuf, recvcounts, displs, recvtype, root, comm);
+  }
+
   static inline void Allgather(const void* sendbuf, int sendcnt, Datatype sendtype, void* recvbuf, int recvcnt,
                                Datatype recvtype, Comm comm) {
     MPI_Allgather(sendbuf, sendcnt, sendtype, recvbuf, recvcnt, recvtype, comm);
@@ -425,6 +435,18 @@ class CMediMPIWrapper : public CBaseMPIWrapper {
                  convertComm(comm));
   }
 
+  static inline void Scatterv(const void* sendbuf, const int* sendcnts, const int* displs, Datatype sendtype,
+                              void* recvbuf, int recvcnt, Datatype recvtype, int root, Comm comm) {
+    AMPI_Scatterv(sendbuf, sendcnts, displs, convertDatatype(sendtype), recvbuf, recvcnt, convertDatatype(recvtype),
+                  root, comm);
+  }
+
+  static inline void Gatherv(const void* sendbuf, int sendcnt, Datatype sendtype, void* recvbuf, const int* recvcounts,
+                             const int* displs, Datatype recvtype, int root, Comm comm) {
+    MPI_Gatherv(sendbuf, sendcnt, convertDatatype(sendtype), recvbuf, recvcounts, displs, convertDatatype(recvtype),
+                root, comm);
+  }
+
   static inline void Allgather(const void* sendbuf, int sendcnt, Datatype sendtype, void* recvbuf, int recvcnt,
                                Datatype recvtype, Comm comm) {
     AMPI_Allgather(sendbuf, sendcnt, convertDatatype(sendtype), recvbuf, recvcnt, convertDatatype(recvtype),
@@ -572,6 +594,16 @@ class CBaseMPIWrapper {
     CopyData(sendbuf, recvbuf, sendcnt, sendtype);
   }
 
+  static inline void Scatterv(const void* sendbuf, const int* sendcnts, const int* displs, Datatype sendtype,
+                              void* recvbuf, int recvcnt, Datatype recvtype, int root, Comm comm) {
+    CopyData(sendbuf, recvbuf, sendcnts[0], sendtype);
+  }
+
+  static inline void Gatherv(const void* sendbuf, int sendcnt, Datatype sendtype, void* recvbuf, const int* recvcounts,
+                             const int* displs, Datatype recvtype, int root, Comm comm) {
+    CopyData(sendbuf, recvbuf, sendcnt, sendtype);
+  }
+
   static inline void Allgatherv(const void* sendbuf, int sendcnt, Datatype sendtype, void* recvbuf, const int* recvcnt,
                                 const int* displs, Datatype recvtype, Comm comm) {
     CopyData(sendbuf, recvbuf, sendcnt, sendtype, displs[0]);

Common/src/CConfig.cpp

@@ -1631,6 +1631,10 @@ void CConfig::SetConfig_Options() {
   addTurboPerfOption("MARKER_TURBOMACHINERY", nMarker_Turbomachinery, Marker_TurboBoundIn, Marker_TurboBoundOut);
   /*!\brief NUM_SPANWISE_SECTIONS \n DESCRIPTION: Integer number of spanwise sections to compute 3D turbo BC and Performance for turbomachinery */
   addUnsignedShortOption("NUM_SPANWISE_SECTIONS", nSpanWiseSections_User, 1);
+  /*!\brief PassageNumber\n DESCRIPTION: User defined number of passages. DEFAULT: 1 \ingroup Config*/
+  addUnsignedShortOption("PASSAGE_NUMBER", nPassages, 1);
+  /*!\brief TURBOMACHINERY_MULTIPASSAGE_RESTART \n DESCRIPTION: Restart multi passage simulation from single passage results \n Options: NO, YES \ingroup Config */
+  addBoolOption("TURBOMACHINERY_MULTIPASSAGE_RESTART", Turbo_MultiPsgs, false);
   /*!\brief SPANWISE_KIND \n DESCRIPTION: type of algorithm to identify the span-wise sections at the turbo boundaries.
    \n OPTIONS: see \link SpanWise_Map \endlink \n Default: AUTOMATIC */
   addEnumOption("SPANWISE_KIND", Kind_SpanWise, SpanWise_Map, AUTOMATIC);