SST  6.1.0
StructuralSimulationToolkit
objectComms.h
1 // Copyright 2009-2016 Sandia Corporation. Under the terms
2 // of Contract DE-AC04-94AL85000 with Sandia Corporation, the U.S.
3 // Government retains certain rights in this software.
4 //
5 // Copyright (c) 2009-2016, Sandia Corporation
6 // All rights reserved.
7 //
8 // This file is part of the SST software package. For license
9 // information, see the LICENSE file in the top level directory of the
10 // distribution.
11 
12 
13 #ifndef SST_CORE_OBJECTCOMMS_H
14 #define SST_CORE_OBJECTCOMMS_H
15 
16 #ifdef SST_CONFIG_HAVE_MPI
17 #include <mpi.h>
18 #endif
19 
20 #include <sst/core/serialization/serializer.h>
21 
22 #include <typeinfo>
23 
24 namespace SST {
25 
26 namespace Comms {
27 
28 
29 template <typename dataType>
30 std::vector<char> serialize(dataType &data)
31 {
33 
34  ser.start_sizing();
35  ser & data;
36 
37  int size = ser.size();
38 
39  std::vector<char> buffer;
40  buffer.resize(size);
41 
42  ser.start_packing(buffer.data(),size);
43  ser & data;
44 
45  return buffer;
46 }
47 
48 // template <typename dataType>
49 // std::vector<char> serialize(dataType* data)
50 // {
51 // SST::Core::Serialization::serializer ser;
52 
53 // std::cout << typeid(data).name() << std::endl;
54 // ser.start_sizing();
55 // ser & data;
56 
57 // ser.start_sizing();
58 // ser & data;
59 
60 // int size = ser.size();
61 // std::cout << "serialize size = " << size << std::endl;
62 
63 // std::vector<char> buffer;
64 // buffer.resize(size);
65 
66 // ser.start_packing(buffer.data(),size);
67 // ser & data;
68 
69 // return buffer;
70 // }
71 
72 
73 template <typename dataType>
74 dataType* deserialize(std::vector<char> &buffer)
75 {
76  dataType *tgt;
77 
79 
80  ser.start_unpacking(buffer.data(), buffer.size());
81  ser & tgt;
82 
83  return tgt;
84 }
85 
86 template <typename dataType>
87 void deserialize(std::vector<char> &buffer, dataType &tgt)
88 {
90 
91  ser.start_unpacking(buffer.data(), buffer.size());
92  ser & tgt;
93 }
94 
95 template <typename dataType>
96 void deserialize(char *buffer, int blen, dataType &tgt)
97 {
99 
100  ser.start_unpacking(buffer, blen);
101  ser & tgt;
102 }
103 
104 #ifdef SST_CONFIG_HAVE_MPI
105 template <typename dataType>
106 void broadcast(dataType& data, int root) {
107  int rank = 0;
108  MPI_Comm_rank(MPI_COMM_WORLD, &rank);
109  if ( root == rank ) {
110  // Serialize the data
111  std::vector<char> buffer = Comms::serialize(data);
112 
113  // Now broadcast the size of the data
114  int size = buffer.size();
115  MPI_Bcast(&size, 1, MPI_INT, root, MPI_COMM_WORLD);
116 
117  // Now broadcast the data
118  MPI_Bcast(buffer.data(), buffer.size(), MPI_BYTE, root, MPI_COMM_WORLD);
119  }
120  else {
121  // Get the size of the broadcast
122  int size = 0;
123  MPI_Bcast(&size, 1, MPI_INT, root, MPI_COMM_WORLD);
124 
125  // Now get the data
126  char* buffer = new char[size];
127  MPI_Bcast(buffer, size, MPI_BYTE, root, MPI_COMM_WORLD);
128 
129  // Now deserialize data
130  Comms::deserialize(buffer, size, data);
131  }
132 }
133 
134 template <typename dataType>
135 void send(int dest, int tag, dataType& data) {
136  // Serialize the data
137  std::vector<char> buffer = Comms::serialize<dataType>(data);
138 
139  // Now send the data. Send size first, then payload
140  // std::cout<< sizeof(buffer.size()) << std::endl;
141  int64_t size = buffer.size();
142  MPI_Send(&size, 1, MPI_INT64_T, dest, tag, MPI_COMM_WORLD);
143 
144  int32_t fragment_size = 1000000000;
145  int64_t offset = 0;
146 
147  while ( size >= fragment_size ) {
148  MPI_Send(buffer.data() + offset, fragment_size, MPI_BYTE, dest, tag, MPI_COMM_WORLD);
149  size -= fragment_size;
150  offset += fragment_size;
151  }
152  MPI_Send(buffer.data() + offset, size, MPI_BYTE, dest, tag, MPI_COMM_WORLD);
153 }
154 
155 template <typename dataType>
156 void recv(int src, int tag, dataType& data) {
157  // Get the size of the broadcast
158  int64_t size = 0;
159  MPI_Status status;
160  MPI_Recv(&size, 1, MPI_INT64_T, src, tag, MPI_COMM_WORLD, &status);
161 
162  // Now get the data
163  char* buffer = new char[size];
164  int64_t offset = 0;
165  int32_t fragment_size = 1000000000;
166  int64_t rem_size = size;
167 
168  while ( rem_size >= fragment_size ) {
169  MPI_Recv(buffer + offset, fragment_size, MPI_BYTE, src, tag, MPI_COMM_WORLD, &status);
170  rem_size -= fragment_size;
171  offset += fragment_size;
172  }
173  MPI_Recv(buffer + offset, rem_size, MPI_BYTE, src, tag, MPI_COMM_WORLD, &status);
174 
175 
176  // Now deserialize data
177  Comms::deserialize(buffer, size, data);
178 }
179 
180 
181 template <typename dataType>
182 void all_gather(dataType& data, std::vector<dataType> &out_data) {
183  int rank = 0, world = 0;
184  MPI_Comm_rank(MPI_COMM_WORLD, &rank);
185  MPI_Comm_size(MPI_COMM_WORLD, &world);
186 
187  // Serialize the data
188  std::vector<char> buffer = Comms::serialize(data);
189 
190 
191  size_t sendSize = buffer.size();
192  int allSizes[world];
193  int displ[world];
194 
195  memset(allSizes, '\0', world * sizeof(int));
196  memset(displ, '\0', world * sizeof(int));
197 
198  MPI_Allgather(&sendSize, sizeof(int), MPI_BYTE,
199  &allSizes, sizeof(int), MPI_BYTE, MPI_COMM_WORLD);
200 
201  int totalBuf = 0;
202  for ( int i = 0 ; i < world ; i++ ) {
203  totalBuf += allSizes[i];
204  if ( i > 0 )
205  displ[i] = displ[i-1] + allSizes[i-1];
206  }
207 
208  char *bigBuff = new char[totalBuf];
209 
210  MPI_Allgatherv(buffer.data(), buffer.size(), MPI_BYTE,
211  bigBuff, allSizes, displ, MPI_BYTE, MPI_COMM_WORLD);
212 
213  out_data.resize(world);
214  for ( int i = 0 ; i < world ; i++ ) {
215  Comms::deserialize(&bigBuff[displ[i]], allSizes[i], out_data[i]);
216  }
217 
218  delete [] bigBuff;
219 
220 }
221 
222 
223 
224 
225 #endif
226 
227 }
228 
229 } //namespace SST
230 
231 #endif // SST_CORE_OBJECTCOMMS_H
This class is basically a wrapper for objects to declare the order in which their members should be s...
Definition: serializer.h:35
Definition: action.cc:17