SST  11.1.0
StructuralSimulationToolkit
tunneldef.h
1 // Copyright 2009-2021 NTESS. Under the terms
2 // of Contract DE-NA0003525 with NTESS, the U.S.
3 // Government retains certain rights in this software.
4 //
5 // Copyright (c) 2009-2021, NTESS
6 // All rights reserved.
7 //
8 // This file is part of the SST software package. For license
9 // information, see the LICENSE file in the top level directory of the
10 // distribution.
11 
12 #ifndef SST_CORE_INTERPROCESS_TUNNEL_DEF_H
13 #define SST_CORE_INTERPROCESS_TUNNEL_DEF_H
14 
15 /*
16  * This may be compiled into both SST and an Intel Pin3 tool
17  * Therefore it has the following restrictions:
18  * - Nothing that uses runtime type info (e.g., dynamic cast)
19  * - No c++11 (auto, nullptr, etc.)
20  * - Includes must all be PinCRT-enabled but not require PinCRT
21  */
22 
23 #include "sst/core/interprocess/circularBuffer.h"
24 
25 #include <inttypes.h>
26 #include <unistd.h>
27 #include <vector>
28 
29 namespace SST {
30 namespace Core {
31 namespace Interprocess {
32 
33 extern uint32_t globalMMAPIPCCount;
34 
35 /* Internal bookkeeping */
37 {
38  volatile uint32_t expectedChildren;
39  size_t shmSegSize;
40  size_t numBuffers;
41  size_t offsets[0]; // offset[0] points to user region, offset[1]... points to circular buffers
42 };
43 
44 /**
45  * This class defines a shared-memory region between a master process and
46  * one or more child processes
47  * Region has three data structures:
48  * - internal bookkeeping (InternalSharedData),
49  * - user defined shared data (ShareDataType)
50  * - multiple circular-buffer queues with entries of type MsgType
51  *
52  * @tparam ShareDataType Type to put in the shared data region
53  * @tparam MsgType Type of messages being sent in the circular buffers
54  */
55 template <typename ShareDataType, typename MsgType>
56 class TunnelDef
57 {
58 
60 
61 public:
62  /** Create a new tunnel
63  *
64  * @param numBuffers Number of buffers for which we should tunnel
65  * @param bufferSize How large each buffer should be
66  * @param expectedChildren Number of child processes that will connect to this tunnel
67  */
68  TunnelDef(size_t numBuffers, size_t bufferSize, uint32_t expectedChildren) : master(true), shmPtr(NULL)
69  {
70  // Locally buffer info
71  numBuffs = numBuffers;
72  buffSize = bufferSize;
73  children = expectedChildren;
74  shmSize = calculateShmemSize(numBuffers, bufferSize);
75  }
76 
77  /** Access an existing tunnel
78  * Child creates the TunnelDef, reads the shmSize, and then resizes its map accordingly
79  * @param sPtr Location of shared memory region
80  */
81  TunnelDef(void* sPtr) : master(false), shmPtr(sPtr)
82  {
83  isd = (InternalSharedData*)shmPtr;
84  shmSize = isd->shmSegSize;
85  }
86 
87  /** Finish setting up a tunnel once the manager knows the correct size of the tunnel
88  * and has mmap'd a large enough region for it
89  * @param sPtr Location of shared memory region
90  * returns number of children left to attach
91  */
92  uint32_t initialize(void* sPtr)
93  {
94  shmPtr = sPtr;
95 
96  if ( master ) {
97  nextAllocPtr = (uint8_t*)shmPtr;
98 
99  // Reserve space for InternalSharedData
100  // Including an offset array entry for each buffer & sharedData structure
101  std::pair<size_t, InternalSharedData*> aResult =
102  reserveSpace<InternalSharedData>((1 + numBuffs) * sizeof(size_t));
103  isd = aResult.second;
104  isd->expectedChildren = children;
105  isd->shmSegSize = shmSize;
106  isd->numBuffers = numBuffs;
107 
108  // Reserve space for ShareDataType structure
109  std::pair<size_t, ShareDataType*> bResult = reserveSpace<ShareDataType>(0);
110  isd->offsets[0] = bResult.first;
111  sharedData = bResult.second;
112 
113  // Reserve space for circular buffers
114  const size_t cbSize = sizeof(MsgType) * buffSize;
115  for ( size_t c = 0; c < isd->numBuffers; c++ ) {
116  CircBuff_t* cPtr = NULL;
117 
118  std::pair<size_t, CircBuff_t*> cResult = reserveSpace<CircBuff_t>(cbSize);
119  isd->offsets[1 + c] = cResult.first;
120  cPtr = cResult.second;
121  if ( !cPtr->setBufferSize(buffSize) ) exit(1); // function prints error message
122  circBuffs.push_back(cPtr);
123  }
124  return isd->expectedChildren;
125  }
126  else {
127  isd = (InternalSharedData*)shmPtr;
128  shmSize = isd->shmSegSize;
129 
130  sharedData = (ShareDataType*)((uint8_t*)shmPtr + isd->offsets[0]);
131 
132  for ( size_t c = 0; c < isd->numBuffers; c++ ) {
133  circBuffs.push_back((CircBuff_t*)((uint8_t*)shmPtr + isd->offsets[c + 1]));
134  }
135  numBuffs = isd->numBuffers;
136 
137  return --(isd->expectedChildren);
138  }
139  }
140 
141  /** Destructor */
142  virtual ~TunnelDef() { shutdown(); }
143 
144  /** Clean up a region */
145  void shutdown()
146  {
147  if ( master ) {
148  for ( size_t i = 0; i < circBuffs.size(); i++ ) {
149  circBuffs[i]->~CircBuff_t();
150  }
151  }
152 
153  if ( shmPtr ) {
154  shmPtr = NULL;
155  isd = NULL;
156  sharedData = NULL;
157  shmSize = 0;
158  }
159  }
160 
161  /** return size of tunnel */
162  size_t getTunnelSize() { return shmSize; }
163 
164  /** return a pointer to the ShareDataType region */
165  ShareDataType* getSharedData() { return sharedData; }
166 
167  /** Write data to buffer, blocks until space is available
168  * @param buffer which buffer index to write to
169  * @param command message to write to buffer
170  */
171  void writeMessage(size_t buffer, const MsgType& command) { circBuffs[buffer]->write(command); }
172 
173  /** Read data from buffer, blocks until message received
174  * @param buffer which buffer to read from
175  * return the message
176  */
177  MsgType readMessage(size_t buffer) { return circBuffs[buffer]->read(); }
178 
179  /** Read data from buffer, non-blocking
180  * @param buffer which buffer to read from
181  * @param result pointer to return read message at
182  * return whether a message was read
183  */
184  bool readMessageNB(size_t buffer, MsgType* result) { return circBuffs[buffer]->readNB(result); }
185 
186  /** Empty the messages in a buffer
187  * @param buffer which buffer to empty
188  */
189  void clearBuffer(size_t buffer) { circBuffs[buffer]->clearBuffer(); }
190 
191  /** return whether this is a master-side tunnel or a child*/
192  bool isMaster() { return master; }
193 
194 private:
195  /** Allocate space for a data structure in the shared region
196  * @tparam T data structure type to allocate space for
197  * @param extraSpace how many extra bytes to reserve with this allocation
198  * return offset from shmPtr where structure was allocated and pointer to structure
199  */
200  template <typename T>
201  std::pair<size_t, T*> reserveSpace(size_t extraSpace = 0)
202  {
203  size_t space = sizeof(T) + extraSpace;
204  if ( (size_t)((nextAllocPtr + space) - (uint8_t*)shmPtr) > shmSize ) return std::make_pair<size_t, T*>(0, NULL);
205  T* ptr = (T*)nextAllocPtr;
206  nextAllocPtr += space;
207  new (ptr) T(); // Call constructor if need be
208  return std::make_pair((uint8_t*)ptr - (uint8_t*)shmPtr, ptr);
209  }
210 
211  /** Calculate the size of the tunnel */
212  static size_t calculateShmemSize(size_t numBuffers, size_t bufferSize)
213  {
214  long pagesize = sysconf(_SC_PAGESIZE);
215  /* Count how many pages are needed, at minimum */
216  size_t isd = 1 + ((sizeof(InternalSharedData) + (1 + numBuffers) * sizeof(size_t)) / pagesize);
217  size_t buffer = 1 + ((sizeof(CircularBuffer<MsgType>) + bufferSize * sizeof(MsgType)) / pagesize);
218  size_t shdata = 1 + ((sizeof(ShareDataType) + sizeof(InternalSharedData)) / pagesize);
219 
220  /* Alloc 2 extra pages just in case */
221  return (2 + isd + shdata + numBuffers * buffer) * pagesize;
222  }
223 
224 protected:
225  /** Pointer to the Shared Data Region */
226  ShareDataType* sharedData;
227 
228  /** Return the number of buffers */
229  size_t getNumBuffers() { return numBuffs; }
230 
231 private:
232  bool master;
233  void* shmPtr;
234 
235  uint8_t* nextAllocPtr;
236  size_t shmSize;
237 
238  // Local data
239  size_t numBuffs;
240  size_t buffSize;
241  uint32_t children;
242 
243  // Shared objects
244  InternalSharedData* isd;
245  std::vector<CircBuff_t*> circBuffs;
246 };
247 
248 } // namespace Interprocess
249 } // namespace Core
250 } // namespace SST
251 
252 #endif // SST_CORE_INTERPROCESS_TUNNEL_DEF_H
ShareDataType * getSharedData()
return a pointer to the ShareDataType region
Definition: tunneldef.h:165
void clearBuffer(size_t buffer)
Empty the messages in a buffer.
Definition: tunneldef.h:189
size_t getTunnelSize()
return size of tunnel
Definition: tunneldef.h:162
ShareDataType * sharedData
Pointer to the Shared Data Region.
Definition: tunneldef.h:226
TunnelDef(void *sPtr)
Access an existing tunnel Child creates the TunnelDef, reads the shmSize, and then resizes its map ac...
Definition: tunneldef.h:81
size_t getNumBuffers()
Return the number of buffers.
Definition: tunneldef.h:229
uint32_t initialize(void *sPtr)
Finish setting up a tunnel once the manager knows the correct size of the tunnel and has mmap&#39;d a lar...
Definition: tunneldef.h:92
Definition: circularBuffer.h:22
void shutdown()
Clean up a region.
Definition: tunneldef.h:145
This class defines a shared-memory region between a master process and one or more child processes Re...
Definition: tunneldef.h:56
virtual ~TunnelDef()
Destructor.
Definition: tunneldef.h:142
bool readMessageNB(size_t buffer, MsgType *result)
Read data from buffer, non-blocking.
Definition: tunneldef.h:184
void writeMessage(size_t buffer, const MsgType &command)
Write data to buffer, blocks until space is available.
Definition: tunneldef.h:171
MsgType readMessage(size_t buffer)
Read data from buffer, blocks until message received.
Definition: tunneldef.h:177
TunnelDef(size_t numBuffers, size_t bufferSize, uint32_t expectedChildren)
Create a new tunnel.
Definition: tunneldef.h:68
bool isMaster()
return whether this is a master-side tunnel or a child
Definition: tunneldef.h:192