SST  12.0.1
StructuralSimulationToolkit
simulation_impl.h
1 // -*- c++ -*-
2 
3 // Copyright 2009-2022 NTESS. Under the terms
4 // of Contract DE-NA0003525 with NTESS, the U.S.
5 // Government retains certain rights in this software.
6 //
7 // Copyright (c) 2009-2022, NTESS
8 // All rights reserved.
9 //
10 // This file is part of the SST software package. For license
11 // information, see the LICENSE file in the top level directory of the
12 // distribution.
13 
14 #ifndef SST_CORE_SIMULATION_IMPL_H
15 #define SST_CORE_SIMULATION_IMPL_H
16 
17 #include "sst/core/clock.h"
18 #include "sst/core/componentInfo.h"
19 #include "sst/core/oneshot.h"
20 #include "sst/core/output.h"
21 #include "sst/core/rankInfo.h"
22 #include "sst/core/simulation.h"
23 #include "sst/core/sst_types.h"
24 #include "sst/core/unitAlgebra.h"
25 
26 #include <atomic>
27 #include <cstdio>
28 #include <iostream>
29 #include <signal.h>
30 #include <thread>
31 #include <unordered_map>
32 
33 /* Forward declare for Friendship */
34 extern int main(int argc, char** argv);
35 
36 namespace SST {
37 
38 #define _SIM_DBG(fmt, args...) __DBG(DBG_SIM, Sim, fmt, ##args)
39 #define STATALLFLAG "--ALLSTATS--"
40 
41 class Activity;
42 class Component;
43 class Config;
44 class ConfigGraph;
45 class Exit;
46 class Factory;
47 class SimulatorHeartbeat;
48 class Link;
49 class LinkMap;
50 class Params;
51 class SharedRegionManager;
52 class SimulatorHeartbeat;
53 class SyncBase;
54 class SyncManager;
55 class ThreadSync;
56 class TimeConverter;
57 class TimeLord;
58 class TimeVortex;
59 class UnitAlgebra;
60 class SharedRegionManager;
61 namespace Statistics {
62 class StatisticOutput;
63 class StatisticProcessingEngine;
64 } // namespace Statistics
65 
66 namespace Statistics {
67 class StatisticOutput;
68 class StatisticProcessingEngine;
69 } // namespace Statistics
70 
71 /**
72  * Main control class for a SST Simulation.
73  * Provides base features for managing the simulation
74  */
76 {
77 
78 public:
79  /******** Public API inherited from Simulation ********/
80  /** Get the run mode of the simulation (e.g. init, run, both etc) */
81  Mode_t getSimulationMode() const override { return runMode; };
82 
83  /** Return the current simulation time as a cycle count*/
84  SimTime_t getCurrentSimCycle() const override;
85 
86  /** Return the end simulation time as a cycle count*/
87  SimTime_t getEndSimCycle() const override;
88 
89  /** Return the current priority */
90  int getCurrentPriority() const override;
91 
92  /** Return the elapsed simulation time as a time */
93  UnitAlgebra getElapsedSimTime() const override;
94 
95  /** Return the end simulation time as a time */
96  UnitAlgebra getEndSimTime() const override;
97 
98  /** Return the end simulation time as a time */
99  UnitAlgebra getFinalSimTime() const override;
100 
101  /** Get this instance's parallel rank */
102  RankInfo getRank() const override { return my_rank; }
103 
104  /** Get the number of parallel ranks in the simulation */
105  RankInfo getNumRanks() const override { return num_ranks; }
106 
107  /**
108  Returns the output directory of the simulation
109  @return Directory in which simulation outputs are placed
110  */
111  std::string& getOutputDirectory() override { return output_directory; }
112 
113  /** Signifies that an event type is required for this simulation
114  * Causes the Factory to verify that the required event type can be found.
115  * @param name fully qualified libraryName.EventName
116  */
117  virtual void requireEvent(const std::string& name) override;
118 
119  /** Signifies that a library is required for this simulation.
120  * @param name Name of the library
121  */
122  virtual void requireLibrary(const std::string& name) override;
123 
124  /** Causes the current status of the simulation to be printed to stderr.
125  * @param fullStatus - if true, call printStatus() on all components as well
126  * as print the base Simulation's status
127  */
128  virtual void printStatus(bool fullStatus) override;
129 
130  virtual double getRunPhaseElapsedRealTime() const override;
131  virtual double getInitPhaseElapsedRealTime() const override;
132  virtual double getCompletePhaseElapsedRealTime() const override;
133 
134  /******** End Public API from Simulation ********/
135 
136  typedef std::map<std::pair<SimTime_t, int>, Clock*> clockMap_t; /*!< Map of times to clocks */
137  typedef std::map<std::pair<SimTime_t, int>, OneShot*> oneShotMap_t; /*!< Map of times to OneShots */
138 
139  ~Simulation_impl();
140 
141  /********* Static Core-only Functions *********/
142 
143  /** Return a pointer to the singleton instance of the Simulation */
144  static Simulation_impl* getSimulation() { return instanceMap.at(std::this_thread::get_id()); }
145 
146  /** Return the TimeLord associated with this Simulation */
147  static TimeLord* getTimeLord(void) { return &timeLord; }
148 
149  /** Return the base simulation Output class instance */
150  static Output& getSimulationOutput() { return sim_output; }
151 
152  /** Create new simulation
153  * @param config - Configuration of the simulation
154  * @param my_rank - Parallel Rank of this simulation object
155  * @param num_ranks - How many Ranks are in the simulation
156  */
157  static Simulation_impl* createSimulation(Config* config, RankInfo my_rank, RankInfo num_ranks);
158 
159  /**
160  * Used to signify the end of simulation. Cleans up any existing Simulation Objects
161  */
162  static void shutdown();
163 
164  /** Sets an internal flag for signaling the simulation. Used internally */
165  static void setSignal(int signal);
166 
167  /** Insert an activity to fire at a specified time */
168  void insertActivity(SimTime_t time, Activity* ev);
169 
170  /** Return the exit event */
171  Exit* getExit() const { return m_exit; }
172 
173  /******** Core only API *************/
174 
175  /** Processes the ConfigGraph to pull out any need information
176  * about relationships among the threads
177  */
178  void processGraphInfo(ConfigGraph& graph, const RankInfo& myRank, SimTime_t min_part);
179 
180  int prepareLinks(ConfigGraph& graph, const RankInfo& myRank, SimTime_t min_part);
181  int performWireUp(ConfigGraph& graph, const RankInfo& myRank, SimTime_t min_part);
182  void exchangeLinkInfo();
183 
184  /** Set cycle count, which, if reached, will cause the simulation to halt. */
185  void setStopAtCycle(Config* cfg);
186 
187  /** Perform the init() phase of simulation */
188  void initialize();
189 
190  /** Perform the complete() phase of simulation */
191  void complete();
192 
193  /** Perform the setup() and run phases of the simulation. */
194  void setup();
195 
196  void run();
197 
198  void finish();
199 
200  bool isIndependentThread() { return independent; }
201 
202  void printPerformanceInfo();
203 
204  /** Register a OneShot event to be called after a time delay
205  Note: OneShot cannot be canceled, and will always callback after
206  the timedelay.
207  */
208  TimeConverter* registerOneShot(const std::string& timeDelay, OneShot::HandlerBase* handler, int priority);
209 
210  TimeConverter* registerOneShot(const UnitAlgebra& timeDelay, OneShot::HandlerBase* handler, int priority);
211 
212  const std::vector<SimTime_t>& getInterThreadLatencies() const { return interThreadLatencies; }
213 
214  SimTime_t getInterThreadMinLatency() const { return interThreadMinLatency; }
215 
216  static TimeConverter* getMinPartTC() { return minPartTC; }
217 
218  LinkMap* getComponentLinkMap(ComponentId_t id) const
219  {
220  ComponentInfo* info = compInfoMap.getByID(id);
221  if ( nullptr == info ) { return nullptr; }
222  else {
223  return info->getLinkMap();
224  }
225  }
226 
227  /** Returns reference to the Component Info Map */
228  const ComponentInfoMap& getComponentInfoMap(void) { return compInfoMap; }
229 
230  /** returns the component with the given ID */
231  BaseComponent* getComponent(const ComponentId_t& id) const
232  {
233  ComponentInfo* i = compInfoMap.getByID(id);
234  // CompInfoMap_t::const_iterator i = compInfoMap.find(id);
235  if ( nullptr != i ) { return i->getComponent(); }
236  else {
237  printf("Simulation::getComponent() couldn't find component with id = %" PRIu64 "\n", id);
238  exit(1);
239  }
240  }
241 
242  /** returns the ComponentInfo object for the given ID */
243  ComponentInfo* getComponentInfo(const ComponentId_t& id) const
244  {
245  ComponentInfo* i = compInfoMap.getByID(id);
246  // CompInfoMap_t::const_iterator i = compInfoMap.find(id);
247  if ( nullptr != i ) { return i; }
248  else {
249  printf("Simulation::getComponentInfo() couldn't find component with id = %" PRIu64 "\n", id);
250  exit(1);
251  }
252  }
253 
254  /**
255  Set the output directory for this simulation
256  @param outDir Path of directory to place simulation outputs in
257  */
258  void setOutputDirectory(const std::string& outDir) { output_directory = outDir; }
259 
260  /**
261  * Gets the minimum next activity time across all TimeVortices in
262  * the Rank
263  */
264  static SimTime_t getLocalMinimumNextActivityTime();
265 
266  /**
267  * Returns true when the Wireup is finished.
268  */
269  bool isWireUpFinished() { return wireUpFinished; }
270 
271  uint64_t getTimeVortexMaxDepth() const;
272 
273  uint64_t getTimeVortexCurrentDepth() const;
274 
275  uint64_t getSyncQueueDataSize() const;
276 
277  /******** API provided through BaseComponent only ***********/
278 
279  /** Register a handler to be called on a set frequency */
280  TimeConverter* registerClock(const std::string& freq, Clock::HandlerBase* handler, int priority);
281 
282  TimeConverter* registerClock(const UnitAlgebra& freq, Clock::HandlerBase* handler, int priority);
283 
284  TimeConverter* registerClock(TimeConverter* tcFreq, Clock::HandlerBase* handler, int priority);
285 
286  void registerClockHandler(SST::ComponentId_t id, HandlerId_t handler);
287 
288  /** Remove a clock handler from the list of active clock handlers */
289  void unregisterClock(TimeConverter* tc, Clock::HandlerBase* handler, int priority);
290 
291  /** Reactivate an existing clock and handler.
292  * @return time when handler will next fire
293  */
294  Cycle_t reregisterClock(TimeConverter* tc, Clock::HandlerBase* handler, int priority);
295 
296  /** Returns the next Cycle that the TImeConverter would fire. */
297  Cycle_t getNextClockCycle(TimeConverter* tc, int priority = CLOCKPRIORITY);
298 
299  /** Return the Statistic Processing Engine associated with this Simulation */
301 
302  // private:
303 
304  friend class Link;
305  friend class Action;
306  friend class Output;
307  // To enable main to set up globals
308  friend int ::main(int argc, char** argv);
309 
310  // Simulation_impl() {}
311  Simulation_impl(Config* config, RankInfo my_rank, RankInfo num_ranks);
312  Simulation_impl(Simulation_impl const&); // Don't Implement
313  void operator=(Simulation_impl const&); // Don't implement
314 
315  /** Get a handle to a TimeConverter
316  * @param cycles Frequency which is the base of the TimeConverter
317  */
318  TimeConverter* minPartToTC(SimTime_t cycles) const;
319 
320  /** Factory used to generate the simulation components */
321  static Factory* factory;
322 
323  static void resizeBarriers(uint32_t nthr);
324  static Core::ThreadSafe::Barrier initBarrier;
325  static Core::ThreadSafe::Barrier completeBarrier;
326  static Core::ThreadSafe::Barrier setupBarrier;
327  static Core::ThreadSafe::Barrier runBarrier;
328  static Core::ThreadSafe::Barrier exitBarrier;
329  static Core::ThreadSafe::Barrier finishBarrier;
330  static std::mutex simulationMutex;
331 
332  static std::map<LinkId_t, Link*> cross_thread_links;
333  bool direct_interthread;
334 
335  Component* createComponent(ComponentId_t id, const std::string& name, Params& params);
336 
337  TimeVortex* getTimeVortex() const { return timeVortex; }
338 
339  /** Emergency Shutdown
340  * Called when a SIGINT or SIGTERM has been seen
341  */
342  static void emergencyShutdown();
343  /** Normal Shutdown
344  */
345  void endSimulation(void);
346  void endSimulation(SimTime_t end);
347 
348  typedef enum {
349  SHUTDOWN_CLEAN, /* Normal shutdown */
350  SHUTDOWN_SIGNAL, /* SIGINT or SIGTERM received */
351  SHUTDOWN_EMERGENCY, /* emergencyShutdown() called */
352  } ShutdownMode_t;
353 
354  friend class SyncManager;
355 
356  TimeVortex* timeVortex;
357  TimeConverter* threadMinPartTC;
358  Activity* current_activity;
359  static SimTime_t minPart;
360  static TimeConverter* minPartTC;
361  std::vector<SimTime_t> interThreadLatencies;
362  SimTime_t interThreadMinLatency;
363  SyncManager* syncManager;
364  // ThreadSync* threadSync;
365  ComponentInfoMap compInfoMap;
366  clockMap_t clockMap;
367  oneShotMap_t oneShotMap;
368  static Exit* m_exit;
369  SimulatorHeartbeat* m_heartbeat;
370  bool endSim;
371  bool independent; // true if no links leave thread (i.e. no syncs required)
372  static std::atomic<int> untimed_msg_count;
373  unsigned int untimed_phase;
374  volatile sig_atomic_t lastRecvdSignal;
375  ShutdownMode_t shutdown_mode;
376  bool wireUpFinished;
377 
378  /** TimeLord of the simulation */
380  /** Output */
382 
383  /** Performance Tracking Information **/
384 
385 #if SST_PERFORMANCE_INSTRUMENTING
386  FILE* fp;
387  std::map<SST::HandlerId_t, SST::ComponentId_t> handler_mapping;
388 #endif
389 
390 #if SST_PERIODIC_PRINT
391  uint64_t periodicCounter = 0;
392 #endif
393 
394 #if SST_RUNTIME_PROFILING
395  uint64_t sumtime = 0;
396  uint64_t runtime = 0;
397  struct timeval start, end, diff;
398  struct timeval sumstart, sumend, sumdiff;
399 #endif
400 
401 #if SST_CLOCK_PROFILING
402  std::map<SST::HandlerId_t, uint64_t> clockHandlers;
403  std::map<SST::HandlerId_t, uint64_t> clockCounters;
404 #endif
405 
406 #if SST_EVENT_PROFILING
407  uint64_t rankLatency = 0;
408  uint64_t rankExchangeCounter = 0;
409  std::map<std::string, uint64_t> eventHandlers;
410  std::map<std::string, uint64_t> eventRecvCounters;
411  std::map<std::string, uint64_t> eventSendCounters;
412  uint64_t messageXferSize = 0;
413 #endif
414 
415 #if SST_SYNC_PROFILING
416  uint64_t syncCounter = 0;
417  uint64_t rankSyncTime = 0;
418  uint64_t threadSyncTime = 0;
419  uint64_t rankSyncCounter = 0;
420 #endif
421 
422 #if SST_HIGH_RESOLUTION_CLOCK
423  uint64_t clockDivisor = 1e9;
424  std::string clockResolution = "ns";
425 #else
426  uint64_t clockDivisor = 1e6;
427  std::string clockResolution = "us";
428 #endif
429 
430  Mode_t runMode;
431  SimTime_t currentSimCycle;
432  SimTime_t endSimCycle;
433  int currentPriority;
434  RankInfo my_rank;
435  RankInfo num_ranks;
436 
437  std::string output_directory;
438  static SharedRegionManager* sharedRegionManager;
439 
440  double run_phase_start_time;
441  double run_phase_total_time;
442  double init_phase_start_time;
443  double init_phase_total_time;
444  double complete_phase_start_time;
445  double complete_phase_total_time;
446 
447  static std::unordered_map<std::thread::id, Simulation_impl*> instanceMap;
448  static std::vector<Simulation_impl*> instanceVec;
449 
450  friend void wait_my_turn_start();
451  friend void wait_my_turn_end();
452 
453 private:
454  /**
455  * Returns the time of the next item to be executed
456  * that is in the TImeVortex of the Simulation
457  */
458  SimTime_t getNextActivityTime() const;
459 };
460 
461 // Function to allow for easy serialization of threads while debugging
462 // code. Can only be used when you can guarantee all threads will be
463 // taking the same code path. If you can't guarantee that, then use a
464 // spinlock to make sure only one person is in a given region at a
465 // time. ONLY FOR DEBUG USE.
466 void wait_my_turn_start(Core::ThreadSafe::Barrier& barrier, int thread, int total_threads);
467 
468 void wait_my_turn_end(Core::ThreadSafe::Barrier& barrier, int thread, int total_threads);
469 
470 } // namespace SST
471 
472 #endif // SST_CORE_SIMULATION_IMPL_H
Output object provides consistent method for outputting data to stdout, stderr and/or sst debug file...
Definition: output.h:51
An Action is a schedulable Activity which is not an Event.
Definition: action.h:26
Definition: ssthandler.h:100
SimTime_t getCurrentSimCycle() const override
Return the current simulation time as a cycle count.
Definition: simulation.cc:70
virtual double getCompletePhaseElapsedRealTime() const override
Get the amount of real-time spent executing the complete phase of the simulation. ...
Definition: simulation.cc:862
ComponentInfo * getComponentInfo(const ComponentId_t &id) const
returns the ComponentInfo object for the given ID
Definition: simulation_impl.h:243
Main control class for a SST Simulation.
Definition: simulation.h:34
Base class for all Activities in the SST Event Queue.
Definition: activity.h:46
Mode_t getSimulationMode() const override
Get the run mode of the simulation (e.g.
Definition: simulation_impl.h:81
Class to contain SST Simulation Configuration variables.
Definition: config.h:29
TimeConverter * registerOneShot(const std::string &timeDelay, OneShot::HandlerBase *handler, int priority)
Register a OneShot event to be called after a time delay Note: OneShot cannot be canceled, and will always callback after the timedelay.
Definition: simulation.cc:951
void insertActivity(SimTime_t time, Activity *ev)
Insert an activity to fire at a specified time.
Definition: simulation.cc:977
A class to convert between a component&#39;s view of time and the core&#39;s view of time.
Definition: timeConverter.h:26
RankInfo getRank() const override
Get this instance&#39;s parallel rank.
Definition: simulation_impl.h:102
Main component object for the simulation.
Definition: component.h:30
A Configuration Graph A graph representing Components and Links.
Definition: configGraph.h:389
Primary Event Queue.
Definition: timeVortex.h:25
static Output sim_output
Output.
Definition: simulation_impl.h:381
bool isWireUpFinished()
Returns true when the Wireup is finished.
Definition: simulation_impl.h:269
void setOutputDirectory(const std::string &outDir)
Set the output directory for this simulation.
Definition: simulation_impl.h:258
static Output & getSimulationOutput()
Return the base simulation Output class instance.
Definition: simulation_impl.h:150
void setup()
Perform the setup() and run phases of the simulation.
Definition: simulation.cc:596
SSTHandlerBaseNoArgs< void, false > HandlerBase
Base handler for OneShot callbacks.
Definition: oneshot.h:38
const ComponentInfoMap & getComponentInfoMap(void)
Returns reference to the Component Info Map.
Definition: simulation_impl.h:228
void initialize()
Perform the init() phase of simulation.
Definition: simulation.cc:515
Exit Event Action.
Definition: exit.h:33
Definition: componentInfo.h:249
Exit * getExit() const
Return the exit event.
Definition: simulation_impl.h:171
BaseComponent * getComponent(const ComponentId_t &id) const
returns the component with the given ID
Definition: simulation_impl.h:231
virtual void printStatus(bool fullStatus) override
Causes the current status of the simulation to be printed to stderr.
Definition: simulation.cc:827
int getCurrentPriority() const override
Return the current priority.
Definition: simulation.cc:82
static Simulation_impl * createSimulation(Config *config, RankInfo my_rank, RankInfo num_ranks)
Create new simulation.
Definition: simulation.cc:158
A Clock class.
Definition: clock.h:32
uint64_t clockDivisor
Performance Tracking Information.
Definition: simulation_impl.h:426
static void setSignal(int signal)
Sets an internal flag for signaling the simulation.
Definition: simulation.cc:820
UnitAlgebra getEndSimTime() const override
Return the end simulation time as a time.
Definition: simulation.cc:94
Cycle_t reregisterClock(TimeConverter *tc, Clock::HandlerBase *handler, int priority)
Reactivate an existing clock and handler.
Definition: simulation.cc:917
RankInfo getNumRanks() const override
Get the number of parallel ranks in the simulation.
Definition: simulation_impl.h:105
Statistics::StatisticProcessingEngine * getStatisticsProcessingEngine(void) const
Return the Statistic Processing Engine associated with this Simulation.
Definition: simulation.cc:1002
static TimeLord * getTimeLord(void)
Return the TimeLord associated with this Simulation.
Definition: simulation_impl.h:147
virtual double getInitPhaseElapsedRealTime() const override
Get the amount of real-time spent executing the init phase of the simulation.
Definition: simulation.cc:852
std::map< std::pair< SimTime_t, int >, OneShot * > oneShotMap_t
Definition: simulation_impl.h:137
virtual double getRunPhaseElapsedRealTime() const override
Get the amount of real-time spent executing the run phase of the simulation.
Definition: simulation.cc:842
static SimTime_t getLocalMinimumNextActivityTime()
Gets the minimum next activity time across all TimeVortices in the Rank.
Definition: simulation.cc:254
void unregisterClock(TimeConverter *tc, Clock::HandlerBase *handler, int priority)
Remove a clock handler from the list of active clock handlers.
Definition: simulation.cc:941
Main control class for a SST Simulation.
Definition: simulation_impl.h:75
Definition: rankInfo.h:21
TimeConverter * minPartToTC(SimTime_t cycles) const
Get a handle to a TimeConverter.
Definition: simulation.cc:110
Main component object for the simulation.
Definition: baseComponent.h:49
static Simulation_impl * getSimulation()
Return a pointer to the singleton instance of the Simulation.
Definition: simulation_impl.h:144
UnitAlgebra getElapsedSimTime() const override
Return the elapsed simulation time as a time.
Definition: simulation.cc:88
void complete()
Perform the complete() phase of simulation.
Definition: simulation.cc:560
static Factory * factory
Factory used to generate the simulation components.
Definition: simulation_impl.h:321
std::string & getOutputDirectory() override
Returns the output directory of the simulation.
Definition: simulation_impl.h:111
Cycle_t getNextClockCycle(TimeConverter *tc, int priority=CLOCKPRIORITY)
Returns the next Cycle that the TImeConverter would fire.
Definition: simulation.cc:929
A OneShot Event class.
Definition: oneshot.h:32
static void emergencyShutdown()
Emergency Shutdown Called when a SIGINT or SIGTERM has been seen.
Definition: simulation.cc:751
static TimeLord timeLord
TimeLord of the simulation.
Definition: simulation_impl.h:379
Parameter store.
Definition: params.h:55
void endSimulation(void)
Normal Shutdown.
Definition: simulation.cc:769
void setStopAtCycle(Config *cfg)
Set cycle count, which, if reached, will cause the simulation to halt.
Definition: simulation.cc:219
Class for creating and managing TimeConverter objects.
Definition: timeLord.h:37
virtual void requireLibrary(const std::string &name) override
Signifies that a library is required for this simulation.
Definition: simulation.cc:242
Class for instantiating Components, Links and the like out of element libraries.
Definition: factory.h:42
TimeConverter * registerClock(const std::string &freq, Clock::HandlerBase *handler, int priority)
Register a handler to be called on a set frequency.
Definition: simulation.cc:872
virtual void requireEvent(const std::string &name) override
Signifies that an event type is required for this simulation Causes the Factory to verify that the re...
Definition: simulation.cc:236
Definition: componentInfo.h:39
void processGraphInfo(ConfigGraph &graph, const RankInfo &myRank, SimTime_t min_part)
Processes the ConfigGraph to pull out any need information about relationships among the threads...
Definition: simulation.cc:265
An SST core component that handles timing and event processing informing all registered Statistics to...
Definition: statengine.h:51
Mode_t
Type of Run Modes.
Definition: simulation.h:38
UnitAlgebra getFinalSimTime() const override
Return the end simulation time as a time.
Definition: simulation.cc:100
std::map< std::pair< SimTime_t, int >, Clock * > clockMap_t
Definition: simulation_impl.h:136
SimTime_t getEndSimCycle() const override
Return the end simulation time as a cycle count.
Definition: simulation.cc:76
Performs Unit math in full precision.
Definition: unitAlgebra.h:106
Definition: threadsafe.h:47
static void shutdown()
Used to signify the end of simulation.
Definition: simulation.cc:171