Merge pull request #1038 from ie3-institute/ms/#1034-consider-PrimaryData-that-start-before-simulation

Considering primary data that start before simulation.

Author: sebastian-peter

CHANGELOG.md

@@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Enhancing config with more default value [#1413](https://github.com/ie3-institute/simona/issues/1413)
 - Updated Authors.md file [#1503](https://github.com/ie3-institute/simona/issues/1503)
 - Quantity to squants conversion in WeatherService [#1506](https://github.com/ie3-institute/simona/issues/1506)
+- Considering primary data that start before simulation [#1034](https://github.com/ie3-institute/simona/issues/1034)
 
 ### Changed
 - Upgraded `scala2` to `scala3` [#53](https://github.com/ie3-institute/simona/issues/53)

src/main/scala/edu/ie3/simona/service/SimonaService.scala

@@ -32,6 +32,7 @@ import org.apache.pekko.actor.typed.scaladsl.{
   StashBuffer,
 }
 import org.apache.pekko.actor.typed.{ActorRef, Behavior}
+import org.slf4j.Logger
 
 import scala.util.{Failure, Success, Try}
 
@@ -96,7 +97,7 @@ abstract class SimonaService {
     case (ctx, Activation(INIT_SIM_TICK)) =>
       // init might take some time and could go wrong if invalid initialize service data is received
       // execute complete and unstash only if init is carried out successfully
-      init(initializeStateData) match {
+      init(initializeStateData)(using ctx.log) match {
         case Success((serviceStateData, maybeNewTick)) =>
           scheduler ! Completion(
             ctx.self,
@@ -223,13 +224,15 @@ abstract class SimonaService {
     *
     * @param initServiceData
     *   The data that should be used for initialization
+    * @param log
+    *   The logger for logging.
     * @return
     *   The state data of this service and optional tick that should be included
     *   in the completion message
     */
   def init(
       initServiceData: InitializeServiceStateData
-  ): Try[(S, Option[Long])]
+  )(using log: Logger): Try[(S, Option[Long])]
 
   /** Handle a request to register for information from this service
     *

src/main/scala/edu/ie3/simona/service/ev/ExtEvDataService.scala

@@ -34,6 +34,7 @@ import edu.ie3.simona.util.ReceiveDataMap
 import edu.ie3.simona.util.SimonaConstants.INIT_SIM_TICK
 import org.apache.pekko.actor.typed.ActorRef
 import org.apache.pekko.actor.typed.scaladsl.ActorContext
+import org.slf4j.Logger
 
 import java.util.UUID
 import scala.jdk.CollectionConverters.*
@@ -59,7 +60,7 @@ object ExtEvDataService extends SimonaService with ExtDataSupport {
 
   override def init(
       initServiceData: ServiceStateData.InitializeServiceStateData
-  ): Try[
+  )(using log: Logger): Try[
     (
         ExtEvStateData,
         Option[Long],

src/main/scala/edu/ie3/simona/service/load/LoadProfileService.scala

@@ -30,6 +30,7 @@ import edu.ie3.simona.util.SimonaConstants.FIRST_TICK_IN_SIMULATION
 import edu.ie3.simona.util.TickUtil.TickLong
 import org.apache.pekko.actor.typed.ActorRef
 import org.apache.pekko.actor.typed.scaladsl.ActorContext
+import org.slf4j.Logger
 
 import java.time.ZonedDateTime
 import scala.util.{Failure, Success, Try}
@@ -78,7 +79,7 @@ object LoadProfileService extends SimonaService {
 
   override def init(
       initServiceData: InitializeServiceStateData
-  ): Try[(LoadProfileInitializedStateData, Option[Long])] =
+  )(using log: Logger): Try[(LoadProfileInitializedStateData, Option[Long])] =
     initServiceData match {
       case InitLoadProfileServiceStateData(
             dataSource,

src/main/scala/edu/ie3/simona/service/primary/PrimaryServiceWorker.scala

@@ -155,18 +155,10 @@ object PrimaryServiceWorker extends SimonaService {
 
   override type S = PrimaryServiceInitializedStateData[Value]
 
-  /** Initialize the actor with the given information. Try to figure out the
-    * initialized state data and the next activation ticks, that will then be
-    * sent to the scheduler
-    *
-    * @param initServiceData
-    *   The data that should be used for initialization
-    * @return
-    *   The state data of this service actor and optional tick that should be
-    *   included in the completion message
-    */
   override def init(
       initServiceData: InitializeServiceStateData
+  )(using
+      log: Logger
   ): Try[(PrimaryServiceInitializedStateData[Value], Option[Long])] = {
     (initServiceData match {
       case PrimaryServiceWorker.CsvInitPrimaryServiceStateData(
@@ -255,11 +247,13 @@ object PrimaryServiceWorker extends SimonaService {
             .map(_.toTick)
         ).pop
 
-        (maybeNextTick, furtherActivationTicks) match {
-          case (Some(tick), furtherActivationTicks) if tick == 0L =>
+        val previousOption =
+          source.getLastTimeKeyBefore(simulationStart).toScala
+
+        (maybeNextTick, previousOption) match {
+          case (Some(tick), _) if tick == 0L =>
             /* Set up the state data and determine the next activation tick. */
-            val initializedStateData
-                : PrimaryServiceInitializedStateData[Value] =
+            val initializedStateData =
               PrimaryServiceInitializedStateData(
                 maybeNextTick,
                 furtherActivationTicks,
@@ -270,14 +264,54 @@ object PrimaryServiceWorker extends SimonaService {
 
             Success(initializedStateData, maybeNextTick)
 
-          case (Some(tick), _) if tick > 0L =>
-            /* The start of the data needs to be at the first tick of the simulation. */
+          case (Some(tick), None) if tick > 0L =>
+            /* No data for the first tick or before, but the start of the data needs to be at the first tick of the simulation. */
             Failure(
               new ServiceRegistrationException(
                 s"The data for the timeseries '${source.getTimeSeries.getUuid}' starts after the start of this simulation (tick: $tick)! This is not allowed!"
               )
             )
 
+          case (Some(_), Some(value)) =>
+            /* We have data before and after the start of the simulation, but not at tick 0 */
+            log.debug(
+              s"No data at the start of the simulation. Use last know data for tick: ${value.toTick}"
+            )
+
+            val startTick = Some(0L)
+
+            /* Set up the state data. */
+            val initializedStateData =
+              PrimaryServiceInitializedStateData(
+                startTick,
+                furtherActivationTicks,
+                simulationStart,
+                valueClass,
+                source,
+              )
+
+            Success(initializedStateData, startTick)
+
+          case (_, Some(value)) =>
+            /* We have data before, but not after the start of the simulation */
+            log.warn(
+              s"Only found data before the start of the simulation. Tick: ${value.toTick}"
+            )
+
+            val nextTick = Some(0L)
+
+            /* Set up the state data. */
+            val initializedStateData =
+              PrimaryServiceInitializedStateData(
+                nextTick,
+                furtherActivationTicks,
+                simulationStart,
+                valueClass,
+                source,
+              )
+
+            Success(initializedStateData, nextTick)
+
           case _ =>
             /* No data for the simulation. */
             Failure(
@@ -316,18 +350,6 @@ object PrimaryServiceWorker extends SimonaService {
         )
     }
 
-  /** Send out the information to all registered recipients
-    *
-    * @param tick
-    *   Current tick data should be announced for
-    * @param serviceBaseStateData
-    *   The current state data of this service
-    * @return
-    *   The service stata data that should be used in the next state (normally
-    *   with updated values) together with the completion message that is sent
-    *   in response to the trigger that is sent to start the initialization
-    *   process
-    */
   override protected def announceInformation(
       tick: Long
   )(using
@@ -340,7 +362,7 @@ object PrimaryServiceWorker extends SimonaService {
     /* Get the information to distribute */
     val simulationTime =
       tick.toDateTime(using serviceBaseStateData.startDateTime)
-    serviceBaseStateData.source.getValue(simulationTime).toScala match {
+    serviceBaseStateData.source.getValueOrLast(simulationTime).toScala match {
       case Some(value) =>
         processDataAndAnnounce(tick, value, serviceBaseStateData)(using
           ctx.self,
@@ -438,17 +460,12 @@ object PrimaryServiceWorker extends SimonaService {
       PrimaryServiceInitializedStateData[V],
       Option[Long],
   ) = {
-    val (maybeNextTick, remainderActivationTicks) =
-      serviceBaseStateData.activationTicks.pop
-    val updatedStateData =
-      serviceBaseStateData.copy(
-        maybeNextActivationTick = maybeNextTick,
-        activationTicks = remainderActivationTicks,
-      )
+    val (updatedStateData, maybeNextTick) =
+      updateStateDataAndBuildTriggerMessages(serviceBaseStateData)
 
-    val provisionMessage =
-      DataProvision(tick, self, primaryData, maybeNextTick)
+    val provisionMessage = DataProvision(tick, self, primaryData, maybeNextTick)
     serviceBaseStateData.subscribers.foreach(_ ! provisionMessage)
+
     (updatedStateData, maybeNextTick)
   }
 }

src/main/scala/edu/ie3/simona/service/weather/WeatherService.scala

@@ -33,6 +33,7 @@ import edu.ie3.simona.util.TickUtil.RichZonedDateTime
 import edu.ie3.util.scala.collection.immutable.SortedDistinctSeq
 import org.apache.pekko.actor.typed.ActorRef
 import org.apache.pekko.actor.typed.scaladsl.ActorContext
+import org.slf4j.Logger
 
 import java.time.ZonedDateTime
 import scala.util.{Failure, Success, Try}
@@ -88,23 +89,9 @@ object WeatherService extends SimonaService {
       simulationEnd: ZonedDateTime,
   ) extends InitializeServiceStateData
 
-  /** Initialize the concrete service implementation using the provided
-    * initialization data. This method should perform all heavyweight tasks
-    * before the actor becomes ready. The return values are a) the state data of
-    * the initialized service and b) optional triggers that should be sent to
-    * the [[edu.ie3.simona.scheduler.Scheduler]] together with the completion
-    * message that is sent in response to the trigger that is sent to start the
-    * initialization process
-    *
-    * @param initServiceData
-    *   the data that should be used for initialization
-    * @return
-    *   the state data of this service and optional triggers that should be
-    *   included in the completion message
-    */
   override def init(
       initServiceData: InitializeServiceStateData
-  ): Try[(WeatherInitializedStateData, Option[Long])] =
+  )(using log: Logger): Try[(WeatherInitializedStateData, Option[Long])] =
     initServiceData match {
       case InitWeatherServiceStateData(
             sourceDefinition,

src/test/resources/edu/ie3/simona/service/primary/_it/its_p_b73a7e3f-9045-40cd-b518-c11a9a6a1025.csv

@@ -0,0 +1 @@
+"time";"p"

src/test/scala/edu/ie3/simona/service/primary/PrimaryServiceWorkerSpec.scala

@@ -110,14 +110,16 @@ class PrimaryServiceWorkerSpec
       }
     }
 
-    "fail to init, if time series ends with delay before simulation start" in {
+    "fail to init, if time series has no data" in {
       val initData = validInitData.copy(
-        simulationStart = validInitData.simulationStart.plusHours(1)
+        timeSeriesUuid = uuidEmpty,
+        filePath = Paths.get("its_p_" + uuidEmpty),
+        simulationStart = validInitData.simulationStart.plusHours(1),
       )
 
       PrimaryServiceWorker.init(initData) match {
         case Failure(exception) =>
-          exception.getMessage shouldBe "No appropriate data found within simulation time range in timeseries '9185b8c1-86ba-4a16-8dea-5ac898e8caa5'!"
+          exception.getMessage shouldBe "No appropriate data found within simulation time range in timeseries 'b73a7e3f-9045-40cd-b518-c11a9a6a1025'!"
         case Success(_) =>
           fail("Initialisation with unsupported init data is meant to fail.")
       }
@@ -136,6 +138,34 @@ class PrimaryServiceWorkerSpec
       }
     }
 
+    "init, if there is a value before the simulation start" in {
+      val initData = validInitData.copy(
+        simulationStart = validInitData.simulationStart.plusHours(1)
+      )
+
+      PrimaryServiceWorker.init(initData) match {
+        case Success((_, maybeNextTick)) =>
+          maybeNextTick shouldBe Some(0L)
+
+        case Failure(_) =>
+          fail("Initialisation with init data is meant to succeed.")
+      }
+    }
+
+    "init, if there are values before and after the simulation start" in {
+      val initData = validInitData.copy(
+        simulationStart = validInitData.simulationStart.plusMinutes(5)
+      )
+
+      PrimaryServiceWorker.init(initData) match {
+        case Success((_, maybeNextTick)) =>
+          maybeNextTick shouldBe Some(0L)
+
+        case Failure(_) =>
+          fail("Initialisation with init data is meant to succeed.")
+      }
+    }
+
     "fail, if pointed to the wrong file" in {
       // time series exists, but is malformed
       val tsUuid = UUID.fromString("3fbfaa97-cff4-46d4-95ba-a95665e87c27")
@@ -377,20 +407,10 @@ class PrimaryServiceWorkerSpec
       )
     }
 
-    "should not announce anything, if time step is not covered in source" in {
-
-      serviceRef ! Activation(200)
-
-      val completionMsg = scheduler.expectMessageType[Completion]
-      completionMsg.newTick shouldBe Some(1800)
-
-      systemParticipant.expectNoMessage()
-    }
-
-    "should announce something, if the time step is covered in source" in {
+    "should announce something" in {
       serviceRef ! Activation(900)
       val completionMsg = scheduler.expectMessageType[Completion]
-      completionMsg.newTick shouldBe None
+      completionMsg.newTick shouldBe Some(1800)
 
       inside(
         systemParticipant.expectMessageType[DataProvision[PrimaryData]]
@@ -408,7 +428,7 @@ class PrimaryServiceWorkerSpec
               p should approximate(Kilowatts(1250.0))
             case _ => fail("Expected to get active power only.")
           }
-          nextDataTick shouldBe None
+          nextDataTick shouldBe Some(1800)
       }
     }
   }

src/test/scala/edu/ie3/simona/test/common/input/TimeSeriesTestData.scala

@@ -15,6 +15,8 @@ import java.util.UUID
 trait TimeSeriesTestData {
   protected val uuidP: UUID =
     UUID.fromString("9185b8c1-86ba-4a16-8dea-5ac898e8caa5")
+  protected val uuidEmpty: UUID =
+    UUID.fromString("b73a7e3f-9045-40cd-b518-c11a9a6a1025")
   protected val uuidPq: UUID =
     UUID.fromString("3fbfaa97-cff4-46d4-95ba-a95665e87c26")
   protected val uuidPqh: UUID =