diff --git a/.gitignore b/.gitignore index 2b424aba..c2cd097a 100644 --- a/.gitignore +++ b/.gitignore @@ -153,4 +153,4 @@ testData/download !testData/download/targetFolderExistsAndIsFile.zip # Downloader productive folder -inputData/download \ No newline at end of file +input/download \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 6cfdf8ff..898adbb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- Option to use local files instead of download [#256](https://github.com/ie3-institute/simBench2psdm/issues/256) + + ## [1.0.0] - 2021-08-03 ### Added - Basic functionality to convert SimBench data sets to [PowerSystemDataModel](https://github.com/ie3-institute/powersystemdatamodel) diff --git a/gradle/scripts/tscfg.gradle b/gradle/scripts/tscfg.gradle index fc2026a6..90ac4283 100644 --- a/gradle/scripts/tscfg.gradle +++ b/gradle/scripts/tscfg.gradle @@ -5,7 +5,7 @@ task genConfigClass { doLast { def tscfgJarFile = project.file('build/tscfg-' + tscfgVersion + '.jar') if (!tscfgJarFile.exists() || !tscfgJarFile.isFile()) { - download { + download.run { src 'https://github.com/carueda/tscfg/releases/download/v' + tscfgVersion + '/tscfg-' + tscfgVersion + '.jar' dest buildDir } diff --git a/inputData/config/allNoSwitches.conf b/input/config/allNoSwitches.conf similarity index 100% rename from inputData/config/allNoSwitches.conf rename to input/config/allNoSwitches.conf diff --git a/input/config/localFileSourceConfig.conf b/input/config/localFileSourceConfig.conf new file mode 100644 index 00000000..8b60ebdf --- /dev/null +++ b/input/config/localFileSourceConfig.conf @@ -0,0 +1,34 @@ +# Simple config to convert only one simple lv grid +io { + input { + folder = "input/local/" + + localFile = { + isZipped = true + } + + csv = { + fileEncoding = "UTF-8" + fileEnding = ".csv" + separator = ";" + directoryHierarchy = false + } + } + + output { + csv = { + fileEncoding = "UTF-8" + fileEnding = ".csv" + separator = ";" + directoryHierarchy = false + } + + targetFolder = "convertedData/local/" + compress = false + } + + # code for local files + simbenchCodes = [ + "1-LV-rural1--0-no_sw" + ] +} diff --git a/inputData/config/multiVoltLvlNoSwitches.conf b/input/config/multiVoltLvlNoSwitches.conf similarity index 100% rename from inputData/config/multiVoltLvlNoSwitches.conf rename to input/config/multiVoltLvlNoSwitches.conf diff --git a/inputData/config/mvLvNoSwitches.conf b/input/config/mvLvNoSwitches.conf similarity index 100% rename from inputData/config/mvLvNoSwitches.conf rename to input/config/mvLvNoSwitches.conf diff --git a/inputData/config/mvWithSwitches.conf b/input/config/mvWithSwitches.conf similarity index 100% rename from inputData/config/mvWithSwitches.conf rename to input/config/mvWithSwitches.conf diff --git a/inputData/config/singleVoltLvlNoSwitches.conf b/input/config/singleVoltLvlNoSwitches.conf similarity index 100% rename from inputData/config/singleVoltLvlNoSwitches.conf rename to input/config/singleVoltLvlNoSwitches.conf diff --git a/inputData/config/smallGrid.conf b/input/config/smallGrid.conf similarity index 100% rename from inputData/config/smallGrid.conf rename to input/config/smallGrid.conf diff --git a/input/local/1-LV-rural1--0-no_sw.zip b/input/local/1-LV-rural1--0-no_sw.zip new file mode 100644 index 00000000..3d361450 Binary files /dev/null and b/input/local/1-LV-rural1--0-no_sw.zip differ diff --git a/src/main/resources/config-template.conf b/src/main/resources/config-template.conf index 811b2a47..59e99e3f 100644 --- a/src/main/resources/config-template.conf +++ b/src/main/resources/config-template.conf @@ -9,14 +9,23 @@ CsvConfig { io { simbenchCodes = ["String"] input { - download.baseUrl = "String" | "https://daks.uni-kassel.de/bitstreams" - download.folder = "String" | "inputData/download/" - download.failOnExistingFiles = "Boolean" | false + directory = "String" | "input/download/" + + #@optional + localFile { + isZipped = "Boolean" | true + failOnExistingFiles = "Boolean" | false + } + #@optional + download { + baseUrl = "String" | "https://daks.uni-kassel.de/bitstreams" + failOnExistingFiles = "Boolean" | false + } csv = CsvConfig } output { csv = CsvConfig - targetFolder = "String" | "convertedData" + targetDir = "String" | "convertedData" compress = "Boolean" | true } } diff --git a/src/main/scala/edu/ie3/simbench/config/ConfigValidator.scala b/src/main/scala/edu/ie3/simbench/config/ConfigValidator.scala index 9ce76698..7e859cf6 100644 --- a/src/main/scala/edu/ie3/simbench/config/ConfigValidator.scala +++ b/src/main/scala/edu/ie3/simbench/config/ConfigValidator.scala @@ -36,6 +36,7 @@ case object ConfigValidator { @throws[SimbenchException] private def checkValidity(io: SimbenchConfig.Io): Unit = { checkSimbenchCodes(io.simbenchCodes) + checkFileSource(io.input) } /** Checks the validity of the provided codes with the help of the permissible @@ -47,6 +48,10 @@ case object ConfigValidator { */ @throws[CodeValidationException] private def checkSimbenchCodes(codes: List[java.lang.String]): Unit = { + if (codes.isEmpty) { + throw new SimbenchConfigException(s"No simbench codes were provided!") + } + for (code <- codes) { SimbenchCode.isValid(code) match { case Success(_) => @@ -54,4 +59,22 @@ case object ConfigValidator { } } } + + @throws[SimbenchException] + private def checkFileSource(cfg: SimbenchConfig.Io.Input): Unit = { + val sources = Vector( + cfg.localFile, + cfg.download + ).filter(_.isDefined) + + sources.size match { + case 0 => + throw new SimbenchConfigException(s"No file sources defined in: $cfg") + case 1 => + case n => + throw new SimbenchConfigException( + s"Too many file sources ($n) defined in: $cfg" + ) + } + } } diff --git a/src/main/scala/edu/ie3/simbench/config/SimbenchConfig.scala b/src/main/scala/edu/ie3/simbench/config/SimbenchConfig.scala index 19b59cf6..f0e2fc03 100644 --- a/src/main/scala/edu/ie3/simbench/config/SimbenchConfig.scala +++ b/src/main/scala/edu/ie3/simbench/config/SimbenchConfig.scala @@ -1,4 +1,4 @@ -// generated by tscfg 0.9.986 on Mon Aug 09 20:12:24 CEST 2021 +// generated by tscfg 1.1.3 on Mon Dec 09 08:47:34 CET 2024 // source: src/main/resources/config-template.conf package edu.ie3.simbench.config @@ -47,9 +47,8 @@ object SimbenchConfig { $tsCfgValidator: $TsCfgValidator ): SimbenchConfig.Conversion = { SimbenchConfig.Conversion( - removeSwitches = c.hasPathOrNull("removeSwitches") && c.getBoolean( - "removeSwitches" - ) + removeSwitches = + c.hasPathOrNull("removeSwitches") && c.getBoolean("removeSwitches") ) } } @@ -62,13 +61,14 @@ object SimbenchConfig { object Io { final case class Input( csv: SimbenchConfig.CsvConfig, - download: SimbenchConfig.Io.Input.Download + directory: java.lang.String, + download: scala.Option[SimbenchConfig.Io.Input.Download], + localFile: scala.Option[SimbenchConfig.Io.Input.LocalFile] ) object Input { final case class Download( baseUrl: java.lang.String, - failOnExistingFiles: scala.Boolean, - directory: java.lang.String + failOnExistingFiles: scala.Boolean ) object Download { def apply( @@ -80,13 +80,30 @@ object SimbenchConfig { baseUrl = if (c.hasPathOrNull("baseUrl")) c.getString("baseUrl") else "https://daks.uni-kassel.de/bitstreams", + failOnExistingFiles = + c.hasPathOrNull("failOnExistingFiles") && c.getBoolean( + "failOnExistingFiles" + ) + ) + } + } + + final case class LocalFile( + failOnExistingFiles: scala.Boolean, + isZipped: scala.Boolean + ) + object LocalFile { + def apply( + c: com.typesafe.config.Config, + parentPath: java.lang.String, + $tsCfgValidator: $TsCfgValidator + ): SimbenchConfig.Io.Input.LocalFile = { + SimbenchConfig.Io.Input.LocalFile( failOnExistingFiles = c.hasPathOrNull("failOnExistingFiles") && c.getBoolean( "failOnExistingFiles" ), - directory = - if (c.hasPathOrNull("folder")) c.getString("folder") - else "inputData/download/" + isZipped = !c.hasPathOrNull("isZipped") || c.getBoolean("isZipped") ) } } @@ -103,12 +120,29 @@ object SimbenchConfig { parentPath + "csv.", $tsCfgValidator ), - download = SimbenchConfig.Io.Input.Download( - if (c.hasPathOrNull("download")) c.getConfig("download") - else com.typesafe.config.ConfigFactory.parseString("download{}"), - parentPath + "download.", - $tsCfgValidator - ) + directory = + if (c.hasPathOrNull("directory")) c.getString("directory") + else "input/download/", + download = + if (c.hasPathOrNull("download")) + scala.Some( + SimbenchConfig.Io.Input.Download( + c.getConfig("download"), + parentPath + "download.", + $tsCfgValidator + ) + ) + else None, + localFile = + if (c.hasPathOrNull("localFile")) + scala.Some( + SimbenchConfig.Io.Input.LocalFile( + c.getConfig("localFile"), + parentPath + "localFile.", + $tsCfgValidator + ) + ) + else None ) } } @@ -133,7 +167,7 @@ object SimbenchConfig { $tsCfgValidator ), targetDir = - if (c.hasPathOrNull("targetFolder")) c.getString("targetFolder") + if (c.hasPathOrNull("targetDir")) c.getString("targetDir") else "convertedData" ) } @@ -208,7 +242,7 @@ object SimbenchConfig { java.lang.String.valueOf(cv.unwrapped()) } - private final class $TsCfgValidator { + final class $TsCfgValidator { private val badPaths = scala.collection.mutable.ArrayBuffer[java.lang.String]() diff --git a/src/main/scala/edu/ie3/simbench/io/Extractor.scala b/src/main/scala/edu/ie3/simbench/io/Extractor.scala index 68e555ce..c330eef0 100644 --- a/src/main/scala/edu/ie3/simbench/io/Extractor.scala +++ b/src/main/scala/edu/ie3/simbench/io/Extractor.scala @@ -7,7 +7,7 @@ import scala.io.Source import scala.util.matching.Regex import com.typesafe.scalalogging.LazyLogging -class Extractor(simbenchConfig: SimbenchConfig) extends LazyLogging { +class Extractor(simbenchConfig: SimbenchConfig.Io.Input) extends LazyLogging { def download(): Unit = { val url = new URL( @@ -16,7 +16,7 @@ class Extractor(simbenchConfig: SimbenchConfig) extends LazyLogging { val inputStream = url.openStream() try { - val downloadFolder = simbenchConfig.io.input.download.directory + val downloadFolder = simbenchConfig.directory val outputPath = s"$downloadFolder/simbench_datalinks.csv" val path = Paths.get(outputPath) Files.createDirectories(path.getParent) @@ -30,7 +30,7 @@ class Extractor(simbenchConfig: SimbenchConfig) extends LazyLogging { /** Extracts a map of simbench-code to UUID from the downloaded file. */ def extractUUIDMap(): Map[String, String] = { - val downloadDir = simbenchConfig.io.input.download.directory + val downloadDir = simbenchConfig.directory val outputPath = s"$downloadDir/simbench_datalinks.csv" val uuidPattern: Regex = """[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}""".r diff --git a/src/main/scala/edu/ie3/simbench/io/SimbenchReader.scala b/src/main/scala/edu/ie3/simbench/io/SimbenchReader.scala index 2f9d6676..ec4f39e9 100644 --- a/src/main/scala/edu/ie3/simbench/io/SimbenchReader.scala +++ b/src/main/scala/edu/ie3/simbench/io/SimbenchReader.scala @@ -87,6 +87,12 @@ final case class SimbenchReader( * Await is okay here */ val modelClassToRawData = getFieldToValueMaps + if (modelClassToRawData.forall(_._2.isEmpty)) { + throw SimbenchDataModelException( + s"Model is empty. Please check the provided data." + ) + } + /* Extracting all profiles */ val loadProfiles = buildModels(modelClassToRawData, LoadProfile) val powerPlantProfiles = buildModels(modelClassToRawData, PowerPlantProfile) @@ -330,7 +336,7 @@ final case class SimbenchReader( read(clazz, fields) } ), - Duration("10 s") + Duration("30 s") ) .toMap } diff --git a/src/main/scala/edu/ie3/simbench/main/RunSimbench.scala b/src/main/scala/edu/ie3/simbench/main/RunSimbench.scala index 81772c83..8edd0c85 100644 --- a/src/main/scala/edu/ie3/simbench/main/RunSimbench.scala +++ b/src/main/scala/edu/ie3/simbench/main/RunSimbench.scala @@ -1,14 +1,8 @@ package edu.ie3.simbench.main -import java.nio.file.{Path, Paths} -import edu.ie3.datamodel.io.naming.{ - DefaultDirectoryHierarchy, - EntityPersistenceNamingStrategy, - FileNamingStrategy -} -import edu.ie3.datamodel.io.sink.CsvFileSink import edu.ie3.simbench.config.{ConfigValidator, SimbenchConfig} import edu.ie3.simbench.convert.GridConverter +import edu.ie3.simbench.io.SimbenchReader import edu.ie3.simbench.exception.CodeValidationException import edu.ie3.simbench.io.{ Downloader, @@ -21,12 +15,7 @@ import edu.ie3.simbench.model.SimbenchCode import edu.ie3.util.io.FileIOUtils import org.apache.commons.io.FilenameUtils -import scala.concurrent.Await -import scala.concurrent.ExecutionContext.Implicits.global -import scala.concurrent.duration.Duration import scala.jdk.CollectionConverters._ -import scala.jdk.FutureConverters.CompletionStageOps -import scala.util.{Failure, Success} /** This is not meant to be final production code. It is more a place for * "testing" the full method stack. @@ -42,10 +31,6 @@ object RunSimbench extends SimbenchHelper { /* Validate the config */ ConfigValidator.checkValidity(simbenchConfig) - val extractor = new Extractor(simbenchConfig) - extractor.download() - val uuidMap = extractor.extractUUIDMap() - simbenchConfig.io.simbenchCodes.foreach { simbenchCode => // todo: replace these two if statements with a proper handling of switches if (!simbenchConfig.conversion.removeSwitches) { @@ -55,38 +40,17 @@ object RunSimbench extends SimbenchHelper { logger.warn(s"Using a simbench grid with '-sw' might not work.") } - logger.info(s"$simbenchCode - Downloading data set from SimBench website") - val downloader = - Downloader( - simbenchConfig.io.input.download.directory, - simbenchConfig.io.input.download.baseUrl, - uuidMap, - simbenchConfig.io.input.download.failOnExistingFiles - ) - val downloadedFile = - downloader.download( - SimbenchCode(simbenchCode).getOrElse( - throw CodeValidationException( - s"'$simbenchCode' is no valid SimBench code." - ) - ) - ) - val dataFolder = - Zipper.unzip( - downloadedFile, - downloader.downloadDir, - simbenchConfig.io.input.download.failOnExistingFiles, - flattenDirectories = true - ) + val gridPath = getGridPath(simbenchCode, simbenchConfig.io.input) logger.info(s"$simbenchCode - Reading in the SimBench data set") val simbenchReader = SimbenchReader( simbenchCode, - dataFolder, + gridPath, simbenchConfig.io.input.csv.separator, simbenchConfig.io.input.csv.fileEnding, simbenchConfig.io.input.csv.fileEncoding ) + val simbenchModel = simbenchReader.readGrid() logger.info(s"$simbenchCode - Converting to PowerSystemDataModel") @@ -103,55 +67,16 @@ object RunSimbench extends SimbenchHelper { ) logger.info(s"$simbenchCode - Writing converted data set to files") - /* Check, if a directory hierarchy is needed or not */ - val baseTargetDirectory = - IoUtils.ensureHarmonizedAndTerminatingFileSeparator( - simbenchConfig.io.output.targetDir - ) - val csvSink = if (simbenchConfig.io.output.csv.directoryHierarchy) { - new CsvFileSink( - Path.of(baseTargetDirectory), - new FileNamingStrategy( - new EntityPersistenceNamingStrategy(), - new DefaultDirectoryHierarchy( - Path.of(baseTargetDirectory), - simbenchCode - ) - ), - simbenchConfig.io.output.csv.separator - ) - } else { - new CsvFileSink( - Path.of(baseTargetDirectory + simbenchCode), - new FileNamingStrategy(), - simbenchConfig.io.output.csv.separator - ) - } + val (csvSink, baseTargetDirectory) = + createCsvSink(simbenchCode, simbenchConfig) csvSink.persistJointGrid(jointGridContainer) timeSeries.foreach(csvSink.persistTimeSeries(_)) csvSink.persistAllIgnoreNested(timeSeriesMapping.asJava) csvSink.persistAll(powerFlowResults.asJava) if (simbenchConfig.io.output.compress) { - logger.info(s"$simbenchCode - Adding files to compressed archive") - val rawOutputPath = Paths.get(baseTargetDirectory + simbenchCode) - val archivePath = Paths.get( - FilenameUtils.concat(baseTargetDirectory, simbenchCode + ".tar.gz") - ) - val compressFuture = - FileIOUtils.compressDir(rawOutputPath, archivePath).asScala - compressFuture.onComplete { - case Success(_) => - FileIOUtils.deleteRecursively(rawOutputPath) - case Failure(exception) => - logger.error( - s"Compression of output files to '$archivePath' has failed. Keep raw data.", - exception - ) - } - - Await.ready(compressFuture, Duration("180s")) + compressCsv(simbenchCode, baseTargetDirectory) } } } diff --git a/src/main/scala/edu/ie3/simbench/main/SimbenchHelper.scala b/src/main/scala/edu/ie3/simbench/main/SimbenchHelper.scala index ad2e0a7f..ca61babf 100644 --- a/src/main/scala/edu/ie3/simbench/main/SimbenchHelper.scala +++ b/src/main/scala/edu/ie3/simbench/main/SimbenchHelper.scala @@ -2,10 +2,28 @@ package edu.ie3.simbench.main import com.typesafe.config.{ConfigFactory, Config => TypesafeConfig} import com.typesafe.scalalogging.LazyLogging -import edu.ie3.simbench.config.ArgsParser +import edu.ie3.datamodel.io.naming.{ + DefaultDirectoryHierarchy, + EntityPersistenceNamingStrategy, + FileNamingStrategy +} +import edu.ie3.datamodel.io.sink.CsvFileSink import edu.ie3.simbench.config.ArgsParser.Arguments +import edu.ie3.simbench.config.{ArgsParser, SimbenchConfig} +import edu.ie3.simbench.exception.CodeValidationException import edu.ie3.simbench.exception.io.SimbenchConfigException +import edu.ie3.simbench.io.{Downloader, Extractor, IoUtils, Zipper} +import edu.ie3.simbench.model.SimbenchCode +import edu.ie3.util.io.FileIOUtils import edu.ie3.util.quantities.QuantityAdjustments +import org.apache.commons.io.FilenameUtils + +import java.nio.file.{Path, Paths} +import scala.concurrent.Await +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.duration.Duration +import scala.jdk.FutureConverters.CompletionStageOps +import scala.util.{Failure, Success} trait SimbenchHelper extends LazyLogging { @@ -64,4 +82,140 @@ trait SimbenchHelper extends LazyLogging { println("( V ) SimBench2PDSM ( V )") println("--m-m-------------------m-m--") } + + /** Method for returning the grid path. If downloading is used, the specified + * simbench grid model will be automatically downloaded. + * @param simbenchCode + * of the grid + * @param cfg + * config that specifies the folder and the download option + * @return + * the path to the grid + */ + private[main] def getGridPath( + simbenchCode: String, + cfg: SimbenchConfig.Io.Input + ): Path = { + (cfg.localFile, cfg.download) match { + case (Some(local), None) => + if (local.isZipped) { + Zipper.unzip( + Path.of(cfg.directory, simbenchCode + ".zip"), + cfg.directory, + local.failOnExistingFiles, + flattenDirectories = true + ) + } else { + Path.of(cfg.directory, simbenchCode) + } + case (None, Some(download)) => + logger.info( + s"$simbenchCode - Downloading data set from SimBench website" + ) + + val extractor = new Extractor(cfg) + extractor.download() + val uuidMap = extractor.extractUUIDMap() + + val downloadCfg = cfg.download.getOrElse( + throw new SimbenchConfigException( + s"There was no download configuration provided!" + ) + ) + + val downloader = + Downloader( + cfg.directory, + downloadCfg.baseUrl, + uuidMap, + downloadCfg.failOnExistingFiles + ) + val downloadedFile = + downloader.download( + SimbenchCode(simbenchCode).getOrElse( + throw CodeValidationException( + s"'$simbenchCode' is no valid SimBench code." + ) + ) + ) + + Zipper.unzip( + downloadedFile, + downloader.downloadDir, + download.failOnExistingFiles, + flattenDirectories = true + ) + } + } + + /** Method for creating a [[CsvFileSink]] + * @param simbenchCode + * for the folder name + * @param simbenchConfig + * config that specifies the output path + * @return + * a file sink and the baseTargetDirectory + */ + private[main] def createCsvSink( + simbenchCode: String, + simbenchConfig: SimbenchConfig + ): (CsvFileSink, String) = { + /* Check, if a directory hierarchy is needed or not */ + val baseTargetDirectory = + IoUtils.ensureHarmonizedAndTerminatingFileSeparator( + simbenchConfig.io.output.targetDir + ) + + val csvSink = if (simbenchConfig.io.output.csv.directoryHierarchy) { + new CsvFileSink( + Path.of(baseTargetDirectory), + new FileNamingStrategy( + new EntityPersistenceNamingStrategy(), + new DefaultDirectoryHierarchy( + Path.of(baseTargetDirectory), + simbenchCode + ) + ), + simbenchConfig.io.output.csv.separator + ) + } else { + new CsvFileSink( + Path.of(baseTargetDirectory + simbenchCode), + new FileNamingStrategy(), + simbenchConfig.io.output.csv.separator + ) + } + + (csvSink, baseTargetDirectory) + } + + /** Method to compress a given PSDM grid. + * @param simbenchCode + * of the grid + * @param baseTargetDirectory + * of the grid + */ + private[main] def compressCsv( + simbenchCode: String, + baseTargetDirectory: String + ): Unit = { + logger.info(s"$simbenchCode - Adding files to compressed archive") + val rawOutputPath = Paths.get(baseTargetDirectory + simbenchCode) + val archivePath = Paths.get( + FilenameUtils.concat(baseTargetDirectory, simbenchCode + ".tar.gz") + ) + val compressFuture = + FileIOUtils.compressDir(rawOutputPath, archivePath).asScala + compressFuture.onComplete { + case Success(_) => + FileIOUtils.deleteRecursively(rawOutputPath) + case Failure(exception) => + logger.error( + s"Compression of output files to '$archivePath' has failed. Keep raw data.", + exception + ) + } + + Await.ready(compressFuture, Duration("180s")) + } } diff --git a/src/test/scala/edu/ie3/simbench/config/ConfigValidatorSpec.scala b/src/test/scala/edu/ie3/simbench/config/ConfigValidatorSpec.scala index 9c174c0c..4c2cc1af 100644 --- a/src/test/scala/edu/ie3/simbench/config/ConfigValidatorSpec.scala +++ b/src/test/scala/edu/ie3/simbench/config/ConfigValidatorSpec.scala @@ -1,6 +1,8 @@ package edu.ie3.simbench.config +import edu.ie3.simbench.config.SimbenchConfig.Io.Input import edu.ie3.simbench.exception.CodeValidationException +import edu.ie3.simbench.exception.io.SimbenchConfigException import edu.ie3.test.common.{ConfigTestData, UnitSpec} class ConfigValidatorSpec extends UnitSpec with ConfigTestData { @@ -27,5 +29,40 @@ class ConfigValidatorSpec extends UnitSpec with ConfigTestData { ) } } + + "throw a SimbenchConfigException if no source is defined" in { + val method = PrivateMethod[Unit](Symbol("checkFileSource")) + val config = validIoInputConfig.copy(download = None) + + intercept[SimbenchConfigException] { + ConfigValidator invokePrivate method(config) + } match { + case correctException: SimbenchConfigException => + correctException.getMessage shouldBe s"No file sources defined in: $config" + case _ => + fail("The validation did not fail properly.") + } + } + + "throw a SimbenchConfigException if too many sources are defined" in { + val method = PrivateMethod[Unit](Symbol("checkFileSource")) + val config = validIoInputConfig.copy(localFile = + Some( + new Input.LocalFile( + failOnExistingFiles = false, + isZipped = true + ) + ) + ) + + intercept[SimbenchConfigException] { + ConfigValidator invokePrivate method(config) + } match { + case correctException: SimbenchConfigException => + correctException.getMessage shouldBe s"Too many file sources (2) defined in: $config" + case _ => + fail("The validation did not fail properly.") + } + } } } diff --git a/src/test/scala/edu/ie3/simbench/io/ExtractorSpec.scala b/src/test/scala/edu/ie3/simbench/io/ExtractorSpec.scala index d592d7a1..8285ac91 100644 --- a/src/test/scala/edu/ie3/simbench/io/ExtractorSpec.scala +++ b/src/test/scala/edu/ie3/simbench/io/ExtractorSpec.scala @@ -14,17 +14,18 @@ class ExtractorSpec extends UnitSpec with IoUtils with Matchers { // Define configurations for the extractor val download: Download = Download( baseUrl = "https://daks.uni-kassel.de/bitstreams", - failOnExistingFiles = false, - directory = "testData/download" // Valid download location + failOnExistingFiles = false ) val input: Input = Input( + directory = "testData/download", // Valid download location csv = SimbenchConfig.CsvConfig( directoryHierarchy = false, fileEncoding = "UTF-8", fileEnding = ".csv", separator = ";" ), - download = download + download = Some(download), + localFile = None ) val output: Output = Output( compress = true, @@ -42,7 +43,7 @@ class ExtractorSpec extends UnitSpec with IoUtils with Matchers { io = io ) - val extractor = new Extractor(simbenchConfig) + val extractor = new Extractor(input) val uuidPattern: Regex = "[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}".r @@ -72,7 +73,7 @@ class ExtractorSpec extends UnitSpec with IoUtils with Matchers { "ensure that the download correctly downloads and saves the expected data file" in { extractor.download() val downloadedFile = new File( - s"${simbenchConfig.io.input.download.directory}/simbench_datalinks.csv" + s"${simbenchConfig.io.input.directory}/simbench_datalinks.csv" ) downloadedFile.exists() shouldBe true } @@ -95,14 +96,10 @@ class ExtractorSpec extends UnitSpec with IoUtils with Matchers { ) val invalidConfig = simbenchConfig.copy( io = simbenchConfig.io.copy( - input = simbenchConfig.io.input.copy( - download = simbenchConfig.io.input.download.copy( - directory = invalidDir - ) - ) + input = simbenchConfig.io.input.copy(directory = invalidDir) ) ) - val invalidExtractor = new Extractor(invalidConfig) + val invalidExtractor = new Extractor(invalidConfig.io.input) an[IllegalArgumentException] should be thrownBy invalidExtractor .extractUUIDMap() deleteDir(invalidDir) @@ -116,14 +113,10 @@ class ExtractorSpec extends UnitSpec with IoUtils with Matchers { ) val invalidUUIDConfig = simbenchConfig.copy( io = simbenchConfig.io.copy( - input = simbenchConfig.io.input.copy( - download = simbenchConfig.io.input.download.copy( - directory = invalidUUIDPath - ) - ) + input = simbenchConfig.io.input.copy(directory = invalidUUIDPath) ) ) - val invalidUUIDExtractor = new Extractor(invalidUUIDConfig) + val invalidUUIDExtractor = new Extractor(invalidUUIDConfig.io.input) val uuidMap = invalidUUIDExtractor.extractUUIDMap() uuidMap shouldBe empty deleteDir(invalidUUIDPath) diff --git a/src/test/scala/edu/ie3/simbench/main/SimbenchHelperSpec.scala b/src/test/scala/edu/ie3/simbench/main/SimbenchHelperSpec.scala new file mode 100644 index 00000000..28ee91b4 --- /dev/null +++ b/src/test/scala/edu/ie3/simbench/main/SimbenchHelperSpec.scala @@ -0,0 +1,30 @@ +package edu.ie3.simbench.main + +import edu.ie3.simbench.main.RunSimbench.getGridPath +import edu.ie3.test.common.{ConfigTestData, UnitSpec} +import edu.ie3.util.io.FileIOUtils +import org.scalatest.prop.TableDrivenPropertyChecks + +import java.nio.file.Path + +class SimbenchHelperSpec + extends UnitSpec + with ConfigTestData + with TableDrivenPropertyChecks { + "The SimbenchHelper" should { + "return the specified grid path correctly" in { + val code = "1-LV-rural1--0-no_sw" + + val config = validIoInputConfigWithLocalFileSource.copy(directory = + "testData/localSource/" + ) + + getGridPath( + code, + config + ) shouldBe Path.of("testData", "localSource", code) + + FileIOUtils.deleteRecursively(Path.of(config.directory, code)) + } + } +} diff --git a/src/test/scala/edu/ie3/test/common/ConfigTestData.scala b/src/test/scala/edu/ie3/test/common/ConfigTestData.scala index ae9f0162..4dda2221 100644 --- a/src/test/scala/edu/ie3/test/common/ConfigTestData.scala +++ b/src/test/scala/edu/ie3/test/common/ConfigTestData.scala @@ -8,10 +8,24 @@ import edu.ie3.simbench.config.SimbenchConfig.Io.Input trait ConfigTestData { val validIoInputConfig = new SimbenchConfig.Io.Input( new SimbenchConfig.CsvConfig(false, "UTF-8", ".csv", ";"), - new Input.Download( - "http://141.51.193.167/simbench/gui/usecase/download", - false, - "testData/download/" + "testData/download/", + Some( + new Input.Download( + "http://141.51.193.167/simbench/gui/usecase/download", + false + ) + ), + None + ) + val validIoInputConfigWithLocalFileSource = new SimbenchConfig.Io.Input( + new SimbenchConfig.CsvConfig(false, "UTF-8", ".csv", ";"), + "testData/download/", + None, + Some( + new Input.LocalFile( + failOnExistingFiles = false, + isZipped = true + ) ) ) val validIoOutputConfig = new SimbenchConfig.Io.Output( diff --git a/testData/localSource/1-LV-rural1--0-no_sw.zip b/testData/localSource/1-LV-rural1--0-no_sw.zip new file mode 100644 index 00000000..3d361450 Binary files /dev/null and b/testData/localSource/1-LV-rural1--0-no_sw.zip differ