Refactor to one scheduler, separated csv/grib under one folder

This commit is contained in:
Guntis Smaukstelis
2025-02-14 14:46:31 +02:00
parent 3b8bb7d826
commit 15cfbd1801
27 changed files with 56 additions and 216 deletions
@@ -1,51 +0,0 @@
package fetch
import base.IOSuite
import cats.effect.{Clock, IO}
import cats.effect.kernel.Ref
import cats.implicits.catsSyntaxTuple3Semigroupal
import db.PostgresService
import org.scalatest.matchers.should.Matchers
import org.scalatest.wordspec.AsyncWordSpec
import org.typelevel.log4cats.slf4j.Slf4jLogger
import fs2.Stream
import org.scalamock.scalatest.MockFactory
class FileFetchSchedulerSpec extends AsyncWordSpec with Matchers with MockFactory with IOSuite {
"FileFetchScheduler" should {
"save into db" in runIO {
for {
refDb <- Ref.of[IO, Option[String]](None)
refFetch <- Ref.of[IO, Option[(String, String)]](None)
refScheduler <- Ref.of[IO, Option[Either[Throwable, (String, String)]]](None)
log <- Slf4jLogger.create[IO]
mockDatabaseOps = mock[PostgresService]
fileNameService = new FileNameService {
override def generateCurrentHour(implicit clock: Clock[IO]): IO[String] = IO.pure("file_test")
}
fetchService = new FetchService(fileNameService, log) {
override def fetchSingleFile(fileName: String): IO[Either[Throwable, (String, String)]] = {
refFetch.set(Some((fileName, "content"))).as(Right((fileName, "content")))
}
}
scheduler = new Scheduler(log) {
override def scheduleTask(task: IO[Either[Throwable, (String, String)]]): Stream[IO, Either[Throwable, (String, String)]] = {
Stream.eval(task).flatMap { result =>
Stream.eval(refScheduler.set(Some(result))).as(result)
}
}
}
res = new FileFetchScheduler(mockDatabaseOps, fetchService, fileNameService, scheduler, log)
.run.compile.drain *>
(refDb.get, refFetch.get, refScheduler.get).tupled.map { case (db, fetch, scheduler) =>
db shouldBe Some("file_test")
fetch shouldBe Some(("file_test", "content"))
scheduler shouldBe Some(Right(("file_test", "content")))
}
} yield res
}
}
}
@@ -1,8 +1,8 @@
import cats.Applicative
import cats.effect.unsafe.implicits.global
import fetch.FileNameService
import org.scalatest.funsuite.AnyFunSuite
import cats.effect.{Clock, IO}
import fetch.csv.FileNameService
import java.util.concurrent.TimeUnit
import java.time.LocalDateTime
-25
View File
@@ -1,25 +0,0 @@
package fetch
import cats.effect.{Clock, IO}
import fs2.Stream
class SchedulerSpec {
// private def testTask: IO[Unit] = {
// log.info("Running task")
// }
//
// def run(implicit clock: Clock[IO]): Stream[IO, Nothing] = {
// Stream.eval(durationToNextHalfHour).flatMap { delay =>
// (Stream.sleep[IO](delay) ++ Stream.awakeEvery[IO](1.hour)).evalMap(_ => testTask).drain
// }
// }
//
// def main(args: Array[String]): Unit = {
// // run.compile.drain.unsafeRunSync()
// for {
// scheduler <- Scheduler.of
// fetch <- FetchService.of
// fetchTask = new FileNameService().generateCurrentHour.flatMap(fetch.fetchSingleFile)
// } yield scheduler.scheduleTask(fetchTask).compile.drain.unsafeRunSync()
// }
}
+1 -1
View File
@@ -4,7 +4,7 @@ import cats.effect.unsafe.implicits.global
//import db.FileService
import org.scalatest.funsuite.AnyFunSuite
import org.scalatest.matchers.should.Matchers
import parse.Aggregate.{AggregateKey, DoubleValue, TimeDoubleList, UserQuery}
import parse.csv.Aggregate.{AggregateKey, DoubleValue, TimeDoubleList, UserQuery}
import java.time.LocalDateTime
import java.time.format.DateTimeFormatter