diff --git a/.env-sample b/.env-sample index 5b6bf86..3c650cb 100644 --- a/.env-sample +++ b/.env-sample @@ -6,6 +6,10 @@ METEO_USER=aaa METEO_PASSWORD=aaa METEO_URL=aaa +LVGMC_URL=aaa +LVGMC_USER=aaa +LVGMC_PASSWORD=aaa + HARMONIE_EDR_API_KEY=aaa HARMONIE_EDR_URL=aaa HARMONIE_STAC_API_KEY=aaa diff --git a/build.sbt b/build.sbt index 7437e68..6d9b7b6 100644 --- a/build.sbt +++ b/build.sbt @@ -30,6 +30,8 @@ libraryDependencies ++= Seq( "org.http4s" %% "http4s-ember-client" % http4sVersion, "org.http4s" %% "http4s-circe" % http4sVersion, + "commons-net" % "commons-net" % "3.9.0", + "org.tpolecat" %% "doobie-core" % doobieVersion, "org.tpolecat" %% "doobie-postgres" % doobieVersion, diff --git a/docker-compose.yml b/docker-compose.yml index c2068d6..ddc5f1e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,6 +31,9 @@ services: METEO_USER: ${METEO_USER} METEO_PASSWORD: ${METEO_PASSWORD} METEO_URL: ${METEO_URL} + LVGMC_URL: ${LVGMC_URL} + LVGMC_USER: ${LVGMC_USER} + LVGMC_PASSWORD: ${LVGMC_PASSWORD} HARMONIE_EDR_API_KEY: ${HARMONIE_EDR_API_KEY} HARMONIE_EDR_URL: ${HARMONIE_EDR_URL} HARMONIE_STAC_API_KEY: ${HARMONIE_STAC_API_KEY} diff --git a/src/main/scala/fetch/lvgmc/FetchService.scala b/src/main/scala/fetch/lvgmc/FetchService.scala new file mode 100644 index 0000000..214ee33 --- /dev/null +++ b/src/main/scala/fetch/lvgmc/FetchService.scala @@ -0,0 +1,88 @@ +package fetch.lvgmc + +import cats.effect._ +import org.apache.commons.net.ftp.{FTP, FTPClient, FTPReply} +import org.typelevel.log4cats.Logger +import org.typelevel.log4cats.slf4j.Slf4jLogger + +import java.io.ByteArrayOutputStream + +final case class LVGMCServerConfig( + username: String, + password: String, + url: String, +) + +object FetchService { + def of: IO[FetchService] = { + Slf4jLogger.create[IO].map(logger => new FetchService(logger)) + } +} + +class FetchService(log: Logger[IO]) { + private val serverConfig: LVGMCServerConfig = ( + sys.env.get("LVGMC_USER"), + sys.env.get("LVGMC_PASSWORD"), + sys.env.get("LVGMC_URL") + ) match { + case (Some(user), Some(password), Some(url)) => + LVGMCServerConfig(user, password, url) + case _ => + throw new RuntimeException("Unable to load lvgmc config: Missing required environment variables") + } + + private def createFtpClient: Resource[IO, FTPClient] = { + Resource.make { + IO.blocking { + val ftpClient = new FTPClient() + ftpClient.connect(serverConfig.url) + + val reply = ftpClient.getReplyCode + if (!FTPReply.isPositiveCompletion(reply)) { + ftpClient.disconnect() + throw new RuntimeException(s"FTP server refused connection, reply code: $reply") + } + + val loggedIn = ftpClient.login(serverConfig.username, serverConfig.password) + if (!loggedIn) { + ftpClient.disconnect() + throw new RuntimeException("Failed to login to FTP server") + } + + ftpClient.enterLocalPassiveMode() + ftpClient.setFileType(FTP.ASCII_FILE_TYPE) + ftpClient + } + } { client => + IO.blocking { + if (client.isConnected) { + client.logout() + client.disconnect() + } + }.handleErrorWith(e => log.error(e)("Error closing FTP connection")) + } + } + + private def retrieveFileAsString(ftpClient: FTPClient, remotePath: String): IO[String] = IO.blocking { + val outputStream = new ByteArrayOutputStream() + + val success = ftpClient.retrieveFile(remotePath, outputStream) + if (!success) { + throw new RuntimeException(s"Failed to retrieve file: $remotePath, reply: ${ftpClient.getReplyString}") + } + + outputStream.toString("UTF-8") + }.onError(e => log.error(e)(s"Error retrieving file $remotePath")) + + def fetchFile(fileName: String): IO[String] = { + val remotePath = s"/ltv/tabulas/$fileName" + + createFtpClient.use { ftpClient => + for { + _ <- log.info(s"Fetching file: $remotePath") + content <- retrieveFileAsString(ftpClient, remotePath) + _ <- log.info(s"Successfully fetched file: $fileName") + } yield content + } + } +} \ No newline at end of file diff --git a/src/main/scala/fetch/lvgmc/FetchServiceTest.scala b/src/main/scala/fetch/lvgmc/FetchServiceTest.scala new file mode 100644 index 0000000..24915c3 --- /dev/null +++ b/src/main/scala/fetch/lvgmc/FetchServiceTest.scala @@ -0,0 +1,25 @@ +package fetch.lvgmc + +import cats.effect.IO +import cats.effect.unsafe.implicits.global + +object FetchServiceTest { + def main(args: Array[String]): Unit = { + fetchFile().unsafeRunSync() + } + +// Eiropa_LTV_pilsetas_nakama_dn.csv +// Eiropa_LTV_pilsetas_tekosa_dn.csv +// Latvija_LTV_pilsetas_nakama_dnn.csv +// Latvija_LTV_pilsetas_tekosa_dn.csv +// Latvija_faktiskais_laiks.csv + + private def fetchFile(): IO[Unit] = { + val program = for { + fetch <- FetchService.of + result <- fetch.fetchFile("Latvija_LTV_pilsetas_tekosa_dn.csv") + _ <- IO.println(result) + } yield () + program + } +}