diff --git a/.env-sample b/.env-sample index 48de821..f3201ce 100644 --- a/.env-sample +++ b/.env-sample @@ -12,10 +12,6 @@ ENABLE_SCHEDULED_JOBS=true # and would otherwise crash-loop the whole app rather than just fail quietly. ENABLE_LEGACY_PROVIDER_JOBS=false -METEO_USER=aaa -METEO_PASSWORD=aaa -METEO_URL=aaa - LVGMC_URL=aaa LVGMC_USER=aaa LVGMC_PASSWORD=aaa diff --git a/deploy/vps/compose.yml b/deploy/vps/compose.yml index b3ed31d..fca0296 100644 --- a/deploy/vps/compose.yml +++ b/deploy/vps/compose.yml @@ -32,12 +32,10 @@ services: POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} POSTGRES_HOST: postgres ENABLE_SCHEDULED_JOBS: "false" + ENABLE_LEGACY_PROVIDER_JOBS: "false" DEBUG: "false" # Provider services are currently constructed at startup even with jobs # disabled, so inert values are required for synthetic-data UAT. - METEO_USER: disabled - METEO_PASSWORD: disabled - METEO_URL: https://invalid.local/ LVGMC_USER: disabled LVGMC_PASSWORD: disabled LVGMC_URL: https://invalid.local/ diff --git a/docker-compose.yml b/docker-compose.yml index 823083c..89f6d41 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,9 +26,6 @@ services: POSTGRES_USER: ${POSTGRES_USER} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} POSTGRES_HOST: ${POSTGRES_HOST} - METEO_USER: ${METEO_USER} - METEO_PASSWORD: ${METEO_PASSWORD} - METEO_URL: ${METEO_URL} LVGMC_URL: ${LVGMC_URL} LVGMC_USER: ${LVGMC_USER} LVGMC_PASSWORD: ${LVGMC_PASSWORD} @@ -37,6 +34,7 @@ services: HARMONIE_STAC_API_KEY: ${HARMONIE_STAC_API_KEY} HARMONIE_STAC_URL: ${HARMONIE_STAC_URL} ENABLE_SCHEDULED_JOBS: ${ENABLE_SCHEDULED_JOBS:-true} + ENABLE_LEGACY_PROVIDER_JOBS: ${ENABLE_LEGACY_PROVIDER_JOBS:-false} build: context: . dockerfile: Dockerfile.local diff --git a/src/main/scala/fetch/csv/FetchService.scala b/src/main/scala/fetch/csv/FetchService.scala deleted file mode 100644 index d3d6755..0000000 --- a/src/main/scala/fetch/csv/FetchService.scala +++ /dev/null @@ -1,82 +0,0 @@ -package fetch.csv - -import cats.effect._ -import cats.implicits._ -import org.http4s._ -import org.http4s.client.Client -import org.http4s.ember.client.EmberClientBuilder -import org.http4s.headers.Authorization -import org.typelevel.log4cats.Logger -import org.typelevel.log4cats.slf4j.Slf4jLogger - -import java.time.{LocalDate, LocalDateTime} - -final case class WeatherServerConfig( - username: String, - password: String, - url: String, -) - -object FetchService { - def of: IO[FetchService] = { - Slf4jLogger.create[IO].map(logger => new FetchService(new FileNameService, logger)) - } -} - -class FetchService(fileNameService: FileNameService, log: Logger[IO]) {private val weatherServerConfig: WeatherServerConfig = ( - sys.env.get("METEO_USER"), - sys.env.get("METEO_PASSWORD"), - sys.env.get("METEO_URL") -) match { - case (Some(user), Some(password), Some(url)) => - WeatherServerConfig(user, password, url) - case _ => - throw new RuntimeException("Unable to load meteo config: Missing required environment variables") -} - - private val basicCredentials: BasicCredentials = - BasicCredentials(weatherServerConfig.username, weatherServerConfig.password) - - private val baseUrl: IO[Uri] = IO(Uri.unsafeFromString(weatherServerConfig.url)) - - private def makeRequest(client: Client[IO], url: Uri): IO[Either[Throwable, (String, String)]] = { - val fileName = url.path.toString().tail - val request = Request[IO](Method.GET, url).withHeaders(Authorization(basicCredentials)) - - client.expect[String](request).redeemWith( - error => IO(Left(error)) // <* log.error(s"Request failed to url: $url with error: ${error.getMessage}") - , - fileContent => IO(Right((fileName, fileContent))) // <* log.info(s"Fetched: $fileName") - ) - } - - private def fetchFiles(fileNames: List[String]): IO[List[Either[Throwable, (String, String)]]] = { - val IOUrls = baseUrl.map(baseUrl => fileNames.map(baseUrl / _)) - EmberClientBuilder.default[IO].build.use { client => - IOUrls.flatMap(_.traverse(url => makeRequest(client, url))) - } - } - - def fetchSingleFile(fileName: String): IO[Either[Throwable, (String, String)]] = { - fetchFiles(List(fileName)).map { results => - results.headOption match { - case Some(Right(result)) => - log.info(s"fetched: $fileName").as(Right(result)) - case Some(Left(err)) => - log.error(s"failed fetch: $fileName with error: ${err.getMessage}").as(Left(err)) - case None => - log.error(s"failed fetch: $fileName").as(Left(new Exception("No file fetched"))) - } - }.flatten - } - - def fetchInRange(from: LocalDateTime, to: LocalDateTime): IO[List[Either[Throwable, (String, String)]]] = { - val fileNames = fileNameService.generate(from, to) - fetchFiles(fileNames) - } - - def fetchFromDate(date: LocalDate): IO[List[Either[Throwable, (String, String)]]] = { - val fileNames = fileNameService.generateFromDate(date) - fetchFiles(fileNames) - } -} \ No newline at end of file diff --git a/src/main/scala/server/Server.scala b/src/main/scala/server/Server.scala index 5ae1a3c..8e04433 100644 --- a/src/main/scala/server/Server.scala +++ b/src/main/scala/server/Server.scala @@ -6,7 +6,6 @@ import com.comcast.ip4s.IpLiteralSyntax import data.DataService import db.PostgresService import fetch.csv.FileNameService -//import fetch.csv.FetchService import fetch.lvgmc.FetchService import fetch.warnings.WarningService import fs2.io.file.{Files, Path} diff --git a/src/test/scala/fetch/FetchServiceSpec.scala b/src/test/scala/fetch/FetchServiceSpec.scala deleted file mode 100644 index e7e3d98..0000000 --- a/src/test/scala/fetch/FetchServiceSpec.scala +++ /dev/null @@ -1,51 +0,0 @@ -package fetch - -import cats.effect.IO -import cats.effect.unsafe.implicits.global -import cats.implicits.toTraverseOps -import org.scalatest.funsuite.AnyFunSuite -import org.scalatest.matchers.should.Matchers -import org.typelevel.log4cats.slf4j.Slf4jLogger - -import java.time.LocalDateTime - -class FetchServiceSpec extends AnyFunSuite with Matchers { -// def fetchInRange: IO[Unit] = { -// val from = LocalDateTime.of(2023, 4, 28, 10, 0) -// val to = LocalDateTime.of(2023, 4, 28, 13, 30) -// for { -// log <- Slf4jLogger.create[IO] -// fetch <- FetchService.of -// // fetchResultEither <- fetch.fetchFromDate(LocalDate.of(2023, 4, 28)).attempt -// fetchResultEither <- fetch.fetchInRange(from, to).attempt -// fetchServiceError = fetchResultEither.left.toOption.map(e => s"FetchServiceError: ${e.getMessage}").toList -// fetchResult = fetchResultEither.getOrElse(List.empty) -// (fetchErrors, successDownloads) = fetchResult.partitionMap(identity) -// dbService <- DBService.of -// saveResults <- successDownloads.traverse { case (name, content) => dbService.save(name, content) } -// (saveErrors, successSaves) = saveResults.partitionMap(identity) -// successes = successDownloads.map(s => s"fetched: ${s._1}") ++ successSaves.map(s => s"saved: $s") -// errors = fetchServiceError ++ fetchErrors.map(e => s"FetchError: ${e.getMessage}") ++ saveErrors.map(e => s"SaveError: ${e.getMessage}") -// _ <- log.info(s"errors: $errors") -// _ <- log.info(s"successes: $successes") -// } yield (successes, errors) -// } -// -// def fetchSingleFile: IO[Unit] = { -// for { -// fetch <- FetchService.of -// fetchResultEither <- fetch.fetchSingleFile("20230524_0030.csv").attempt -// fetchResultEither <- fetch.fetchSingleFile("20230522_0130.csv").attempt -// fetchServiceError = fetchResultEither.left.toOption.map(e => s"FetchServiceError: ${e.getMessage}").toList -// fetchResult = fetchResultEither.flatMap(res => res.flatMap(aaa => { -// println(s"fffffff: ${aaa._1}") -// Right(aaa._1) -// })) -// // _ = println(s"${fetchResult.map()}") -// } yield () -// } -// -// def main(args: Array[String]): Unit = { -// run.unsafeRunSync() -// } -}