Initial commit with weather data fetcher and parser

2023-04-13 16:58:56 +03:00
commit 40156d6283
15 changed files with 746 additions and 0 deletions
@@ -0,0 +1,51 @@
+package fetch
+
+import cats.effect._
+import cats.implicits._
+import org.http4s._
+import org.http4s.blaze.client.BlazeClientBuilder
+import org.http4s.client.Client
+import org.http4s.headers.Authorization
+
+import com.typesafe.config.ConfigFactory
+
+import java.nio.file.{Files, Paths}
+import scala.concurrent.ExecutionContext.global
+
+object FetchData extends IOApp.Simple {
+  private val config = ConfigFactory.load()
+  private val basicCredentials = BasicCredentials(config.getString("username"), config.getString("password"))
+  private val baseUrl = Uri.unsafeFromString(config.getString("url")) // 20220831_1330.csv
+//  private val baseUrl = Uri.unsafeFromString("https://jsonplaceholder.typicode.com/") // todos/1
+
+  def saveToFile(fileName: String, content: String): IO[Unit] = {
+    val path = Paths.get(s"data/$fileName")
+    IO(Files.writeString(path, content)).attempt.flatMap {
+      case Right(_) => IO(println(s"write: $fileName"))
+      case Left(error) => IO(println(s"Write file '$fileName' failed with error: ${error.getMessage}"))
+    }
+  }
+
+  def makeRequest(client: Client[IO], url: Uri): IO[Unit] = {
+    val fileName = url.path.toString()
+    for {
+      request <- Request[IO](Method.GET, url)
+        .withHeaders(Authorization(basicCredentials))
+        .pure[IO]
+      responseOrError <- client.expect[String](request).attempt
+      _ <- responseOrError match {
+        case Right(response) => saveToFile(fileName, response)
+        case Left(error) => IO(println(s"Request failed to url: $url with error: ${error.getMessage}"))
+      }
+    } yield ()
+  }
+
+  def run: IO[Unit] = {
+    val fileNames = FileName.generateLast5Hours()
+    //    fileNames.foreach(println)
+    val urls = fileNames.map(baseUrl / _)
+    BlazeClientBuilder[IO](global).resource.use { client =>
+      urls.traverse(url => makeRequest(client, url)) // urls.map(...).sequence
+    }
+  }.as(ExitCode.Success)
+}
@@ -0,0 +1,40 @@
+package fetch
+
+import java.time.format.DateTimeFormatter
+import java.time.{LocalDateTime, Duration}
+
+object FileName {
+  private val interval = Duration.ofMinutes(30)
+  private val formatter = DateTimeFormatter.ofPattern("yyyyMMdd_HHmm")s
+
+  def generateLastHour(): List[String] = {
+    val now = LocalDateTime.now
+    generate(now.minusHours(1), now)
+  }
+
+  def generateLast5Hours(): List[String] = {
+    val now = roundToInterval(LocalDateTime.now, false)
+    generate(now.minusHours(5), now)
+  }
+
+  private def roundToInterval(time: LocalDateTime, roundUp: Boolean): LocalDateTime = {
+    val minutes = time.getMinute
+    val adjustment = if (roundUp) (interval.toMinutes - minutes % interval.toMinutes) % interval.toMinutes
+    else -minutes % interval.toMinutes
+
+    time.plusMinutes(adjustment)
+  }
+
+  // not sure either data are each 10 minutes or each 30 mins of hour
+  def generate(startTime: LocalDateTime, endTime: LocalDateTime): List[String] = {
+    val roundStartTime = roundToInterval(startTime, true)
+    val roundEndTime = roundToInterval(endTime, false)
+
+    Iterator.iterate(roundStartTime) { time =>
+      time.plus(interval)
+    }.takeWhile(!_.isAfter(roundEndTime))
+      .filter(_.getMinute == 30)
+      .map(time => s"${formatter.format(time)}.csv")
+      .toList
+  }
+}
@@ -0,0 +1,94 @@
+package parse
+
+import java.time.LocalDateTime
+import scala.reflect.runtime.universe.{termNames, typeOf}
+
+
+case class WeatherStationData(
+                               city: String,
+                               timestamp: LocalDateTime,
+                               meteo: MeteoData,
+                               phenomena: List[String],
+                             )
+
+case class MeteoData(
+                      tempMax: Option[Double],
+                      tempMin: Option[Double],
+                      tempAvg: Option[Double],
+                      precipitation: Option[Double],
+                      windSpeedAvg: Option[Double],
+                      windGustMax: Option[Double],
+                      visibilityMin: Option[Double],
+                      visibilityAvg: Option[Double],
+                      snowThicknessAvg: Option[Double],
+                      atmPressure: Option[Double],
+                      dewPoint: Option[Double],
+                      airHumidity: Option[Double],
+                      sunshineDuration: Option[Double]
+                    )
+
+object MeteoData {
+  def fromDoubles(data: List[Option[Double]]): Option[MeteoData] = data match {
+    case List(
+    tempMax,
+    tempMin,
+    tempAvg,
+    precipitation,
+    windSpeedAvg,
+    windGustMax,
+    visibilityMin,
+    visibilityAvg,
+    snowThicknessAvg,
+    atmPressure,
+    dewPoint,
+    airHumidity,
+    sunshineDuration
+    ) => Some(MeteoData(
+      tempMax,
+      tempMin,
+      tempAvg,
+      precipitation,
+      windSpeedAvg,
+      windGustMax,
+      visibilityMin,
+      visibilityAvg,
+      snowThicknessAvg,
+      atmPressure,
+      dewPoint,
+      airHumidity,
+      sunshineDuration,
+    ))
+    case _ => None
+  }
+
+  def getCount: Int = {
+    val constructor = typeOf[MeteoData].decl(termNames.CONSTRUCTOR).asMethod
+    val paramCount = constructor.paramLists.flatten.size
+    paramCount
+  }
+
+  def getKeys: List[String] = {
+    val constructor = typeOf[MeteoData].decl(termNames.CONSTRUCTOR).asMethod
+    constructor.paramLists.flatten.map(_.name.toString)
+  }
+}
+
+//val cityList: List[String] = List("Ainaži", "Alūksne", "Bauska", "Dagda", "Daugavgrīva", "Daugavpils", "Dobele", "Gulbene", "Jelgava", "Kalnciems", "Kolka", "Kuldīga", "Lielpēči", "Liepāja", "Madona", "Mērsrags", "Pāvilosta", "Piedruja", "Priekuļi", "Rēzekne", "Rīga", "Rucava", "Rūjiena", "Saldus", "Sigulda", "Sīļi", "Skrīveri", "Skulte", "Stende", "Ventspils", "Vičaki", "Zīlāni", "Zosēni", "Ainaži", "Alūksne", "Bauska", "Dagda", "Daugavgrīva", "Daugavpils", "Dobele", "Gulbene", "Jelgava", "Kalnciems", "Kolka", "Kuldīga", "Lielpēči", "Liepāja", "Madona", "Mērsrags", "Pāvilosta", "Piedruja", "Priekuļi", "Rēzekne", "Rīga", "Rucava", "Rūjiena", "Saldus", "Sigulda", "Sīļi", "Skrīveri", "Skulte", "Stende", "Ventspils", "Vičaki", "Zīlāni", "Zosēni")
+
+
+//Map(
+//  "tempMax" -> "Stundas maksimālā temperatūra",
+//  "tempMin" -> "Stundas minimālā temperatūra",
+//  "tempAvg" -> "Stundas vidējā temperatūra",
+//  "precipitation" -> "Stundas nokrišņu daudzums",
+//  "windSpeedAvg" -> "Vidējais vēja ātrums novērojumu termiņā (10 min. vidējais)",
+//  "windGustMax" -> "Stundas maksimālās vēja brāzmas",
+//  "visibilityMin" -> "Stundas minimālā redzamība",
+//  "visibilityAvg" -> "Stundas vidējā redzamība",
+//  "snowThicknessAvg" -> "Stundas vidējais sniega segas biezums",
+//  "atmPressure" -> "Atmosfēras spiediens jūras līmenī novērojuma termiņā (milibāros)",
+//  "dewPoint" -> "Rasas punkta temperatūra novērojuma termiņā",
+//  "airHumidity" -> "Relatīvais gaisa mitrums novērojumu termiņā",
+//  "sunshineDuration" -> "Saules spīdēšanas ilgums",
+////  "weatherPhenomena" -> "Laika parādības",
+//)
@@ -0,0 +1,89 @@
+package parse
+
+import java.io.File
+import java.time.LocalDateTime
+import java.time.format.DateTimeFormatter
+import scala.io.Source
+import scala.util.Try
+
+object Parser {
+  val data_path = "/Users/guntissmaukstelis/sandbox/hello/data/"
+
+  def fileToDateTime(file: File): LocalDateTime = {
+    val dateString = file.toString.split("/").last.split("\\.").head
+    val formatter = DateTimeFormatter.ofPattern("yyyyMMdd_HHmm")
+    LocalDateTime.parse(dateString, formatter)
+  }
+
+  def getFiles(path: String): List[File] = new File(path).listFiles.toList
+
+  def getFilesInRange(start: LocalDateTime, end: LocalDateTime): List[File] = {
+    val fileMap = getFiles(data_path).map(file => fileToDateTime(file) -> file).toMap
+    fileMap
+      .filter { case (k, _) =>
+        k.plusSeconds(1).isAfter(start) && k.minusSeconds(1).isBefore(end)
+      }
+      .map { case (_, v) => v }.toList
+  }
+
+  // TODO I guess this should be wrapped in IO
+  def readFromFile(file: File): List[String] = {
+    val source = Source.fromFile(file)
+    val lineList = source.getLines.toList
+    source.close()
+    lineList.tail // remove header line
+  }
+
+  def parseLine(line: String): Option[WeatherStationData] = {
+    val paramCount = MeteoData.getCount
+
+    def parseTimestamp(timestampStr: String): Option[LocalDateTime] = {
+      val formatter = DateTimeFormatter.ofPattern("yyyyMM.dd HH:mm")
+      Try(LocalDateTime.parse(s"2023${timestampStr.trim}", formatter)).toEither match {
+        case Right(timestamp) => Some(timestamp)
+        case Left(_) => None
+      }
+    }
+
+    def splitParts(parts: List[String]): Option[(String, String, List[String])] = {
+      parts.splitAt(2) match {
+        case (cityStr :: timestampStr :: Nil, rest) if rest.size >= paramCount => Some((cityStr, timestampStr, rest))
+        case _ => None
+      }
+    }
+
+    for {
+      (city, timestampStr, rest) <- splitParts(line.trim.split(";", -1).toList)
+      timestamp <- parseTimestamp(timestampStr)
+      (strList, meteoPhenomenaList) = rest.splitAt(paramCount)
+      meteoData <- MeteoData.fromDoubles(strList.map(_.toDoubleOption))
+      phenomena <- Some(meteoPhenomenaList.map(_.trim))
+    } yield WeatherStationData(city, timestamp, meteoData, phenomena)
+  }
+
+  def readFromVariable(str: String): List[String] = str.split("\n").toList // Data.csv
+
+
+  def main(args: Array[String]): Unit = {
+    println("================ start parser")
+
+    val formatter = DateTimeFormatter.ofPattern("yyyyMMdd_HHmm")
+    val start = LocalDateTime.parse("20230409_2200", formatter)
+    val end = LocalDateTime.parse("20230501_1230", formatter)
+    val files = getFilesInRange(start, end)
+    val weatherStationData = files
+      .flatMap(readFromFile)
+      .flatMap(parseLine)
+
+    //    weatherStationData.foreach(println)
+
+    val liepajaWeather = weatherStationData.filter(_.city == "Liepāja")
+    val maxTempLiepaja = liepajaWeather.flatMap(_.meteo.tempMax).max
+    val minTempLiepaja = liepajaWeather.flatMap(_.meteo.tempMin).min
+    val avgTemps = liepajaWeather.flatMap(_.meteo.tempAvg)
+    val avgTempLiepaja = avgTemps.sum / avgTemps.size
+    println(s"Liepaja max: $maxTempLiepaja, min: $minTempLiepaja, avg: $avgTempLiepaja")
+    //    liepaja.foreach(println)
+
+  }
+}
@@ -0,0 +1,5 @@
+package server
+
+class Server {
+  // TODO proly BlazeServerBuilder which accepts requests from client and serves parsed data
+}