No Description

AkkaStreamComponents.scala 4.5KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136
  1. package fr.natan.akkastreamfileprocessingapi.service
  2. import akka.actor.ActorSystem
  3. import akka.stream.alpakka.csv.scaladsl.{CsvParsing, CsvToMap}
  4. import akka.stream.scaladsl.{Compression, FileIO, Flow, Sink, Source}
  5. import akka.{Done, NotUsed}
  6. import com.typesafe.scalalogging.slf4j.Logger
  7. import fr.natan.akkastreamfileprocessingapi.businessexceptions.FileNotFoundException
  8. import fr.natan.akkastreamfileprocessingapi.models.Models.{Person, TvSerie}
  9. import fr.natan.akkastreamfileprocessingapi.valitator.Validators.fileExists
  10. import fr.natan.akkastreamfileprocessingapi.models.ModelsBuilder.{buildPersonModel, buildTvSerieModel}
  11. import java.io.File
  12. import java.nio.file.Paths
  13. import scala.concurrent.Future
  14. //noinspection SpellCheckingInspection
  15. object AkkaStreamComponents {
  16. implicit val actor: ActorSystem = ActorSystem("AkkaStreamActor")
  17. //flows building
  18. def buildPersonFlow(): Flow[Map[String, String], Person, NotUsed] = {
  19. val personFlow: Flow[Map[String, String], Person, NotUsed] =
  20. Flow[Map[String, String]]
  21. .map((rowMap: Map[String, String]) => {
  22. buildPersonModel(rowMap)
  23. })
  24. personFlow
  25. }
  26. def filterPersonByIdFlow(personID: String): Flow[Map[String, String], Person, NotUsed]={
  27. val personFlowFilter: Flow[Map[String, String], Person, NotUsed]=
  28. Flow[Map[String, String]]
  29. .filter((rowMap:Map[String, String])=>{
  30. rowMap.getOrElse(key="nconst",default="")==personID
  31. })
  32. .map(rowMap=>{
  33. buildPersonModel(personMap = rowMap)
  34. })
  35. personFlowFilter
  36. }
  37. def filterPersonByNameFlow(primaryName: String): Flow[Map[String, String], Person, NotUsed] ={
  38. val personFlowFilter: Flow[Map[String, String], Person, NotUsed] =
  39. Flow[Map[String, String]]
  40. .filter((rowMap: Map[String, String]) =>{
  41. rowMap.getOrElse(key = "primaryName", default = "")==primaryName
  42. })
  43. .map((rowMap: Map[String, String])=>{
  44. buildPersonModel(personMap = rowMap)
  45. })
  46. personFlowFilter
  47. }
  48. def buildTvSerieFlow(): Flow[Map[String, String], TvSerie, NotUsed] = {
  49. val tvSerieFlow: Flow[Map[String, String], TvSerie, NotUsed] =
  50. Flow[Map[String, String]]
  51. .map((rowMap: Map[String, String]) => {
  52. buildTvSerieModel(tvSerieMap = rowMap)
  53. })
  54. tvSerieFlow
  55. }
  56. def filterTvSerieByIdFlow(tvSerieID: String): Flow[Map[String, String], TvSerie, NotUsed] = {
  57. val tvSerieFlowFilter : Flow[Map[String, String], TvSerie, NotUsed] =
  58. Flow[Map[String, String]]
  59. .filter((rowMap: Map[String, String])=>rowMap.getOrElse(key = "tconst", default = "")==tvSerieID)
  60. .map((rowMap: Map[String, String])=>{
  61. buildTvSerieModel(tvSerieMap = rowMap)
  62. })
  63. tvSerieFlowFilter
  64. }
  65. def filterTvSerieByPrimaryTitleFlow(tvSeriePrimaryTitle: String): Flow[Map[String, String], TvSerie, NotUsed] = {
  66. val filterFlowFilter: Flow[Map[String, String], TvSerie, NotUsed] = Flow[Map[String, String]]
  67. .filter((rows: Map[String, String]) => {
  68. rows.getOrElse(key = "primaryTitle", default = "") == tvSeriePrimaryTitle
  69. })
  70. .map(rowMap => {
  71. val tvSerie: TvSerie = buildTvSerieModel(tvSerieMap = rowMap)
  72. tvSerie
  73. })
  74. filterFlowFilter
  75. }
  76. //source building
  77. def buildSource(inputFile: File): Source[Map[String, String], NotUsed] = {
  78. var datasource: Source[Map[String, String], NotUsed] = null
  79. if (!fileExists(inputFile.getPath)) {
  80. return null
  81. }
  82. datasource = Source
  83. .single(inputFile)
  84. .flatMapConcat((filename: File) => {
  85. FileIO.fromPath(
  86. Paths.get(filename.getPath)
  87. )
  88. }
  89. )
  90. .via(Compression.gunzip())
  91. .via(CsvParsing.lineScanner(CsvParsing.Tab, CsvParsing.DoubleQuote, CsvParsing.DoubleQuote))
  92. .via(CsvToMap.toMapAsStrings())
  93. datasource
  94. }
  95. def buildAndValidateSource(inputFile: File): Source[Map[String, String], _] = {
  96. val source: Source[Map[String, String], _] = buildSource(inputFile = inputFile)
  97. if (source == null) {
  98. throw new FileNotFoundException(filename = inputFile.getPath)
  99. }
  100. source
  101. }
  102. //sinks building
  103. def buildAllTvSeriesSink(logger: Logger): Sink[TvSerie, Future[Done]] = {
  104. val tvSeriesSink: Sink[TvSerie, Future[Done]] = Sink
  105. .foreach((tvSerie: TvSerie)=>logger.info(s"${tvSerie.toString}"))
  106. tvSeriesSink
  107. }
  108. def buildAllPersonsSink(logger: Logger): Sink[Person,Future[Done]] = {
  109. val listPersonsSink: Sink[Person, Future[Done]]=
  110. Sink.foreach((person: Person)=>logger.info(s"${person.toString}"))
  111. listPersonsSink
  112. }
  113. }

Powered by TurnKey Linux.