ITPub博客
首页 博客 专栏 学院 社区 IT新媒体评选
发博

首页 > 大数据 > 数据分析 > 使用scala编写wordcount程序--从多个文件中读取计算，最终存入MySQL数据库

使用scala编写wordcount程序--从多个文件中读取计算，最终存入MySQL数据库

原创 数据分析 作者：loveheping 时间：2018-02-12 16:00:49 0 删除 编辑
   啥也不说，直接上程序吧，嘻嘻。。。。。。。。
一、公共类
点击(此处)折叠或打开
  1. package com.ruozedata.scalawroks

  3. import java.io.{FileSystem => _, _}
  4. import org.apache.hadoop.fs._
  5. import scala.collection.mutable.ListBuffer
  6. import org.apache.hadoop.fs.{FileSystem, Path}

  8. object ScalaToHDFSUtils {

  10.   def isDir(fs : FileSystem, dirName : String) : Boolean = {
  11.     fs.isDirectory(new Path(dirName))
  12.   }

  14.   def isDir(fs : FileSystem, dirName : Path) : Boolean = {
  15.     fs.isDirectory(dirName)
  16.   }

  18.   def isFile(fs : FileSystem, fileName : String) : Boolean = {
  19.     fs.isFile(new Path(fileName))
  20.   }

  22.   def isFile(fs : FileSystem, fileName : Path) : Boolean = {
  23.     fs.isFile(fileName)
  24.   }

  26.   def createFile(fs : FileSystem, fileName : String) : Boolean = {
  27.     fs.createNewFile(new Path(fileName))
  28.   }

  30.   def createFile(fs : FileSystem, name : Path) : Boolean = {
  31.     fs.createNewFile(name)
  32.   }

  34.   def createFolder(fs : FileSystem, folderName : String) : Boolean = {
  35.     fs.mkdirs(new Path(folderName))
  36.   }

  38.   def createFolder(fs : FileSystem, folderName : Path) : Boolean = {
  39.     fs.mkdirs(folderName)
  40.   }

  42.   def exists(fs : FileSystem, name : String) : Boolean = {
  43.     fs.exists(new Path(name))
  44.   }

  46.   def exists(fs : FileSystem, name : Path) : Boolean = {
  47.     fs.exists(name)
  48.   }

  50.   def rename(fs : FileSystem, srcPath : String, distPath : String) = {
  51.     fs.rename( new Path( srcPath ), new Path( distPath ) )
  52.   }

  54.   def move(fs : FileSystem, srcPath : String, distPath : String) = {
  55.     fs.rename( new Path( srcPath ), new Path( distPath ) )
  56.   }

  58.   class MyPathFilter extends PathFilter {
  59.     override def accept(path: Path): Boolean = true
  60.   }

  62.   def listFiles(fs : FileSystem, fullName : String, holder : ListBuffer[String],
  63.                 flgFile : Boolean = false) : ListBuffer[String] = {

  65.     val filesStatus = fs.listStatus(new Path(fullName), new MyPathFilter)

  67.     for(status <- filesStatus){
  68.       val filePath : Path = status.getPath
  69.       if(isFile(fs,filePath))
  70.         holder += filePath.toString
  71.       else {
  72.         if(flgFile==true)
  73.           listFiles(fs, filePath.toString, holder)
  74.       }
  75.     }
  76.     holder
  77.   }

  79.   def copyFile(fs : FileSystem, source: String, target: String): Unit = {

  81.     val sourcePath = new Path(source)
  82.     val targetPath = new Path(target)

  84.     if(!exists(fs, targetPath))
  85.       createFile(fs, targetPath)

  87.     val inputStream : FSDataInputStream = fs.open(sourcePath)
  88.     val outputStream : FSDataOutputStream = fs.create(targetPath)
  89.     transport(inputStream, outputStream)
  90.   }

  92.   def transport(inputStream : InputStream, outputStream : OutputStream): Unit ={
  93.     val buffer = new Array[Byte](64 * 1000)
  94.     var len = inputStream.read(buffer)
  95.     try {
  96.       while (len != -1) {
  97.         outputStream.write(buffer, 0, len)
  98.         len = inputStream.read(buffer)
  99.       }
  100.       outputStream.flush()
  101.     } catch {
  102.       case e : IOException => {
  103.         e.printStackTrace
  104.       }
  105.     } finally {
  106.       inputStream.close()
  107.       outputStream.close()
  108.     }
  109.   }

  111.   def readFiles(fs : FileSystem, source: String, values: ListBuffer[String]) : ListBuffer[String] = {
  112.     val sourcePath = new Path(source)
  113.     val inputStream : FSDataInputStream = fs.open(sourcePath)
  114.     var bufferedReader: BufferedReader = new BufferedReader(new InputStreamReader(inputStream))
  115.     var lineTxt : String = bufferedReader.readLine()

  117.     try {
  118.       while (lineTxt != null) {
  119.         values.append(lineTxt)
  120.         lineTxt = bufferedReader.readLine()
  121.       }
  122.       values
  123.     } finally {
  124.       if (bufferedReader != null) {
  125.         bufferedReader.close()
  126.       }
  127.       if (inputStream != null) {
  128.         inputStream.close()
  129.       }
  130.     }
  131.   }
  132. }
二、移动文件类
点击(此处)折叠或打开
  1. package com.ruozedata.scalawroks

  3. import java.io.IOException
  4. import java.net.URI
  5. import java.text.SimpleDateFormat
  6. import java.util.Date

  8. import org.apache.hadoop.fs._
  9. import org.apache.hadoop.conf.Configuration

  11. import scala.collection.mutable.ListBuffer

  13.   object ScalaToHDFSApp extends App {
  14.     val conf = new Configuration()
  15.     var fs: FileSystem = null

  17.     val sourcePath = "/ruozedata/" // 打包jar时，可以取args(0)值
  18.     val targetPath = "/ruozedata/scala/" // 打包jar时，可以取args(1)值
  19.     val uriPath = "hdfs://192.168.137.130:9000/" // 打包jar时，可以取args(2)值

  21.     // 新文件名
  22.     var newFileName = ""

  24.     // 文件序号
  25.     var fSeq = 1

  27.     // 设置访问HDFS用户
  28.     System.setProperty("HADOOP_USER_NAME", "hadoop")

  30.     try {
  31.       fs = FileSystem.get(new URI(uriPath), conf)

  33.       val holder : ListBuffer[String] = new ListBuffer[String]

  35.       // 获指定目录中文件列表(如果为true则级联查找文件,如果为false则只查找当前目录文件)。默认值：false
  36.       val paths : List[String] = ScalaToHDFSUtils.listFiles(fs, sourcePath, holder).toList

  38.       for(path <- paths) {
  39.         // 拼接新文件名
  40.         newFileName = dateToStr + fSeq.toString + ".txt"
  41.         // 文件序号+1
  42.         fSeq += 1
  43.         // 复制文件到指定目录
  44.         ScalaToHDFSUtils.copyFile(fs, sourcePath + new Path(path).getName, targetPath + newFileName)
  45.       }
  46.     } catch {
  47.       case e : IOException => {
  48.         e.printStackTrace
  49.       }
  50.     } finally {
  51.      if (fs != null) {
  52.        fs.close()
  53.      }
  54.   }

  56.   def dateToStr(): String = {
  57.     // 获取系统日期时间，转为字符串年月日时分格式
  58.     val date = new Date
  59.     val dateFormat: SimpleDateFormat = new SimpleDateFormat("yyyyMMddHHmm")
  60.     var newTmpFileName = dateFormat.format(date) + "-"
  61.     newTmpFileName
  62.   }
  63. }
三、读取文件，并计算，最张存到MyQL数据库中

点击(此处)折叠或打开

  1. package com.ruozedata.scalawroks

  3. import java.io.IOException
  4. import java.net.URI
  5. import java.sql.{Connection, DriverManager}

  7. import org.apache.hadoop.conf.Configuration
  8. import org.apache.hadoop.fs.{FileSystem, Path}

  10. import scala.collection.mutable.ListBuffer

  12. object ScalacReadHDFSApp extends App {
  13.   val conf = new Configuration()
  14.   val sourcePath = "/ruozedata/scala/" // 打包jar时，可以取args(0)值
  15.   val uriPath = "hdfs://192.168.137.130:9000/" // 打包jar时，可以取args(1)值
  16.   val holder: ListBuffer[String] = new ListBuffer[String]
  17.   val readHolder: ListBuffer[String] = new ListBuffer[String]

  19.   var fs: FileSystem = null
  20.   var listBuffer: List[String] = null

  22.   // 设置访问HDFS用户
  23.   System.setProperty("HADOOP_USER_NAME", "hadoop")

  25.   try {
  26.     fs = FileSystem.get(new URI(uriPath), conf)

  28.     // 获指定目录中文件列表(如果为true则级联查找文件,如果为false则只查找当前目录文件)。默认值：false
  29.     val paths : List[String] = ScalaToHDFSUtils.listFiles(fs, sourcePath, holder).toList

  31.     for(path <- paths) {
  32.       listBuffer = ScalaToHDFSUtils.readFiles(fs, sourcePath + new Path(path).getName, readHolder).toList
  33.     }

  35.     // 完成WordCount统计
  36.     val wc = listBuffer.flatMap(_.split("\t")).map((_, 1)).groupBy(_._1).mapValues(_.map(_._2).reduce(_ + _))

  38.     // 调用存入MySQL中
  39.     insertMySQL(wc)
  40.   } catch {
  41.     case e: IOException => {
  42.       e.printStackTrace
  43.     }
  44.   }finally {
  45.     if (fs != null) {
  46.       fs.close()
  47.     }
  48.   }

  50.   def insertMySQL(wc : Map[String,Int]): Unit = {
  51.     // connect to the database named "mysql" on the localhost
  52.     val driver: String = "com.mysql.jdbc.Driver"
  53.     val url = "jdbc:mysql://192.168.137.130:3306/works"
  54.     val username = "root"
  55.     val password = "123456"

  57.     var connection : Connection = null
  58.     var query = ""

  60.     try{
  61.       //Class.forName(driver)
  62.       connection = DriverManager.getConnection(url, username, password)
  63.       val ps = connection.createStatement()

  65.       for ((key, value) <- wc) {
  66.         query="insert into wordcounts(words, counts) values('"+key+"' , '"+value+"')"
  67.         ps.addBatch(query)
  68.       }

  70.       ps.executeBatch()
  71.     } catch {
  72.       case e => e.printStackTrace
  73.     } finally {
  74.       connection.close()
  75.     }
  76.   }
  77. }
四、MySQL表创建
点击(此处)折叠或打开
  1. create table wordcounts(words varchar(50), counts varchar(20));
五、Mavne配置(pom.xml)
点击(此处)折叠或打开
  1. <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
  2.   <modelVersion>4.0.0</modelVersion>
  3.   <groupId>testScala</groupId>
  4.   <artifactId>testScala</artifactId>
  5.   <version>1.0</version>
  6.   <inceptionYear>2008</inceptionYear>
  7.   <properties>
  8.     <scala.version>2.11.8</scala.version>
  9.     <hadoop.version>2.6.0-cdh5.7.0</hadoop.version>
  10.     <spark.version>2.2.1</spark.version>
  11.     <mysql.version>5.1.25</mysql.version>
  12.   </properties>

  14.   <repositories>
  15.     <repository>
  16.       <id>scala-tools.org</id>
  17.       <name>Scala-Tools Maven2 Repository</name>
  18.       <url>http://scala-tools.org/repo-releases</url>
  19.     </repository>

  21.     <repository>
  22.       <id>cloudera</id>
  23.       <name>cloudera</name>
  24.       <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
  25.     </repository>
  26.   </repositories>

  28.   <pluginRepositories>
  29.     <pluginRepository>
  30.       <id>scala-tools.org</id>
  31.       <name>Scala-Tools Maven2 Repository</name>
  32.       <url>http://scala-tools.org/repo-releases</url>
  33.     </pluginRepository>
  34.   </pluginRepositories>

  36.   <dependencies>
  37.     <dependency>
  38.       <groupId>org.apache.spark</groupId>
  39.       <artifactId>spark-core_2.11</artifactId>
  40.       <version>${spark.version}</version>
  41.     </dependency>
  42.     <dependency>
  43.       <groupId>org.scala-lang</groupId>
  44.       <artifactId>scala-library</artifactId>
  45.       <version>${scala.version}</version>
  46.     </dependency>
  47.     <dependency>
  48.       <groupId>junit</groupId>
  49.       <artifactId>junit</artifactId>
  50.       <version>4.4</version>
  51.       <scope>test</scope>
  52.     </dependency>
  53.     <dependency>
  54.       <groupId>org.specs</groupId>
  55.       <artifactId>specs</artifactId>
  56.       <version>1.2.5</version>
  57.       <scope>test</scope>
  58.     </dependency>

  60.     <dependency>
  61.       <groupId>org.apache.hadoop</groupId>
  62.       <artifactId>hadoop-client</artifactId>
  63.       <version>${hadoop.version}</version>
  64.     </dependency>

  66.     <dependency>
  67.       <groupId>mysql</groupId>
  68.       <artifactId>mysql-connector-java</artifactId>
  69.       <version>${mysql.version}</version>
  70.     </dependency>
  71.   </dependencies>

  73.   <build>
  74.     <sourceDirectory>src/main/scala</sourceDirectory>
  75.     <testSourceDirectory>src/test/scala</testSourceDirectory>
  76.     <plugins>
  77.       <plugin>
  78.         <groupId>org.scala-tools</groupId>
  79.         <artifactId>maven-scala-plugin</artifactId>
  80.         <executions>
  81.           <execution>
  82.             <goals>
  83.               <goal>compile</goal>
  84.               <goal>testCompile</goal>
  85.             </goals>
  86.           </execution>
  87.         </executions>
  88.         <configuration>
  89.           <scalaVersion>${scala.version}</scalaVersion>
  90.           <args>
  91.             <arg>-target:jvm-1.5</arg>
  92.           </args>
  93.         </configuration>
  94.       </plugin>
  95.       <plugin>
  96.         <groupId>org.apache.maven.plugins</groupId>
  97.         <artifactId>maven-eclipse-plugin</artifactId>
  98.         <configuration>
  99.           <downloadSources>true</downloadSources>
  100.           <buildcommands>
  101.             <buildcommand>ch.epfl.lamp.sdt.core.scalabuilder</buildcommand>
  102.           </buildcommands>
  103.           <additionalProjectnatures>
  104.             <projectnature>ch.epfl.lamp.sdt.core.scalanature</projectnature>
  105.           </additionalProjectnatures>
  106.           <classpathContainers>
  107.             <classpathContainer>org.eclipse.jdt.launching.JRE_CONTAINER</classpathContainer>
  108.             <classpathContainer>ch.epfl.lamp.sdt.launching.SCALA_CONTAINER</classpathContainer>
  109.           </classpathContainers>
  110.         </configuration>
  111.       </plugin>
  112.     </plugins>
  113.   </build>
  114.   <reporting>
  115.     <plugins>
  116.       <plugin>
  117.         <groupId>org.scala-tools</groupId>
  118.         <artifactId>maven-scala-plugin</artifactId>
  119.         <configuration>
  120.           <scalaVersion>${scala.version}</scalaVersion>
  121.         </configuration>
  122.       </plugin>
  123.     </plugins>
  124.   </reporting>
  125. </project>
六、测试数据
hdfs dfs -ls /ruozedata/scala
Found 3 items
-rw-r--r--   3 hadoop supergroup         24 2018-02-12 14:48 /ruozedata/scala/201802121449-1.txt
-rw-r--r--   3 hadoop supergroup         12 2018-02-12 14:48 /ruozedata/scala/201802121449-2.txt
-rw-r--r--   3 hadoop supergroup         12 2018-02-12 14:48 /ruozedata/scala/201802121449-3.txt
-----------------------------------------------------------------------------------------------------------
hdfs dfs -cat /ruozedata/scala/201802121449-1.txt
hello   world
hello   worls

hdfs dfs -cat /ruozedata/scala/201802121449-2.txt
hello   ruoze

hdfs dfs -cat /ruozedata/scala/201802121449-3.txt
ruoze   hello

七、测试结果


注：后续大家有什么好的建议，请提出。


来自@若泽大数据



来自 “ ITPUB博客 ” ，链接：https://blog.itpub.net/31511218/viewspace-2151064/ ，如需转载，请注明出处，否则将追究法律责任。

0
0
上一篇： 远程启动HADOOP+YARN集群Shell脚本
下一篇： hadoop2.6.0-cdh5.7.0编译，支持snappy、bzip2本地压缩
请登录后发表评论 登录
全部评论
loveheping

注册时间：2017-12-13

  • 博文量
    41
  • 访问量
    80861

最新文章

支持我们 作者招募 用户协议 FAQ Contact Us

北京盛拓优讯信息技术有限公司. 版权所有  京ICP备16024965号-8 北京市公安局海淀分局网监中心备案编号：11010802021510 niuxiaotong@pcpop.com 17352615567 未成年人举报专区

广播电视节目制作经营许可证(京) 字第1234号 中国互联网协会会员

违法和不良信息举报电话:18101376593