适用场景：当hadoop配置了默认的hdfs输出压缩格式，而spark的hdfs文件输出不想用hadoop配置的压缩方式

方式1：调用SparkContext的hadoopRDD，得到RDD[key, value]，然后调用RDD[key, value]的saveAsHadoopFile接口

示例如下：

package spark.local.test

import org.apache.hadoop.conf.Configuration

import org.apache.hadoop.fs.{Path, FileSystem}

import org.apache.hadoop.io.{NullWritable, LongWritable, Text}

import org.apache.hadoop.mapred.{JobConf, TextOutputFormat}

import org.apache.hadoop.mapreduce.Job

import org.apache.hadoop.mapred.{FileInputFormat, TextInputFormat, JobConf}

import org.apache.spark.rdd.RDD

import org.apache.spark.{SparkContext, SparkConf}

import org.apache.log4j.{Logger, Level}

/**

* Created by xuhong on 2015/4/22.

*/

object Test1 {

  def main(args: Array[String]): Unit = {

    val sparkConf: SparkConf = new SparkConf().setAppName("test")

    val sc: SparkContext = new SparkContext(sparkConf)

    val hadoopConfig = new Configuration()

    hadoopConfig.set("fs.default.name", "hdfs://mz-hadoop-1.meizu.com:9000")

    val fileSystem: FileSystem = FileSystem.get(hadoopConfig)*/

    val config = new JobConf(fileSystem.getConf())

    config.set("mapred.output.compress", "false") 

    FileInputFormat.setInputPaths(config, new Path(args(1)))

    val rdd0: RDD[(LongWritable, Text)] = sc.hadoopRDD(config, classOf[TextInputFormat], classOf[LongWritable], classOf[Text])

    rdd0.map(value => (NullWritable, value._2)).saveAsHadoopFile(args(2),classOf[NullWritable], classOf[Text], classOf[TextOutputFormat[NullWritable,Text]], config)

  }

}

方式二：直接获取SparkContext的hadoopconfiguration变量，然后设置hadoopconfiguration的压缩方式为false

示例如下；

package spark.local.test

import org.apache.hadoop.conf.Configuration

import org.apache.hadoop.fs.{Path, FileSystem}

import org.apache.hadoop.io.{NullWritable, LongWritable, Text}

import org.apache.hadoop.mapred.{JobConf, TextOutputFormat}

import org.apache.hadoop.mapreduce.Job

import org.apache.hadoop.mapred.{FileInputFormat, TextInputFormat, JobConf}

import org.apache.spark.rdd.RDD

import org.apache.spark.{SparkContext, SparkConf}

import org.apache.log4j.{Logger, Level}

/**

* Created by xuhong on 2015/4/22.

*/

object Test1 {

  def main(args: Array[String]): Unit = {

    val sparkConf: SparkConf = new SparkConf().setAppName("test")

    val sc: SparkContext = new SparkContext(sparkConf)

    val hadoopConfig: Configuration = sc.hadoopConfiguration

    hadoopConfig.set("mapred.output.compress", "false")

    val rdd: RDD[(String)] = sc.textFile(args(0))

    println("count of rdd is **********************************************  " + rdd.count())

    rdd.saveAsTextFile(args(1))

  }

}

小结：第一种方式相对比较繁琐，需要调用hadoopRDD和saveAsHadoopFile接口，RDD里面的元素必须是KV形式；第二种方式则相对简单易用。