使用json文件创建 dataframe演示常用操作
java版本：

package cn.spark.study.core;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.DataFrame;
import org.apache.spark.sql.SQLContext;

public class dataframecreate {
public static void main(String[] args) {
SparkConf conf = new SparkConf()
.setAppName("dataframecreate");
JavaSparkContext sc = new JavaSparkContext(conf);
SQLContext sqlContext = new SQLContext(sc);

DataFrame df = sqlContext.read().json("hdfs://master:9000/student.json");

df.show();
df.printSchema();
df.select("name").show();
df.select(df.col("name")).show();
df.select(df.col("name"),df.col("age").plus(3)).show();
df.filter(df.col("age").gt(20)).show();
df.groupBy(df.col("name")).count().show();
}
}

scala版本：

package com.spark.study.core

import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.sql.SQLContext

object dataframecreate {
def main(args:Array[String]){
    val conf = new SparkConf()
                  .setAppName("dataframecreate")

    val sc = new SparkContext(conf)
    val sqlcontext = new SQLContext(sc)
    val df = sqlcontext.read.json("hdfs://master:9000/student.json")
    df.show()
    df.printSchema()
    df.select(df.col("name"),df.col("age").plus(1)).show()
    df.select(df.col("name"), df.col("age") + 1).show()
    df.filter(df.col("age").gt(20)).show()
    df.groupBy("name").count().show()


}
}

ITPub博客

spark系列二：sparksql dataframe常用操作

最新文章