package com.gm.hive.SparkHive; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.SaveMode; import org.apache.spark.sql.SparkSession; import java.util.Properties; /** * Spark sql获取Hive数据 * */ public class App { public static void main(String[] args) { SparkSession spark = SparkSession.builder().master("local[2]") .appName("SparkHive") .config("spark.sql.warehouse.dir", "/user/hive/warehouse/").enableHiveSupport() .getOrCreate(); //spark.sql.warehouse.dir为hive的hive.metastore.warehouse.dir路径 spark.sql("show databases").show(); spark.sql("show tables").show(); spark.sql("use db_hive_edu"); Dataset<Row> data = spark .sql("select hc_storetypeid as typeid,count(hc_storetypeid) as kczs from db_hive_edu.hc_casewoodlist where hc_wpstate=2 and hc_storetypeid !='null' group by hc_storetypeid order by hc_storetypeid"); data.show(); //数据库内容 String url = "jdbc:postgresql://192.168.174.200:5432/postgres?charSet=utf-8"; Properties connectionProperties = new Properties(); connectionProperties.put("user","postgres"); connectionProperties.put("password","postgres"); connectionProperties.put("driver","org.postgresql.Driver"); //将数据通过覆盖的形式保存在数据表中 data.write().mode(SaveMode.Overwrite).jdbc(url, "kczyqktj", connectionProperties); } }
Spark2.0 Java实现将Hive运算结果保存到数据库
猜你喜欢
转载自blog.csdn.net/ctwy291314/article/details/80223460
今日推荐
周排行