import org.apache.spark.sql._
import scala.util.matching.Regex
import java.text.SimpleDateFormat
import java.util.Date
object UserProperty {
def main(args: Array[String]): Unit = {
val spark = SparkSession.builder()
.master("local[2]")
.appName("NPS User Property")
.getOrCreate()
/** 调试阶段设定loglevel=WARN */
spark.sparkContext.setLogLevel("WARN")
val UP_INPUT :String = "hdfs://192.168.61.165:9000/nps/NPS-立即导出-utf8.csv"
val UP_OUTPUT :String = "hdfs://192.168.61.165:9000/nps/output/UP"
val df: DataFrame = spark.read
.option("header", "true")
.csv(UP_INPUT)
.withColumnRenamed("客户出生日期", "birthday")
val dates: Regex = """(\d{4})-(\d{2})-(\d{2})\s(\d{2}):(\d{2}):(\d{2})""".r
val nowdate: Long = nowDate()
val seconds: Long = 365*24*60*60
import spark.implicits._
df.select("birthday").rdd.map(x =>
x(0) match {
case "--" => x(0)
case dates(yy, mm, dd, hh, mi, ss) => (nowdate - s"$yy$mm$dd$hh$mi$ss".toLong)/seconds
case _ => (nowdate - x(0).toString.toLong)/seconds
}
).take(375).foreach(println)
}
def nowDate(): Long = {
val now: Date = new Date()
val dateFormat: SimpleDateFormat = new SimpleDateFormat("yyyyMMddHHmmss")
val date = dateFormat.format(now).toLong
date
}
}
case class Birthday(birthday: Long)
spark.sql("SELECT current_date,(case when birthday like '--' then '--' " +
"when birthday like '%-% %:%' then birthday " +
"ELSE CONCAT(substring(birthday,1,4),'-',substring(birthday,5,2),'-',substring(birthday,7,2)) END)" +
" age FROM user_property")