今天在用IDEA写spark sql时遇到这个从未见过的奇葩问题
贴上代码
package udf_sql
import org.apache.spark.sql.{DataFrame, SparkSession}
object sql_udf {
def main(args: Array[String]): Unit = {
val spark = SparkSession.builder().master("local").appName("UDF").getOrCreate()
val nameList: List[String] = List[String]("zhangsan", "lisi", "wangwu", "zhaoliu", "tianqi")
import spark.implicits._
val nameDF: DataFrame = nameList.toDF("name")
nameDF.createOrReplaceTempView("students")
nameDF.show()
spark.udf register("STRLEN", (name: String) => {
name.length
})
spark.sql("select name ,STRLEN(name) as length from students order by length desc").show(100)
}
}
就在 spark.udf register("STRLEN", (name: String)这里一直报错说No TypeTag available for String sqlContext.udf.register(),一咬牙一狠心改成了java.lang.String就成功辽!!!!!可喜可贺可歌可泣,被这玩意儿卡了几小时的感觉真是不太好...