private static void runBasicDataFrameExample(SparkSession spark) throws AnalysisException {
        // $example off:global_temp_view$
        Properties prop = new Properties();
        prop.setProperty("user","root");
        prop.setProperty("password","admin");


        //read data form mysql novel article
        Dataset df2 = spark.read().jdbc("jdbc:mysql://localhost:3306/igogo", "novel_info", prop);
        df2.createGlobalTempView("novel");
        //Dataset sqlresult = spark.sql("SELECT to_char(create_time, 'yyyymmdd hh24') as d, count(*) FROM global_temp.novel group by d");



//        df2.sqlContext().udf().register("d", (String s) -> s.length(), DataType.INTEGER);
        spark.udf().register("d", (String s) -> { return s.length();}, DataTypes.IntegerType);
        Dataset sqlresult = spark.sql("SELECT id, d(title) as myfunc FROM global_temp.novel having myfunc>10");
        sqlresult.show();
        //config mysql
//        spark.newSession().sql("SELECT * FROM global_temp.people").write().mode(org.apache.spark.sql.SaveMode.Append)
//                .jdbc("jdbc:mysql://localhost:3306/test", "test", prop);
    }

发表回复

您的邮箱地址不会被公开。 必填项已用 * 标注