Sqoop将Mysql数据导入Hadoop测试记录

1、使用query将myql数据导到hdfs目录

/usr/lib/sqoop-current/bin/sqoop import \
--connect jdbc:mysql://MysqlHost:6605/jellyfish_hadoop_stat \
--username MysqlUser \
--password MysqlPass \
--query 'select operator_uid,uid,room_id,fans_count,first_rec_start_date,first_rec_end_date,amount,created_time from invite_anchor_sum where $CONDITIONS;' \
--target-dir /tmp/nisj/sqoop/invite_anchor_sum \
--delete-target-dir \
--num-mappers 1 \
--compress \
--compression-codec org.apache.hadoop.io.compress.SnappyCodec \
--direct \
--fields-terminated-by '\t'

hadoop dfs -ls /tmp/nisj/sqoop/
hadoop dfs -ls /tmp/nisj/sqoop/invite_anchor_sum
--query(好像只能到hdfs)Must specify destination with --target-dir.

sqoop import --connect jdbc:mysql://MysqlHost:6605/jellyfish_hadoop_stat?zeroDateTimeBehavior=convertToNull  --username MysqlUser --password MysqlPass --query "select report_date,xappkey,login_count,new_count from game_cps_report where app_source=0 and \$CONDITIONS" --target-dir /tmp/nisj/sqoop/query1 --delete-target-dir --driver com.mysql.jdbc.Driver -m 1 --where "app_source=0"
sqoop import --connect jdbc:mysql://MysqlHost:6605/jellyfish_hadoop_stat?zeroDateTimeBehavior=convertToNull  --username MysqlUser --password MysqlPass --query "select report_date,xappkey,login_count,new_count from game_cps_report where app_source=0 and \$CONDITIONS" --target-dir /tmp/nisj/sqoop/query1 --delete-target-dir --create-hive-table --hive-overwrite --hive-database ods --hive-table query_game_cps_report --driver com.mysql.jdbc.Driver -m 1 --where "app_source=0"

2、mysql数据导入到hive库表中
--mysql数据导入到hive默认库默认表中：

sqoop import --connect jdbc:mysql://MysqlHost:6605/jellyfish_hadoop_stat --username MysqlUser --password MysqlPass --table game_click_report --hive-import --create-hive-table --driver com.mysql.jdbc.Driver -m 1

--mysql数据导入到指定hdfs文件中：

sqoop import --connect jdbc:mysql://MysqlHost:6605/jellyfish_hadoop_stat --username MysqlUser --password MysqlPass --table game_click_report --direct --target-dir /tmp/nisj/sqoop/game_click_report --delete-target-dir --driver com.mysql.jdbc.Driver --input-fields-terminated-by '\t' --lines-terminated-by '\n'

--mysql数据导入到hive指定库指定表中：

sqoop import --connect jdbc:mysql://MysqlHost:6605/jellyfish_hadoop_stat --username MysqlUser --password MysqlPass --table game_click_report --hive-import --create-hive-table --hive-overwrite --hive-database ods --hive-table xxx_game_click_report --driver com.mysql.jdbc.Driver -m 1
sqoop import --connect jdbc:mysql://MysqlHost:6605/jellyfish_hadoop_stat --username MysqlUser --password MysqlPass --table game_click_report --hive-import --delete-target-dir --hive-database ods --hive-table xxx_game_click_report --driver com.mysql.jdbc.Driver -m 1
sqoop import --connect jdbc:mysql://MysqlHost:6605/jellyfish_hadoop_stat --username MysqlUser --password MysqlPass --table game_click_report --hive-import --delete-target-dir --hive-overwrite --hive-database ods --hive-table xxx_game_click_report --driver com.mysql.jdbc.Driver -m 1

--数据导入异常的处理：
java.sql.SQLException: Value '0000-00-00 00:00:00' can not be represented as java.sql.Timestamp
sqoop import --connect jdbc:mysql://MysqlHost:6605/jellyfish_hadoop_stat ?zeroDateTimeBehavior=convertToNull

sqoop import --connect jdbc:mysql://MysqlHost:6605/jellyfish_hadoop_stat?zeroDateTimeBehavior=convertToNull  --username MysqlUser --password MysqlPass --table game_cps_report --hive-import --create-hive-table  --delete-target-dir --hive-overwrite --hive-database ods --driver com.mysql.jdbc.Driver -m 1

--mysql导入表带where条件

sqoop import --connect jdbc:mysql://MysqlHost:6605/jellyfish_hadoop_stat?zeroDateTimeBehavior=convertToNull  --username MysqlUser --password MysqlPass --table game_cps_report --hive-import --create-hive-table  --delete-target-dir --hive-overwrite --hive-database ods --driver com.mysql.jdbc.Driver -m 1 --where "app_source=0"
sqoop import --connect jdbc:mysql://MysqlHost:6605/jellyfish_hadoop_stat?zeroDateTimeBehavior=convertToNull  --username MysqlUser --password MysqlPass --table game_cps_report --hive-import --delete-target-dir --hive-overwrite --hive-database ods --driver com.mysql.jdbc.Driver -m 1 --where "app_source=0"

3、关于普通导入是否先建表还是导入过程中建表
--只是在Hive上建表而不导入数据

sqoop create-hive-table --connect jdbc:mysql://MysqlHost:6605/jellyfish_hadoop_stat --username MysqlUser --password MysqlPass --table game_click_report --create-hive-table --hive-database ods --hive-table game_click_report
sqoop create-hive-table --connect jdbc:mysql://MysqlHost:6605/jellyfish_hadoop_stat --username MysqlUser --password MysqlPass --table game_click_report --hive-database ods --hive-table game_click_report

--建表且导入数据

sqoop import --connect jdbc:mysql://MysqlHost:6605/jellyfish_hadoop_stat --username MysqlUser --password MysqlPass --table game_click_report --create-hive-table --hive-import --hive-overwrite --hive-database ods --hive-table game_click_report

--建好表导入数据

sqoop import --connect jdbc:mysql://MysqlHost:6605/jellyfish_hadoop_stat --username MysqlUser --password MysqlPass --table game_click_report --hive-import --hive-overwrite --hive-database ods --hive-table game_click_report

4、mysql数据导入分区表
导入普通表示例：

sqoop import \
--connect jdbc:mysql://MysqlHost:6605/jellyfish_hadoop_stat?zeroDateTimeBehavior=convertToNull \
--username MysqlUser \
--password MysqlPass \
--table game_click_report \
--hive-import \
--delete-target-dir \
--hive-overwrite \
--hive-database ods \
--hive-table xxx_game_click_report \
--driver com.mysql.jdbc.Driver -m 1 \
--where "Report_Date='2017-01-10'"

--分区表建表：

drop table if exists xxx_game_click_report;
CREATE TABLE `xxx_game_click_report`(
  `id` bigint, 
  `report_date` string, 
  `game` int, 
  `click_count` int)
PARTITIONED BY ( 
  `data_day` varchar(10));
  
alter table xxx_game_click_report add partition (data_day='2017-01-10');
alter table xxx_game_click_report add partition (data_day='2017-01-09');

--mysql数据导入分区表：

sqoop import \
--connect jdbc:mysql://MysqlHost:6605/jellyfish_hadoop_stat?zeroDateTimeBehavior=convertToNull \
--username MysqlUser \
--password MysqlPass \
--table game_click_report \
--hive-import \
--delete-target-dir \
--hive-overwrite \
--hive-database ods \
--hive-table xxx_game_click_report \
--hive-partition-key data_day \
--hive-partition-value 2017-01-09 \
--driver com.mysql.jdbc.Driver -m 1 \
--where "Report_Date='2017-01-09'"

5、SQL测试及生成Java相关
sqoop eval --connect jdbc:mysql://MysqlHost:6605/jellyfish_hadoop_stat --username MysqlUser --password MysqlPass -e "select * from game_cps_report where app_source=0"
sqoop eval --connect jdbc:mysql://MysqlHost:6605/jellyfish_hadoop_stat --username MysqlUser --password MysqlPass -query "select report_date,xappkey,login_count,new_count from game_cps_report where app_source=0"
sqoop codegen --connect jdbc:mysql://MysqlHost:6605/jellyfish_hadoop_stat --username MysqlUser --password MysqlPass --table game_click_report

Sqoop将Mysql数据导入Hadoop测试记录

猜你喜欢