一 说明
已经提前部署好hadoop 集群3.2.1 版本
下载
https://mirror.bit.edu.cn/apache/hive/
这里我们下载的版本是：apache-hive-3.1.2-bin.tar.gz
驱动的jar 包下载
https://dev.mysql.com/downloads/connector/j/
System 要选Platform Independent ，选择后缀tar.gz 的包进行下载
下载
因为我们要用mysql 数据库来存放hive 的元数据，所以需要提前安装好mysql 数据库
https://dev.mysql.com/downloads/mysql/
这里我们选择的版本是：5.7.29
mysql-community-common-5.7.29-1.el6.x86_64.rpm
mysql-community-libs-5.7.29-1.el6.x86_64.rpm
mysql-community-client-5.7.29-1.el6.x86_64.rpm
mysql-community-server-5.7.29-1.el6.x86_64.rpm
二 Mysql 安装
卸载Mariadb
因为Centos 默认支持的数据库是Mariadb, 所以需求先卸载Mariadb
[root@hadoop3 soft]# rpm -qa | grep maria*
marisa-0.2.4-4.el7.x86_64
mariadb-libs-5.5.56-2.el7.x86_64
卸载数据库：
[root@hadoop3 soft]# yum -y remove mari*
安装MySQL 软件安装所需的依赖包
yum -y install gcc* gcc-c++ ncurses* ncurses-devel* cmake* make* perl* bison* libaio-devel* libgcrypt*
上传mysql 安装介质到相应目录
[root@hadoop3 ~]# cd /hadoop/soft/
[root@hadoop3 soft]# ls -rtl
total 1465928
-rw-r--r--. 1 root root 278813748 Jul 8 16:01 apache-hive-3.1.2-bin.tar.gz
-rw-r--r--. 1 root root 3930793 Jul 8 16:02 mysql-connector-java-8.0.20.tar.gz
-rw-r--r--. 1 root root 2596180 Apr 15 10:49 mysql-community-libs-5.7.29-1.el7.x86_64.rpm
-rw-r--r--. 1 root root 4085448 Apr 15 10:54 mysql-community-devel-5.7.29-1.el7.x86_64.rpm
-rw-r--r--. 1 root root 27768112 Apr 15 11:11 mysql-community-client-5.7.29-1.el7.x86_64.rpm
-rw-r--r--. 1 root root 183618644 Apr 15 14:47 mysql-community-server-5.7.29-1.el7.x86_64.rpm
-rw-r--r--. 1 root root 318972 Apr 15 14:52 mysql-community-common-5.7.29-1.el7.x86_64.rpm
安装mysql
[root@hadoop3 soft]# rpm -ivh mysql-community-common-5.7.29-1.el7.x86_64.rpm
[root@hadoop3 soft]# rpm -ivh mysql-community-libs-5.7.29-1.el7.x86_64.rpm
[root@hadoop3 soft]# rpm -ivh mysql-community-devel-5.7.29-1.el7.x86_64.rpm
[root@hadoop3 soft]# rpm -ivh mysql-community-client-5.7.29-1.el7.x86_64.rpm
[root@hadoop3 soft]# rpm -ivh mysql-community-server-5.7.29-1.el7.x86_64.rpm
创建相关目录并赋权
[root@hadoop3 hadoop]# mkdir -p /hadoop/mysql/product/{binlog,relaylog,log,undo,redo,data}
[root@hadoop3 hadoop]# chown -R mysql:mysql /hadoop/mysql/product
修改初始化选项文件
[root@hadoop3 hadoop]# vi /etc/my.cnf
[client] socket = /hadoop/mysql/product/data/mysql.sock [mysql] prompt="\\u@\\h:\\r:\\m:\\s[\\d]>" no-auto-rehash [mysqld] #BASE port = 3306 basedir = /usr datadir = /hadoop/mysql/product/data socket = /hadoop/mysql/product/data/mysql.sock pid-file = /hadoop/mysql/product/data/mysqld.pid skip-name-resolve lower_case_table_names=1 symbolic-links=0 default_time_zone = "+8:00" log_timestamps = SYSTEM open_files_limit = 65535 #Character character-set-server =utf8mb4 #Connection max_connections=1000 max_connect_errors=999999999 interactive_timeout=3600 wait_timeout=3600 #BINLOG server-id = 1 #搭建主从环境两套数据库，server-id需不同 sync_binlog=1 log-bin = /hadoop/mysql/product/binlog/mysql-bin max_binlog_size = 1073741824 binlog_cache_size = 1M binlog_format= ROW expire_logs_days = 7 relay-log=/hadoop/mysql/product/relaylog/mysql-relay-bin #LOG log-error=/hadoop/mysql/product/log/mysqld.err log_output = FILE slow_query_log = ON slow_query_log_file = /hadoop/mysql/product/log/mysql-slow.log long_query_time = 3 general_log = OFF general_log_file = /hadoop/mysql/product/log/mysql-general.log #GTID gtid_mode=on enforce_gtid_consistency=on log-slave-updates=on #INNODB innodb_buffer_pool_size = 12884901888 innodb_buffer_pool_instances = 8 innodb_log_group_home_dir = /hadoop/mysql/product/redo innodb_undo_directory = /hadoop/mysql/product/undo innodb_undo_tablespaces = 2 innodb_log_file_size= 1073741824 innodb_log_files_in_group=3 innodb_flush_method=O_DIRECT innodb_flush_log_at_trx_commit=1 innodb_data_file_path = ibdata1:500M:autoextend innodb_temp_data_file_path = ibtmp1:200M:autoextend #cache buffer key_buffer_size = 32M read_buffer_size = 8M read_rnd_buffer_size = 4M bulk_insert_buffer_size = 64M sort_buffer_size = 4M join_buffer_size = 4M max_allowed_packet = 16M tmp_table_size = 32M query_cache_size = 0 query_cache_type = off
启动数据库服务
安装包自动添加mysql 服务到自启动服务中，但是mysql 在首次安装完毕不会自动启动。需要手动启动数据库服务。
[root@hadoop3 hadoop]# systemctl start mysqld
修改密码
安装完毕，默认安装数据库Root 账号有密码保护，初始密码为随机字符，密码可在错误日志内查看。
[root@hadoop3 mysql]# grep 'temporary password' /hadoop/mysql/product/log/mysqld.err
2020-07-08T17:37:00.359029+08:00 1 [Note] A temporary password is generated for root@localhost: MyzejBjK-6S< -- 初始密码
修改
[root@hadoop3 mysql]# mysqladmin -uroot -p password 'Hzmc321#'
Enter password: -- 输入初始密码
mysqladmin: [Warning] Using a password on the command line interface can be insecure.
Warning: Since password will be sent to server in plain text, use ssl connection to ensure password safety.
注：
MySQL5.7 之后有默认密码策略：密码必须满足：数字、小写字母、大写字母 、特殊字符、长度至少 8 位
测试连接
[root@hadoop3 mysql]# mysql -uroot -pHzmc321#
mysql: [Warning] Using a password on the command line interface can be insecure. Welcome to the MySQL monitor. Commands end with ; or \g. Your MySQL connection id is 3 Server version: 5.7.29-log MySQL Community Server (GPL) Copyright (c) 2000, 2020, Oracle and/or its affiliates. All rights reserved. Oracle is a registered trademark of Oracle Corporation and/or its affiliates. Other names may be trademarks of their respective owners. Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. root@localhost:05:39:58[(none)]>
查看服务是否开机启动
[root@hadoop3 bin]# systemctl is-enabled mysqld
Enabled
可以看到已经设置了开机自启动
如果没有设置，则使用命令systemctl enable mysqld 设置开机自启动。
至此，MySQL 数据库安装完成。
三Hive 安装
Hive 只需在一个节点上部署即可，在三节点的hadoop 集群中，hadoop3 的资源较空闲，故计划将hive 安装在第三个节点
上传介质包到相应目录并解压
[root@hadoop3 soft]# ls -rtl
total 1679204
-rw-r--r--. 1 root root 278813748 Jul 8 16:01 apache-hive-3.1.2-bin.tar.gz
-rw-r--r--. 1 root root 3930793 Jul 8 16:02 mysql-connector-java-8.0.20.tar.gz
解压hive 到相应目录
[root@hadoop3 soft]# tar -xvf apache-hive-3.1.2-bin.tar.gz -C /hadoop/hive/
配置环境变量
vi /etc/profile
添加：
#set hive environment
export HIVE_HOME=/hadoop/hive/apache-hive-3.1.2-bin
export HIVE_CONF_DIR=$HIVE_HOME/conf
export PATH= $PATH:$HIVE_HOME/bin
source /etc/profile 让环境变量生效
hive --version 查看版本号
Jar
包冲突，使用hadoop
下面的jar
包，所以这里重命名掉hive
下的jar
包
[root@hadoop3 lib]# cd /hadoop/hive/apache-hive-3.1.2-bin/lib/
[root@hadoop3 lib]# mv log4j-slf4j-impl-2.10.0.jar log4j-slf4j-impl-2.10.0.jar_bak
重新查看版本，报错
Exception in thread "main" java.lang.NoSuchMethodError: com.google.common.base.Preconditions.checkArgument(ZLjava/lang/String;Ljava/lang/Object;)V
这个是guava.jar 这个包在hadoop 和hive 的版本不统一造成的，检查下hive 下面的guava 的版本为：
而在hadoop3.2.1 下面的版本是:
我们将hadoop 中的高版本替换到hive 中
移除hive 下的包
[root@hadoop3 lib]# mv guava-19.0.jar guava-19.0.jar_bak
将hadoop 下的包拷贝到hive 下
[root@hadoop3 lib]# cp guava-27.0-jre.jar /hadoop/hive/apache-hive-3.1.2-bin/lib/
重新查询
修改配置文件
在修改相应配置文件之前，先创建后续要用到的hdfs 的目录
使用如下目录创建Hive 在HDFS 中的所需目录（hive.metastore.warehouse.dir 对应的目录）并设置权限
[root@hadoop3 ~]# hadoop fs -mkdir -p /hadoop/hive/apache-hive-3.1.2-bin/tmp
[root@hadoop3 ~]# hadoop fs -mkdir -p /hadoop/hive/apache-hive-3.1.2-bin/warehouse
[root@hadoop3 ~]# hadoop fs -chmod -R 777 /hadoop/hive/apache-hive-3.1.2-bin/tmp
[root@hadoop3~]# hadoop fs -chmod -R 777 /hadoop/hive/apache-hive-3.1.2-bin/warehouse
[root@hadoop3 ~]# hadoop fs -ls /
Found 3 items
drwxr-xr-x - root supergroup 0 2020-07-09 11:20 /hadoop
drwxr-xr-x - root supergroup 0 2020-07-03 15:45 /hbase
drwxrwx--- - root supergroup 0 2020-07-09 10:44 /tmproot@hadoop001:~/hadoop-3.2.1/bin# hadoop fs -chmod g+w /hadoop/hive/apache-hive-3.1.2-bin/warehouse
配置hive-site.xml
该文件主要配置mysql 地址，mysql 主要是存一些元数据的数据库，mysql 提前安装好
javax.jdo.option.ConnectionUserName 数据库的用户名
javax.jdo.option.ConnectionPassword 数据库帐号的密码
javax.jdo.option.ConnectionURL 数据库地址
javax.jdo.option.ConnectionDriverName 数据库连接驱动
将 hive-site.xml 文件中的 ${system:java.io.tmpdir} 替换为hive 的本地临时目录，例如我使用的是/hadoop/hive/apache-hive-3.1.2-bin/temp ，如果该目录不存在，需要先进行创建，并且赋予读写权限。
[root@hadoop3 ~]# mkdir /hadoop/hive/apache-hive-3.1.2-bin/temp
[root@hadoop3 ~]# chmod -R 777 /hadoop/hive/apache-hive-3.1.2-bin/temp
cp hive-default.xml.template hive-site.xml
原始文件很大，我没可以选择直接新建一个空的文件进行编辑
[root@hadoop3 conf]# touch hive-site.xml
[root@hadoop3 conf]# vi hive-site.xml
添加如下内容
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<property>
<name>hive.metastore.warehouse.dir</name>
<value>/hadoop/hive/apache-hive-3.1.2-bin/warehouse</value>
</property>
<!--url-->
<property>
<name>javax.jdo.option.ConnectionURL</name>
<value>jdbc:mysql://hadoop3:3306/hive?createDatabaseIfNotExist=true&useSSL=false</value>
</property>
<!--mysql用户名-->
<property>
<name>javax.jdo.option.ConnectionUserName</name>
<value>root</value>
</property>
<!--mysql中root用户密码-->
<property>
<name>javax.jdo.option.ConnectionPassword</name>
<value>Hzmc321#</value>
</property>
<!--mysql驱动-->
<!--数据库的驱动类名称-->
<!--新版本8.0版本的驱动为com.mysql.cj.jdbc.Driver-->
<!--旧版本5.x版本的驱动为com.mysql.jdbc.Driver-->
<property>
<name>javax.jdo.option.ConnectionDriverName</name>
<value>com.mysql.cj.jdbc.Driver</value>
</property>
<property>
<name>hive.exec.local.scratchdir</name>
<value>/hadoop/hive/apache-hive-3.1.2-bin/temp/root</value>
</property>
<property>
<name>hive.downloaded.resources.dir</name>
<value>/hadoop/hive/apache-hive-3.1.2-bin/temp/${hive.session.id}_resources</value>
</property>
<property>
<name>hive.server2.logging.operation.log.location</name>
<value>/hadoop/hive/apache-hive-3.1.2-bin/temp/root/operation_logs</value>
</property>
<property>
<name>hive.querylog.location</name>
<value>/hadoop/hive/apache-hive-3.1.2-bin/temp/root</value>
</property>
</configuration>
将MySQL8 的驱动包拷贝到Hive 安装目录的lib 目录中
[root@hadoop3 soft]# tar -xvf mysql-connector-java-8.0.20.tar.gz
[root@hadoop3 soft]# cd mysql-connector-java-8.0.20
[root@hadoop3 mysql-connector-java-8.0.20]# cp mysql-connector-java-8.0.20.jar /hadoop/hive/apache-hive-3.1.2-bin/lib/
修改hive-env.sh
[root@hadoop3 conf]# cp hive-env.sh.template hive-env.sh
[root@hadoop3 conf]# vi hive-env.sh
添加如下内容
#hadoop 安装目录 export HADOOP_HOME=/hadoop/hadoop-3.2.1 #hive 配置文件目录 export HIVE_CONF_DIR=/hadoop/hive/apache-hive-3.1.2-bin/conf #hive 依赖jar包目录 export HIVE_AUX_JARS_PATH=/hadoop/hive/apache-hive-3.1.2-bin/lib
配置hive-log4j2.properties
[root@hadoop3 conf]# cp hive-log4j2.properties.template hive-log4j2.properties
[root@hadoop3 conf]# vi hive-log4j2.properties
修改内容
property.hive.log.dir = /hadoop/hive/apache-hive-3.1.2-bin/temp/root
数据库初始化、启动和测试
[root@hadoop3 conf]# schematool -dbType mysql -initSchema
没有远程访问mysql 数据库的权限
[root@hadoop3 ~]# mysql -uroot -p
Enter password: root@localhost:12:29:59[(none)]> root@localhost:12:30:00[(none)]>show databases; +--------------------+ | Database | +--------------------+ | information_schema | | mysql | | performance_schema | | sys | +--------------------+ 4 rows in set (0.19 sec) root@localhost:12:30:08[(none)]>use mysql Database changed root@localhost:12:30:13[mysql]>select user, authentication_string, host from user; +---------------+-------------------------------------------+-----------+ | user | authentication_string | host | +---------------+-------------------------------------------+-----------+ | root | *1373DBE8BAF1B4AAFF2A6EE94BF22769E7477566 | localhost | | mysql.session | *THISISNOTAVALIDPASSWORDTHATCANBEUSEDHERE | localhost | | mysql.sys | *THISISNOTAVALIDPASSWORDTHATCANBEUSEDHERE | localhost | +---------------+-------------------------------------------+-----------+ 3 rows in set (0.00 sec) root@localhost:12:30:15[mysql]>update user set host = '%' where user = 'root'; Query OK, 1 row affected (0.05 sec) Rows matched: 1 Changed: 1 Warnings: 0 root@localhost:12:30:50[mysql]>FLUSH PRIVILEGES; Query OK, 0 rows affected (0.02 sec) root@localhost:12:30:59[mysql]>select user, authentication_string, host from user; +---------------+-------------------------------------------+-----------+ | user | authentication_string | host | +---------------+-------------------------------------------+-----------+ | root | *1373DBE8BAF1B4AAFF2A6EE94BF22769E7477566 | % | | mysql.session | *THISISNOTAVALIDPASSWORDTHATCANBEUSEDHERE | localhost | | mysql.sys | *THISISNOTAVALIDPASSWORDTHATCANBEUSEDHERE | localhost | +---------------+-------------------------------------------+-----------+ 3 rows in set (0.00 sec)
修改权限后重新初始化
[root@hadoop3 conf]# schematool -dbType mysql -initSchema
Metastore connection URL: jdbc:mysql://hadoop3:3306/hive?createDatabaseIfNotExist=true&useSSL=false Metastore Connection Driver : com.mysql.cj.jdbc.Driver Metastore connection User: root Starting metastore schema initialization to 3.1.0 Initialization script hive-schema-3.1.0.mysql.sql Initialization script completed schemaTool completed
数据库初始化完成之后，会在MySQL8 数据库里生成如下metadata 表用于存储Hive 的元数据信息
3.4 使用hive
[root@hadoop3 lib]# hive
2020-07-09 12:44:16,884 INFO [main] conf.HiveConf: Found configuration file file:/hadoop/hive/apache-hive-3.1.2-bin/conf/hive-site.xml Hive Session ID = d1f51bdc-ec09-43dd-b6fe-2055e565d5bc 2020-07-09 12:44:21,105 INFO [main] SessionState: Hive Session ID = d1f51bdc-ec09-43dd-b6fe-2055e565d5bc Logging initialized using configuration in file:/hadoop/hive/apache-hive-3.1.2-bin/conf/hive-log4j2.properties Async: true 2020-07-09 12:44:21,168 INFO [main] SessionState: Logging initialized using configuration in file:/hadoop/hive/apache-hive-3.1.2-bin/conf/hive-log4j2.properties Async: true 2020-07-09 12:44:22,684 INFO [main] session.SessionState: Created HDFS directory: /tmp/hive/root/d1f51bdc-ec09-43dd-b6fe-2055e565d5bc 2020-07-09 12:44:22,720 INFO [main] session.SessionState: Created local directory: /hadoop/hive/apache-hive-3.1.2-bin/temp/root/d1f51bdc-ec09-43dd-b6fe-2055e565d5bc 2020-07-09 12:44:22,730 INFO [main] session.SessionState: Created HDFS directory: /tmp/hive/root/d1f51bdc-ec09-43dd-b6fe-2055e565d5bc/_tmp_space.db …… 2020-07-09 12:44:35,646 INFO [pool-10-thread-1] HiveMetaStore.audit: ugi=root ip=unknown-ip-addr cmd=get_tables_by_type: db=@hive#default pat=.*,type=MATERIALIZED_VIEW 2020-07-09 12:44:35,688 INFO [pool-10-thread-1] metastore.HiveMetaStore: 1: get_multi_table : db=default tbls= 2020-07-09 12:44:35,688 INFO [pool-10-thread-1] HiveMetaStore.audit: ugi=root ip=unknown-ip-addr cmd=get_multi_table : db=default tbls= 2020-07-09 12:44:35,704 INFO [pool-10-thread-1] metadata.HiveMaterializedViewsRegistry: Materialized views registry has been initialized hive>
来自 “ ITPUB博客 ” ，链接：https://blog.itpub.net/29956245/viewspace-2933095/ ，如需转载，请注明出处，否则将追究法律责任。
